Terminal: change command-line parser (#2247)

Change the underlying command line handling:
- switch to a custom parser, inspired by redis / sds
- update terminalRegisterCommand signature, pass only bare minimum
- clean-up `help` & `commands`. update settings `set`, `get` and `del`
- allow our custom test suite to run command-line tests
- clean-up Stream IO to allow us to print large things into debug stream (for example, `eeprom.dump`)
- send parsing errors to the debug log

As a proof of concept, introduce `TERMINAL_MQTT_SUPPORT` and `TERMINAL_WEB_API_SUPPORT`
- MQTT subscribes to the `<root>/cmd/set` and sends response to the `<root>/cmd`. We can't output too much, as we don't have any large-send API.
- Web API listens to the `/api/cmd?apikey=...&line=...` (or PUT, params inside the body). This one is intended as a possible replacement of the `API_SUPPORT`. Internals introduce a 'task' around the AsyncWebServerRequest object that will simulate what WiFiClient does and push data into it continuously, switching between CONT and SYS.

Both are experimental. We only accept a single command and not every command is updated to use Print `ctx.output` object. We are also somewhat limited by the Print / Stream overall, perhaps I am overestimating the usefulness of Arduino compatibility to such an extent :)
Web API handler can also sometimes show only part of the result, whenever the command tries to yield() by itself waiting for something. Perhaps we would need to create a custom request handler for that specific use-case.

code/espurna/api.cpp

@@ -8,6 +8,8 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 
 #include "api.h"
 
+// -----------------------------------------------------------------------------
+
 #if API_SUPPORT
 
 #include <vector>
@@ -15,68 +17,25 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #include "system.h"
 #include "web.h"
 #include "rpc.h"
-#include "ws.h"
 
 struct web_api_t {
-    char * key;
-    api_get_callback_f getFn = NULL;
-    api_put_callback_f putFn = NULL;
+    explicit web_api_t(const String& key, api_get_callback_f getFn, api_put_callback_f putFn) :
+        key(key),
+        getFn(getFn),
+        putFn(putFn)
+    {}
+    web_api_t() = delete;
+
+    const String key;
+    api_get_callback_f getFn;
+    api_put_callback_f putFn;
 };
 std::vector<web_api_t> _apis;
 
-// -----------------------------------------------------------------------------
-
-bool _apiEnabled() {
-    return getSetting("apiEnabled", 1 == API_ENABLED);
-}
-
-bool _apiRestFul() {
-    return getSetting("apiRestFul", 1 == API_RESTFUL);
-}
-
-String _apiKey() {
-    return getSetting("apiKey", API_KEY);
-}
-
-bool _apiWebSocketOnKeyCheck(const char * key, JsonVariant& value) {
-    return (strncmp(key, "api", 3) == 0);
-}
-
-void _apiWebSocketOnConnected(JsonObject& root) {
-    root["apiEnabled"] = _apiEnabled();
-    root["apiKey"] = _apiKey();
-    root["apiRestFul"] = _apiRestFul();
-    root["apiRealTime"] = getSetting("apiRealTime", 1 == API_REAL_TIME_VALUES);
-}
-
-void _apiConfigure() {
-    // Nothing to do
-}
-
 // -----------------------------------------------------------------------------
 // API
 // -----------------------------------------------------------------------------
 
-bool _authAPI(AsyncWebServerRequest *request) {
-
-    const auto key = _apiKey();
-    if (!key.length() || !_apiEnabled()) {
-        DEBUG_MSG_P(PSTR("[WEBSERVER] HTTP API is not enabled\n"));
-        request->send(403);
-        return false;
-    }
-
-    AsyncWebParameter* keyParam = request->getParam("apikey", (request->method() == HTTP_PUT));
-    if (!keyParam || !keyParam->value().equals(key)) {
-        DEBUG_MSG_P(PSTR("[WEBSERVER] Wrong / missing apikey parameter\n"));
-        request->send(403);
-        return false;
-    }
-
-    return true;
-
-}
-
 bool _asJson(AsyncWebServerRequest *request) {
     bool asJson = false;
     if (request->hasHeader("Accept")) {
@@ -102,19 +61,18 @@ void _onAPIsText(AsyncWebServerRequest *request) {
     request->send(response);
 }
 
-constexpr const size_t API_JSON_BUFFER_SIZE = 1024;
+constexpr size_t ApiJsonBufferSize = 1024;
 
 void _onAPIsJson(AsyncWebServerRequest *request) {
 
-
-    DynamicJsonBuffer jsonBuffer(API_JSON_BUFFER_SIZE);
+    DynamicJsonBuffer jsonBuffer(ApiJsonBufferSize);
     JsonObject& root = jsonBuffer.createObject();
 
     constexpr const int BUFFER_SIZE = 48;
 
     for (unsigned int i=0; i < _apis.size(); i++) {
         char buffer[BUFFER_SIZE] = {0};
-        int res = snprintf(buffer, sizeof(buffer), "/api/%s", _apis[i].key);
+        int res = snprintf(buffer, sizeof(buffer), "/api/%s", _apis[i].key.c_str());
         if ((res < 0) || (res > (BUFFER_SIZE - 1))) {
             request->send(500);
             return;
@@ -130,7 +88,7 @@ void _onAPIsJson(AsyncWebServerRequest *request) {
 void _onAPIs(AsyncWebServerRequest *request) {
 
     webLog(request);
-    if (!_authAPI(request)) return;
+    if (!apiAuthenticate(request)) return;
 
     bool asJson = _asJson(request);
 
@@ -146,7 +104,7 @@ void _onAPIs(AsyncWebServerRequest *request) {
 void _onRPC(AsyncWebServerRequest *request) {
 
     webLog(request);
-    if (!_authAPI(request)) return;
+    if (!apiAuthenticate(request)) return;
 
     //bool asJson = _asJson(request);
     int response = 404;
@@ -187,19 +145,18 @@ bool _apiRequestCallback(AsyncWebServerRequest *request) {
     // Not API request
     if (!url.startsWith("/api/")) return false;
 
-    for (unsigned char i=0; i < _apis.size(); i++) {
+    for (auto& api : _apis) {
 
-        // Search API url
-        web_api_t api = _apis[i];
+        // Search API url for the exact match
         if (!url.endsWith(api.key)) continue;
 
         // Log and check credentials
         webLog(request);
-        if (!_authAPI(request)) return false;
+        if (!apiAuthenticate(request)) return false;
 
         // Check if its a PUT
         if (api.putFn != NULL) {
-            if (!_apiRestFul() || (request->method() == HTTP_PUT)) {
+            if (!apiRestFul() || (request->method() == HTTP_PUT)) {
                 if (request->hasParam("value", request->method() == HTTP_PUT)) {
                     AsyncWebParameter* p = request->getParam("value", request->method() == HTTP_PUT);
                     (api.putFn)((p->value()).c_str());
@@ -224,9 +181,9 @@ bool _apiRequestCallback(AsyncWebServerRequest *request) {
         if (_asJson(request)) {
             char buffer[64];
             if (isNumber(value)) {
-                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": %s }"), api.key, value);
+                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": %s }"), api.key.c_str(), value);
             } else {
-                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": \"%s\" }"), api.key, value);
+                snprintf_P(buffer, sizeof(buffer), PSTR("{ \"%s\": \"%s\" }"), api.key.c_str(), value);
             }
             request->send(200, "application/json", buffer);
         } else {
@@ -243,25 +200,13 @@ bool _apiRequestCallback(AsyncWebServerRequest *request) {
 
 // -----------------------------------------------------------------------------
 
-void apiRegister(const char * key, api_get_callback_f getFn, api_put_callback_f putFn) {
-
-    // Store it
-    web_api_t api;
-    api.key = strdup(key);
-    api.getFn = getFn;
-    api.putFn = putFn;
-    _apis.push_back(api);
-
+void apiRegister(const String& key, api_get_callback_f getFn, api_put_callback_f putFn) {
+    _apis.emplace_back(key, std::move(getFn), std::move(putFn));
 }
 
 void apiSetup() {
-    _apiConfigure();
-    wsRegister()
-        .onVisible([](JsonObject& root) { root["apiVisible"] = 1; })
-        .onConnected(_apiWebSocketOnConnected)
-        .onKeyCheck(_apiWebSocketOnKeyCheck);
     webRequestRegister(_apiRequestCallback);
-    espurnaRegisterReload(_apiConfigure);
 }
 
 #endif // API_SUPPORT
+

code/espurna/api.h

@@ -11,18 +11,28 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #include "espurna.h"
 #include "web.h"
 
-#include <functional>
+#if WEB_SUPPORT
+
+bool apiAuthenticate(AsyncWebServerRequest*);
+bool apiEnabled();
+bool apiRestFul();
+String apiKey();
+
+#endif // WEB_SUPPORT == 1
 
 #if WEB_SUPPORT && API_SUPPORT
 
+#include <functional>
+
 #include <ESPAsyncTCP.h>
 #include <ArduinoJson.h>
 
 using api_get_callback_f = std::function<void(char * buffer, size_t size)>;
 using api_put_callback_f = std::function<void(const char * payload)> ;
 
-void apiRegister(const char * key, api_get_callback_f getFn, api_put_callback_f putFn = nullptr);
+void apiRegister(const String& key, api_get_callback_f getFn, api_put_callback_f putFn = nullptr);
 
+void apiCommonSetup();
 void apiSetup();
 
 #endif // API_SUPPORT == 1

code/espurna/api_common.cpp

@@ -0,0 +1,79 @@
+/*
+
+Part of the API MODULE
+
+Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#include "espurna.h"
+
+#include "api.h"
+
+#include "ws.h"
+#include "web.h"
+
+// -----------------------------------------------------------------------------
+
+#if WEB_SUPPORT
+
+namespace {
+
+bool _apiWebSocketOnKeyCheck(const char * key, JsonVariant& value) {
+    return (strncmp(key, "api", 3) == 0);
+}
+
+void _apiWebSocketOnConnected(JsonObject& root) {
+    root["apiEnabled"] = apiEnabled();
+    root["apiKey"] = apiKey();
+    root["apiRestFul"] = apiRestFul();
+    root["apiRealTime"] = getSetting("apiRealTime", 1 == API_REAL_TIME_VALUES);
+}
+
+}
+
+// -----------------------------------------------------------------------------
+// Public API
+// -----------------------------------------------------------------------------
+
+bool apiEnabled() {
+    return getSetting("apiEnabled", 1 == API_ENABLED);
+}
+
+bool apiRestFul() {
+    return getSetting("apiRestFul", 1 == API_RESTFUL);
+}
+
+String apiKey() {
+    return getSetting("apiKey", API_KEY);
+}
+
+bool apiAuthenticate(AsyncWebServerRequest *request) {
+
+    const auto key = apiKey();
+    if (!apiEnabled() || !key.length()) {
+        DEBUG_MSG_P(PSTR("[WEBSERVER] HTTP API is not enabled\n"));
+        request->send(403);
+        return false;
+    }
+
+    AsyncWebParameter* keyParam = request->getParam("apikey", (request->method() == HTTP_PUT));
+    if (!keyParam || !keyParam->value().equals(key)) {
+        DEBUG_MSG_P(PSTR("[WEBSERVER] Wrong / missing apikey parameter\n"));
+        request->send(403);
+        return false;
+    }
+
+    return true;
+
+}
+
+void apiCommonSetup() {
+    wsRegister()
+        .onVisible([](JsonObject& root) { root["apiVisible"] = 1; })
+        .onConnected(_apiWebSocketOnConnected)
+        .onKeyCheck(_apiWebSocketOnKeyCheck);
+}
+
+#endif // WEB_SUPPORT == 1

code/espurna/config/dependencies.h

@@ -121,6 +121,20 @@
 #define RELAY_SUPPORT               1           // Most of the time we require it
 #endif
 
+#if TERMINAL_WEB_API_SUPPORT
+#undef TERMINAL_SUPPORT
+#define TERMINAL_SUPPORT            1           // Need terminal command line parser and commands
+#undef WEB_SUPPORT
+#define WEB_SUPPORT                 1           // Registered as web server request handler
+#endif
+
+#if TERMINAL_MQTT_SUPPORT
+#undef TERMINAL_SUPPORT
+#define TERMINAL_SUPPORT            1           // Need terminal command line parser and commands
+#undef MQTT_SUPPORT
+#define MQTT_SUPPORT                1           // Subscribe and publish things
+#endif
+
 //------------------------------------------------------------------------------
 // Hint about ESPAsyncTCP options and our internal one
 // TODO: clean-up SSL_ENABLED and USE_SSL settings for 1.15.0

code/espurna/config/general.h

@@ -175,7 +175,23 @@
 #define TERMINAL_SUPPORT         1              // Enable terminal commands (0.97Kb)
 #endif
 
-#define TERMINAL_BUFFER_SIZE     128            // Max size for commands commands
+#ifndef TERMINAL_SHARED_BUFFER_SIZE
+#define TERMINAL_SHARED_BUFFER_SIZE     128     // Maximum size for command line, shared by the WebUI, Telnet and Serial
+#endif
+
+#ifndef TERMINAL_MQTT_SUPPORT
+#define TERMINAL_MQTT_SUPPORT       0       // MQTT Terminal support built in
+                                            // Depends on MQTT_SUPPORT and TERMINAL_SUPPORT commands being available
+#endif
+
+#ifndef TERMINAL_WEB_API_SUPPORT
+#define TERMINAL_WEB_API_SUPPORT    0       // Web server API Terminal support built in
+                                            // Depends on WEB_SUPPORT and TERMINAL_SUPPORT commands being available
+#endif
+
+#ifndef TERMINAL_WEB_API_PATH
+#define TERMINAL_WEB_API_PATH       "/api/cmd"
+#endif
 
 //------------------------------------------------------------------------------
 // SYSTEM CHECK
@@ -768,7 +784,6 @@
 #define API_REAL_TIME_VALUES        0           // Show filtered/median values by default (0 => median, 1 => real time)
 #endif
 
-
 // -----------------------------------------------------------------------------
 // MDNS / LLMNR / NETBIOS / SSDP
 // -----------------------------------------------------------------------------
@@ -1163,6 +1178,7 @@
 #define MQTT_TOPIC_OTA              "ota"
 #define MQTT_TOPIC_TELNET_REVERSE   "telnet_reverse"
 #define MQTT_TOPIC_CURTAIN          "curtain"
+#define MQTT_TOPIC_CMD              "cmd"
 
 // Light module
 #define MQTT_TOPIC_CHANNEL          "channel"

code/espurna/crash.cpp

@@ -142,7 +142,7 @@ void crashDump() {
 void crashSetup() {
 
     #if TERMINAL_SUPPORT
-        terminalRegisterCommand(F("CRASH"), [](Embedis* e) {
+        terminalRegisterCommand(F("CRASH"), [](const terminal::CommandContext&) {
             crashDump();
             crashClear();
             terminalOK();

code/espurna/debug.cpp

@@ -63,6 +63,11 @@ void _debugSend(const char * format, va_list args) {
 
 }
 
+void debugSendRaw(const char* line, bool timestamp) {
+    if (!_debug_enabled) return;
+    _debugSendInternal(line, timestamp);
+}
+
 void debugSend(const char* format, ...) {
 
     if (!_debug_enabled) return;
@@ -263,7 +268,7 @@ void debugSetup() {
 
     #if DEBUG_LOG_BUFFER_SUPPORT
 
-        terminalRegisterCommand(F("DEBUG.BUFFER"), [](Embedis* e) {
+        terminalRegisterCommand(F("DEBUG.BUFFER"), [](const terminal::CommandContext&) {
             _debug_log_buffer_enabled = false;
             if (!_debug_log_buffer.size()) {
                 DEBUG_MSG_P(PSTR("[DEBUG] Buffer is empty\n"));

code/espurna/debug.h

@@ -32,8 +32,10 @@ void debugConfigure();
 void debugConfigureBoot();
 void debugSetup();
 
+void debugSendRaw(const char* line, bool timestamp = false);
+
 void debugSend(const char* format, ...);
-void debugSend_P(PGM_P format, ...); // PGM_P is `const char*`
+void debugSend_P(const char* format, ...);
 
 #if DEBUG_SUPPORT
     #define DEBUG_MSG(...) debugSend(__VA_ARGS__)

code/espurna/homeassistant.cpp

@@ -502,7 +502,7 @@ void _haWebSocketOnAction(uint32_t client_id, const char * action, JsonObject& d
 
 void _haInitCommands() {
 
-    terminalRegisterCommand(F("HA.CONFIG"), [](Embedis* e) {
+    terminalRegisterCommand(F("HA.CONFIG"), [](const terminal::CommandContext&) {
         for (unsigned char idx=0; idx<relayCount(); ++idx) {
             DynamicJsonBuffer jsonBuffer(1024);
             JsonObject& root = jsonBuffer.createObject();
@@ -521,7 +521,7 @@ void _haInitCommands() {
         terminalOK();
     });
 
-    terminalRegisterCommand(F("HA.SEND"), [](Embedis* e) {
+    terminalRegisterCommand(F("HA.SEND"), [](const terminal::CommandContext&) {
         setSetting("haEnabled", "1");
         _haConfigure();
         #if WEB_SUPPORT
@@ -530,7 +530,7 @@ void _haInitCommands() {
         terminalOK();
     });
 
-    terminalRegisterCommand(F("HA.CLEAR"), [](Embedis* e) {
+    terminalRegisterCommand(F("HA.CLEAR"), [](const terminal::CommandContext&) {
         setSetting("haEnabled", "0");
         _haConfigure();
         #if WEB_SUPPORT

code/espurna/i2c.cpp

@@ -363,12 +363,12 @@ void i2cScan() {
 
 void _i2cInitCommands() {
 
-    terminalRegisterCommand(F("I2C.SCAN"), [](Embedis* e) {
+    terminalRegisterCommand(F("I2C.SCAN"), [](const terminal::CommandContext&) {
         i2cScan();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("I2C.CLEAR"), [](Embedis* e) {
+    terminalRegisterCommand(F("I2C.CLEAR"), [](const terminal::CommandContext&) {
         i2cClearBus();
         terminalOK();
     });

code/espurna/influxdb.cpp

@@ -268,17 +268,13 @@ void idbSetup() {
     espurnaRegisterLoop(_idbFlush);
 
     #if TERMINAL_SUPPORT
-        terminalRegisterCommand(F("IDB.SEND"), [](Embedis* e) {
-            if (e->argc != 4) {
+        terminalRegisterCommand(F("IDB.SEND"), [](const terminal::CommandContext& ctx) {
+            if (ctx.argc != 4) {
                 terminalError(F("idb.send <topic> <id> <value>"));
                 return;
             }
 
-            const String topic = e->argv[1];
-            const auto id = atoi(e->argv[2]);
-            const String value = e->argv[3];
-
-            idbSend(topic.c_str(), id, value.c_str());
+            idbSend(ctx.argv[1].c_str(), ctx.argv[2].toInt(), ctx.argv[3].c_str());
         });
     #endif
 

code/espurna/libs/EmbedisWrap.h

@@ -1,26 +0,0 @@
-// -----------------------------------------------------------------------------
-// Wrap class around Embedis (settings & terminal)
-// -----------------------------------------------------------------------------
-
-#pragma once
-
-#include <Embedis.h>
-
-class EmbedisWrap : public Embedis {
-
-    public:
-
-        EmbedisWrap(Stream& stream, size_t buflen = 128, size_t argvlen = 8) :
-            Embedis(stream, buflen, argvlen)
-        {}
-
-        unsigned char getCommandCount() {
-            return commands.size();
-        }
-
-        String getCommandName(unsigned int i) {
-            if (i < commands.size()) return commands[i].name;
-            return String();
-        }
-
-};

code/espurna/libs/PrintString.h

@@ -0,0 +1,59 @@
+/*
+
+Arduino Print buffer. Size is fixed, unlike StreamString.
+
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#pragma once
+
+#include <Arduino.h>
+#include <Print.h>
+
+#include <core_version.h>
+
+
+struct PrintString final : public Print, public String {
+
+    PrintString(size_t reserved) :
+        _reserved(reserved)
+    {
+        reserve(reserved);
+    }
+
+    size_t write(const uint8_t* data, size_t size) override {
+        if (!size || !data) return 0;
+
+        // we *will* receive C-strings as input
+        size_t want = length() + size;
+        if (data[size - 1] == '\0') {
+            size -= 1;
+            want -= 1;
+        }
+
+        if (want > _reserved) return 0;
+
+// XXX: 2.3.0 uses str... methods that expect '0' at the end of the 'data'
+//      see WString{.cpp,.h} for the implementation
+#if defined(ARDUINO_ESP8266_RELEASE_2_3_0)
+        std::copy(data, data + size, buffer + len);
+        len = want;
+        buffer[len] = '\0';
+#else
+        concat(reinterpret_cast<const char*>(data), size);
+#endif
+
+        return size;
+    }
+
+    size_t write(uint8_t ch) override {
+        if (length() + 1 > _reserved) return 0;
+        return concat(static_cast<char>(ch));
+    }
+
+    private:
+
+    const size_t _reserved;
+
+};

code/espurna/libs/StreamAdapter.h

@@ -0,0 +1,68 @@
+/*
+
+Arduino Stream from a generic generic byte range
+Implementation of the Print is taken by reference and will be proxied
+
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#include <Arduino.h>
+#include <core_version.h>
+
+#include <memory>
+
+#pragma once
+
+template <typename T>
+struct StreamAdapter final : public Stream {
+    StreamAdapter(Print& writer, T&& begin, T&& end) :
+        _writer(writer),
+        _current(std::forward<T>(begin)),
+        _begin(std::forward<T>(begin)),
+        _end(std::forward<T>(end))
+    {}
+
+    int available() override {
+        return (_end - _current);
+    }
+
+    int peek() override {
+        if (available() && (_end != (1 + _current))) {
+            return *(1 + _current);
+        }
+        return -1;
+    }
+
+    int read() override {
+        if (_end != _current) {
+            return *(_current++);
+        }
+        return -1;
+    }
+
+    void flush() override {
+// 2.3.0  - Stream::flush()
+// latest - Print::flush()
+#if not defined(ARDUINO_ESP8266_RELEASE_2_3_0)
+        _writer.flush();
+#endif
+    }
+
+    size_t write(const uint8_t* buffer, size_t size) override {
+        return _writer.write(buffer, size);
+    }
+
+    size_t write(uint8_t ch) override {
+        return _writer.write(ch);
+    }
+
+    private:
+
+    Print& _writer;
+
+    T _current;
+    T const _begin;
+    T const _end;
+};
+

code/espurna/libs/StreamInjector.h

@@ -1,105 +0,0 @@
-/*
-
-StreamInjector
-
-Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-*/
-
-#pragma once
-
-#include <Stream.h>
-
-class StreamInjector : public Stream {
-
-    public:
-
-        typedef std::function<void(uint8_t ch)> writeCallback;
-
-        StreamInjector(size_t buflen = 128) : _buffer_size(buflen) {
-            _buffer = new char[buflen];
-        }
-
-        ~StreamInjector() {
-            delete[] _buffer;
-        }
-
-        // ---------------------------------------------------------------------
-
-        virtual uint8_t inject(char ch) {
-            _buffer[_buffer_write] = ch;
-            _buffer_write = (_buffer_write + 1) % _buffer_size;
-            return 1;
-        }
-
-        virtual uint8_t inject(char *data, size_t len) {
-            for (uint8_t i=0; i<len; i++) {
-                inject(data[i]);
-            }
-            return len;
-        }
-
-        virtual void callback(writeCallback c) {
-            _callback = c;
-        }
-
-        // ---------------------------------------------------------------------
-
-        virtual size_t write(uint8_t ch) {
-            if (_callback) _callback(ch);
-            return 1;
-        }
-
-        virtual int read() {
-            int ch = -1;
-            if (_buffer_read != _buffer_write) {
-                ch = _buffer[_buffer_read];
-                _buffer_read = (_buffer_read + 1) % _buffer_size;
-            }
-            return ch;
-        }
-
-        virtual int available() {
-            unsigned int bytes = 0;
-            if (_buffer_read > _buffer_write) {
-                bytes += (_buffer_write - _buffer_read + _buffer_size);
-            } else if (_buffer_read < _buffer_write) {
-                bytes += (_buffer_write - _buffer_read);
-            }
-            return bytes;
-        }
-
-        virtual int peek() {
-            int ch = -1;
-            if (_buffer_read != _buffer_write) {
-                ch = _buffer[_buffer_read];
-            }
-            return ch;
-        }
-
-        virtual void flush() {
-            _buffer_read = _buffer_write;
-        }
-
-    private:
-
-        char * _buffer;
-        unsigned char _buffer_size;
-        unsigned char _buffer_write = 0;
-        unsigned char _buffer_read = 0;
-        writeCallback _callback = NULL;
-
-};

code/espurna/light.cpp

@@ -1035,21 +1035,21 @@ void _lightChannelDebug(unsigned char id) {
 
 void _lightInitCommands() {
 
-    terminalRegisterCommand(F("BRIGHTNESS"), [](Embedis* e) {
-        if (e->argc > 1) {
-            _lightAdjustBrightness(e->argv[1]);
+    terminalRegisterCommand(F("BRIGHTNESS"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc > 1) {
+            _lightAdjustBrightness(ctx.argv[1].c_str());
             lightUpdate(true, true);
         }
         DEBUG_MSG_P(PSTR("Brightness: %u\n"), lightBrightness());
         terminalOK();
     });
 
-    terminalRegisterCommand(F("CHANNEL"), [](Embedis* e) {
+    terminalRegisterCommand(F("CHANNEL"), [](const terminal::CommandContext& ctx) {
         if (!lightChannels()) return;
 
         auto id = -1;
-        if (e->argc > 1) {
-            id = String(e->argv[1]).toInt();
+        if (ctx.argc > 1) {
+            id = ctx.argv[1].toInt();
         }
 
         if (id < 0 || id >= static_cast<decltype(id)>(lightChannels())) {
@@ -1059,8 +1059,8 @@ void _lightInitCommands() {
             return;
         }
 
-        if (e->argc > 2) {
-            _lightAdjustChannel(id, e->argv[2]);
+        if (ctx.argc > 2) {
+            _lightAdjustChannel(id, ctx.argv[2].c_str());
             lightUpdate(true, true);
         }
 
@@ -1069,27 +1069,27 @@ void _lightInitCommands() {
         terminalOK();
     });
 
-    terminalRegisterCommand(F("COLOR"), [](Embedis* e) {
-        if (e->argc > 1) {
-            lightColor(e->argv[1]);
+    terminalRegisterCommand(F("COLOR"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc > 1) {
+            lightColor(ctx.argv[1].c_str());
             lightUpdate(true, true);
         }
         DEBUG_MSG_P(PSTR("Color: %s\n"), lightColor().c_str());
         terminalOK();
     });
 
-    terminalRegisterCommand(F("KELVIN"), [](Embedis* e) {
-        if (e->argc > 1) {
-            _lightAdjustKelvin(e->argv[1]);
+    terminalRegisterCommand(F("KELVIN"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc > 1) {
+            _lightAdjustKelvin(ctx.argv[1].c_str());
             lightUpdate(true, true);
         }
         DEBUG_MSG_P(PSTR("Color: %s\n"), lightColor().c_str());
         terminalOK();
     });
 
-    terminalRegisterCommand(F("MIRED"), [](Embedis* e) {
-        if (e->argc > 1) {
-            _lightAdjustMireds(e->argv[1]);
+    terminalRegisterCommand(F("MIRED"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc > 1) {
+            _lightAdjustMireds(ctx.argv[1].c_str());
             lightUpdate(true, true);
         }
         DEBUG_MSG_P(PSTR("Color: %s\n"), lightColor().c_str());

code/espurna/lightfox.cpp

@@ -76,12 +76,12 @@ void _lightfoxWebSocketOnAction(uint32_t client_id, const char * action, JsonObj
 
 void _lightfoxInitCommands() {
 
-    terminalRegisterCommand(F("LIGHTFOX.LEARN"), [](Embedis* e) {
+    terminalRegisterCommand(F("LIGHTFOX.LEARN"), [](const terminal::CommandContext&) {
         lightfoxLearn();
         DEBUG_MSG_P(PSTR("+OK\n"));
     });
 
-    terminalRegisterCommand(F("LIGHTFOX.CLEAR"), [](Embedis* e) {
+    terminalRegisterCommand(F("LIGHTFOX.CLEAR"), [](const terminal::CommandContext&) {
         lightfoxClear();
         DEBUG_MSG_P(PSTR("+OK\n"));
     });

code/espurna/main.cpp

@@ -184,8 +184,10 @@ void setup() {
             otaWebSetup();
         #endif
     #endif
-    #if API_SUPPORT
-        apiSetup();
+
+    // Multiple modules depend on the generic 'API' services
+    #if API_SUPPORT || TERMINAL_WEB_API_SUPPORT
+        apiCommonSetup();
     #endif
 
     // lightSetup must be called before relaySetup

code/espurna/mqtt.cpp

@@ -473,13 +473,13 @@ void _mqttWebSocketOnConnected(JsonObject& root) {
 
 void _mqttInitCommands() {
 
-    terminalRegisterCommand(F("MQTT.RESET"), [](Embedis* e) {
+    terminalRegisterCommand(F("MQTT.RESET"), [](const terminal::CommandContext&) {
         _mqttConfigure();
         mqttDisconnect();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("MQTT.INFO"), [](Embedis* e) {
+    terminalRegisterCommand(F("MQTT.INFO"), [](const terminal::CommandContext&) {
         _mqttInfo();
         terminalOK();
     });

code/espurna/nofuss.cpp

@@ -98,7 +98,7 @@ void _nofussLoop() {
 
 void _nofussInitCommands() {
 
-    terminalRegisterCommand(F("NOFUSS"), [](Embedis* e) {
+    terminalRegisterCommand(F("NOFUSS"), [](const terminal::CommandContext&) {
         terminalOK();
         nofussRun();
     });

code/espurna/ntp.cpp

@@ -381,20 +381,20 @@ void ntpSetup() {
     #endif
 
     #if TERMINAL_SUPPORT
-        terminalRegisterCommand(F("NTP"), [](Embedis* e) {
+        terminalRegisterCommand(F("NTP"), [](const terminal::CommandContext&) {
             _ntpReport();
             terminalOK();
         });
 
-        terminalRegisterCommand(F("NTP.SETTIME"), [](Embedis* e) {
-            if (e->argc != 2) return;
+        terminalRegisterCommand(F("NTP.SETTIME"), [](const terminal::CommandContext& ctx) {
+            if (ctx.argc != 2) return;
             _ntp_synced = true;
-            _ntpSetTimestamp(String(e->argv[1]).toInt());
+            _ntpSetTimestamp(ctx.argv[1].toInt());
             terminalOK();
         });
 
         // TODO:
-        // terminalRegisterCommand(F("NTP.SYNC"), [](Embedis* e) { ... }
+        // terminalRegisterCommand(F("NTP.SYNC"), [](const terminal::CommandContext&) { ... }
         //
     #endif
 

code/espurna/ntp_legacy.cpp

@@ -250,7 +250,7 @@ void ntpSetup() {
     _ntpBackwards();
 
     #if TERMINAL_SUPPORT
-        terminalRegisterCommand(F("NTP"), [](Embedis* e) {
+        terminalRegisterCommand(F("NTP"), [](const terminal::CommandContext&) {
             if (ntpSynced()) {
                 _ntpReport();
                 terminalOK();
@@ -259,7 +259,7 @@ void ntpSetup() {
             }
         });
 
-        terminalRegisterCommand(F("NTP.SYNC"), [](Embedis* e) {
+        terminalRegisterCommand(F("NTP.SYNC"), [](const terminal::CommandContext&) {
             _ntpWantSync();
             terminalOK();
         });

code/espurna/ota.cpp

@@ -12,7 +12,7 @@ OTA MODULE COMMON FUNCTIONS
 void otaPrintError() {
     if (Update.hasError()) {
         #if TERMINAL_SUPPORT
-            Update.printError(terminalSerial());
+            Update.printError(terminalDefaultStream());
         #elif DEBUG_SERIAL_SUPPORT && defined(DEBUG_PORT)
             Update.printError(DEBUG_PORT);
         #endif

code/espurna/ota_asynctcp.cpp

@@ -219,11 +219,11 @@ void _otaClientFrom(const String& url) {
 
 void _otaClientInitCommands() {
 
-    terminalRegisterCommand(F("OTA"), [](Embedis* e) {
-        if (e->argc < 2) {
+    terminalRegisterCommand(F("OTA"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc < 2) {
             terminalError(F("OTA <url>"));
         } else {
-            _otaClientFrom(String(e->argv[1]));
+            _otaClientFrom(ctx.argv[1]);
             terminalOK();
         }
     });

code/espurna/ota_httpupdate.cpp

@@ -222,11 +222,11 @@ void _otaClientFrom(const String& url) {
 
 void _otaClientInitCommands() {
 
-    terminalRegisterCommand(F("OTA"), [](Embedis* e) {
-        if (e->argc < 2) {
+    terminalRegisterCommand(F("OTA"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc < 2) {
             terminalError(F("OTA <url>"));
         } else {
-            _otaClientFrom(String(e->argv[1]));
+            _otaClientFrom(ctx.argv[1]);
             terminalOK();
         }
     });

code/espurna/relay.cpp

@@ -1334,19 +1334,19 @@ void _relaySetupProvider() {
 
 void _relayInitCommands() {
 
-    terminalRegisterCommand(F("RELAY"), [](Embedis* e) {
-        if (e->argc < 2) {
+    terminalRegisterCommand(F("RELAY"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc < 2) {
             terminalError(F("Wrong arguments"));
             return;
         }
-        int id = String(e->argv[1]).toInt();
+        int id = ctx.argv[1].toInt();
         if (id >= relayCount()) {
             DEBUG_MSG_P(PSTR("-ERROR: Wrong relayID (%d)\n"), id);
             return;
         }
 
-        if (e->argc > 2) {
-            int value = String(e->argv[2]).toInt();
+        if (ctx.argc > 2) {
+            int value = ctx.argv[2].toInt();
             if (value == 2) {
                 relayToggle(id);
             } else {
@@ -1363,7 +1363,7 @@ void _relayInitCommands() {
     });
 
     #if 0
-    terminalRegisterCommand(F("RELAY.INFO"), [](Embedis* e) {
+    terminalRegisterCommand(F("RELAY.INFO"), [](const terminal::CommandContext&) {
         DEBUG_MSG_P(PSTR("    cur tgt pin type reset lock  delay_on   delay_off  pulse  pulse_ms\n"));
         DEBUG_MSG_P(PSTR("    --- --- --- ---- ----- ---- ---------- ----------- ----- ----------\n"));
         for (unsigned char index = 0; index < _relays.size(); ++index) {

code/espurna/rfbridge.cpp

@@ -638,54 +638,53 @@ void _rfbAPISetup() {
 
 void _rfbInitCommands() {
 
-    terminalRegisterCommand(F("LEARN"), [](Embedis* e) {
+    terminalRegisterCommand(F("LEARN"), [](const terminal::CommandContext& ctx) {
 
-        if (e->argc < 3) {
+        if (ctx.argc != 3) {
             terminalError(F("Wrong arguments"));
             return;
         }
         
-        int id = String(e->argv[1]).toInt();
+        // 1st argument is relayID
+        int id = ctx.argv[1].toInt();
         if (id >= relayCount()) {
             DEBUG_MSG_P(PSTR("-ERROR: Wrong relayID (%d)\n"), id);
             return;
         }
 
-        int status = String(e->argv[2]).toInt();
-
-        rfbLearn(id, status == 1);
+        // 2nd argument is status
+        rfbLearn(id, (ctx.argv[2].toInt()) == 1);
 
         terminalOK();
 
     });
 
-    terminalRegisterCommand(F("FORGET"), [](Embedis* e) {
+    terminalRegisterCommand(F("FORGET"), [](const terminal::CommandContext& ctx) {
 
-        if (e->argc < 3) {
+        if (ctx.argc != 3) {
             terminalError(F("Wrong arguments"));
             return;
         }
         
-        int id = String(e->argv[1]).toInt();
+        // 1st argument is relayID
+        int id = ctx.argv[1].toInt();
         if (id >= relayCount()) {
             DEBUG_MSG_P(PSTR("-ERROR: Wrong relayID (%d)\n"), id);
             return;
         }
 
-        int status = String(e->argv[2]).toInt();
-
-        rfbForget(id, status == 1);
+        // 2nd argument is status
+        rfbForget(id, (ctx.argv[2].toInt()) == 1);
 
         terminalOK();
 
     });
 
     #if !RFB_DIRECT
-        terminalRegisterCommand(F("RFB.WRITE"), [](Embedis* e) {
-            if (e->argc != 2) return;
-            String arg(e->argv[1]);
+        terminalRegisterCommand(F("RFB.WRITE"), [](const terminal::CommandContext& ctx) {
+            if (ctx.argc != 2) return;
             uint8_t data[RF_MAX_MESSAGE_SIZE];
-            size_t bytes = _rfbBytearrayFromHex(arg.c_str(), arg.length(), data, sizeof(data));
+            size_t bytes = _rfbBytearrayFromHex(ctx.argv[1].c_str(), ctx.argv[1].length(), data, sizeof(data));
             if (bytes) {
                 _rfbSendRaw(data, bytes);
             }

code/espurna/rpnrules.cpp

@@ -282,7 +282,7 @@ void _rpnInit() {
 
 void _rpnInitCommands() {
 
-    terminalRegisterCommand(F("RPN.VARS"), [](Embedis* e) {
+    terminalRegisterCommand(F("RPN.VARS"), [](const terminal::CommandContext&) {
         unsigned char num = rpn_variables_size(_rpn_ctxt);
         if (0 == num) {
             DEBUG_MSG_P(PSTR("[RPN] No variables\n"));
@@ -298,7 +298,7 @@ void _rpnInitCommands() {
         terminalOK();
     });
 
-    terminalRegisterCommand(F("RPN.OPS"), [](Embedis* e) {
+    terminalRegisterCommand(F("RPN.OPS"), [](const terminal::CommandContext&) {
         unsigned char num = _rpn_ctxt.operators.size();
         DEBUG_MSG_P(PSTR("[RPN] Operators:\n"));
         for (unsigned char i=0; i<num; i++) {
@@ -307,10 +307,10 @@ void _rpnInitCommands() {
         terminalOK();
     });
 
-    terminalRegisterCommand(F("RPN.TEST"), [](Embedis* e) {
-        if (e->argc == 2) {
-            DEBUG_MSG_P(PSTR("[RPN] Running \"%s\"\n"), e->argv[1]);
-            rpn_process(_rpn_ctxt, e->argv[1], true);
+    terminalRegisterCommand(F("RPN.TEST"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc == 2) {
+            DEBUG_MSG_P(PSTR("[RPN] Running \"%s\"\n"), ctx.argv[1].c_str());
+            rpn_process(_rpn_ctxt, ctx.argv[1].c_str(), true);
             _rpnDump();
             rpn_stack_clear(_rpn_ctxt);
             terminalOK();

code/espurna/rtcmem.cpp

@@ -48,12 +48,12 @@ bool _rtcmemStatus() {
 #if TERMINAL_SUPPORT
 
 void _rtcmemInitCommands() {
-    terminalRegisterCommand(F("RTCMEM.REINIT"), [](Embedis* e) {
+    terminalRegisterCommand(F("RTCMEM.REINIT"), [](const terminal::CommandContext&) {
         _rtcmemInit();
     });
 
     #if DEBUG_SUPPORT
-        terminalRegisterCommand(F("RTCMEM.DUMP"), [](Embedis* e) {
+        terminalRegisterCommand(F("RTCMEM.DUMP"), [](const terminal::CommandContext&) {
 
             DEBUG_MSG_P(PSTR("[RTCMEM] boot_status=%u status=%u blocks_used=%u\n"),
                 _rtcmem_status, _rtcmemStatus(), RtcmemSize);

code/espurna/sensor.cpp

@@ -1164,7 +1164,7 @@ void _sensorMqttCallback(unsigned int type, const char* topic, char* payload) {
 #if TERMINAL_SUPPORT
 
 void _sensorInitCommands() {
-    terminalRegisterCommand(F("MAGNITUDES"), [](Embedis* e) {
+    terminalRegisterCommand(F("MAGNITUDES"), [](const terminal::CommandContext&) {
         char last[64];
         char reported[64];
         for (size_t index = 0; index < _magnitudes.size(); ++index) {

code/espurna/sensors/PZEM004TSensor.h

@@ -368,19 +368,19 @@ PZEM004TSensor* PZEM004TSensor::instance = nullptr;
 
 void pzem004tInitCommands() {
 
-    terminalRegisterCommand(F("PZ.ADDRESS"), [](Embedis* e) {
+    terminalRegisterCommand(F("PZ.ADDRESS"), [](const terminal::CommandContext& ctx) {
         if (!PZEM004TSensor::instance) return;
 
-        if (e->argc == 1) {
+        if (ctx.argc == 1) {
             DEBUG_MSG_P(PSTR("[SENSOR] PZEM004T\n"));
             unsigned char dev_count = PZEM004TSensor::instance->countDevices();
             for(unsigned char dev = 0; dev < dev_count; dev++) {
                 DEBUG_MSG_P(PSTR("Device %d/%s\n"), dev, PZEM004TSensor::instance->getAddress(dev).c_str());
             }
             terminalOK();
-        } else if(e->argc == 2) {
+        } else if(ctx.argc == 2) {
             IPAddress addr;
-            if (addr.fromString(String(e->argv[1]))) {
+            if (addr.fromString(ctx.argv[1])) {
                 if(PZEM004TSensor::instance->setDeviceAddress(&addr)) {
                     terminalOK();
                 }
@@ -392,12 +392,12 @@ void pzem004tInitCommands() {
         }
     });
 
-    terminalRegisterCommand(F("PZ.RESET"), [](Embedis* e) {
-        if(e->argc > 2) {
+    terminalRegisterCommand(F("PZ.RESET"), [](const terminal::CommandContext& ctx) {
+        if(ctx.argc > 2) {
             terminalError(F("Wrong arguments"));
         } else {
-            unsigned char init = e->argc == 2 ? String(e->argv[1]).toInt() : 0;
-            unsigned char limit = e->argc == 2 ? init +1 : PZEM004TSensor::instance->countDevices();
+            unsigned char init = ctx.argc == 2 ? ctx.argv[1].toInt() : 0;
+            unsigned char limit = ctx.argc == 2 ? init +1 : PZEM004TSensor::instance->countDevices();
             DEBUG_MSG_P(PSTR("[SENSOR] PZEM004T\n"));
             for(unsigned char dev = init; dev < limit; dev++) {
                 PZEM004TSensor::instance->resetEnergy(dev);
@@ -406,12 +406,12 @@ void pzem004tInitCommands() {
         }
     });
 
-    terminalRegisterCommand(F("PZ.VALUE"), [](Embedis* e) {
-        if(e->argc > 2) {
+    terminalRegisterCommand(F("PZ.VALUE"), [](const terminal::CommandContext& ctx) {
+        if(ctx.argc > 2) {
             terminalError(F("Wrong arguments"));
         } else {
-            unsigned char init = e->argc == 2 ? String(e->argv[1]).toInt() : 0;
-            unsigned char limit = e->argc == 2 ? init +1 : PZEM004TSensor::instance->countDevices();
+            unsigned char init = ctx.argc == 2 ? ctx.argv[1].toInt() : 0;
+            unsigned char limit = ctx.argc == 2 ? init +1 : PZEM004TSensor::instance->countDevices();
             DEBUG_MSG_P(PSTR("[SENSOR] PZEM004T\n"));
             for(unsigned char dev = init; dev < limit; dev++) {
                 DEBUG_MSG_P(PSTR("Device %d/%s - Current: %s Voltage: %s Power: %s Energy: %s\n"), //

code/espurna/settings.cpp

@@ -8,6 +8,8 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 
 #include "settings.h"
 
+#include "terminal.h"
+
 #include <vector>
 #include <cstdlib>
 
@@ -492,4 +494,107 @@ void settingsSetup() {
         #endif
     );
 
+    terminalRegisterCommand(F("CONFIG"), [](const terminal::CommandContext& ctx) {
+        // TODO: enough of a buffer?
+        DynamicJsonBuffer jsonBuffer(1024);
+        JsonObject& root = jsonBuffer.createObject();
+        settingsGetJson(root);
+        root.prettyPrintTo(ctx.output);
+        terminalOK(ctx);
+    });
+
+    terminalRegisterCommand(F("KEYS"), [](const terminal::CommandContext& ctx) {
+        // Get sorted list of keys
+        auto keys = settingsKeys();
+
+        // Write key-values
+        ctx.output.println(F("Current settings:"));
+        for (unsigned int i=0; i<keys.size(); i++) {
+            const auto value = getSetting(keys[i]);
+            ctx.output.printf("> %s => \"%s\"\n", (keys[i]).c_str(), value.c_str());
+        }
+
+        unsigned long freeEEPROM [[gnu::unused]] = SPI_FLASH_SEC_SIZE - settingsSize();
+        ctx.output.printf("Number of keys: %u\n", keys.size());
+        ctx.output.printf("Current EEPROM sector: %u\n", EEPROMr.current());
+        ctx.output.printf("Free EEPROM: %lu bytes (%lu%%)\n", freeEEPROM, 100 * freeEEPROM / SPI_FLASH_SEC_SIZE);
+
+        terminalOK(ctx);
+    });
+
+    terminalRegisterCommand(F("DEL"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc != 2) {
+            terminalError(ctx, F("del <key> [<key>...]"));
+            return;
+        }
+
+        int result = 0;
+        for (auto it = (ctx.argv.begin() + 1); it != ctx.argv.end(); ++it) {
+            result += Embedis::del(*it);
+        }
+
+        if (result) {
+            terminalOK(ctx);
+        } else {
+            terminalError(ctx, F("no keys were removed"));
+        }
+    });
+
+    terminalRegisterCommand(F("SET"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc != 3) {
+            terminalError(ctx, F("set <key> <value>"));
+            return;
+        }
+
+        if (Embedis::set(ctx.argv[1], ctx.argv[2])) {
+            terminalOK(ctx);
+            return;
+        }
+
+        terminalError(ctx, F("could not set the key"));
+    });
+
+    terminalRegisterCommand(F("GET"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc < 2) {
+            terminalError(ctx, F("Wrong arguments"));
+            return;
+        }
+
+        for (auto it = (ctx.argv.begin() + 1); it != ctx.argv.end(); ++it) {
+            const String& key = *it;
+            String value;
+            if (!Embedis::get(key, value)) {
+                const auto maybeDefault = settingsQueryDefaults(key);
+                if (maybeDefault.length()) {
+                    ctx.output.printf("> %s => %s (default)\n", key.c_str(), maybeDefault.c_str());
+                } else {
+                    ctx.output.printf("> %s =>\n", key.c_str());
+                }
+                continue;
+            }
+
+            ctx.output.printf("> %s => \"%s\"\n", key.c_str(), value.c_str());
+        }
+
+        terminalOK(ctx);
+    });
+
+    terminalRegisterCommand(F("RELOAD"), [](const terminal::CommandContext&) {
+        espurnaReload();
+        terminalOK();
+    });
+
+    terminalRegisterCommand(F("FACTORY.RESET"), [](const terminal::CommandContext&) {
+        resetSettings();
+        terminalOK();
+    });
+
+    #if not SETTINGS_AUTOSAVE
+        terminalRegisterCommand(F("SAVE"), [](const terminal::CommandContext&) {
+            eepromCommit();
+            terminalOK();
+        });
+    #endif
+
+
 }

code/espurna/settings.h

@@ -8,16 +8,16 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 
 #pragma once
 
-#include <Arduino.h>
+#include "espurna.h"
 
 #include <functional>
 #include <utility>
 #include <vector>
+
 #include <ArduinoJson.h>
+#include <Embedis.h>
 
-#include "espurna.h"
 #include "broker.h"
-#include "libs/EmbedisWrap.h"
 
 BrokerDeclare(ConfigBroker, void(const String& key, const String& value));
 

code/espurna/storage_eeprom.cpp

@@ -65,7 +65,7 @@ void eepromBackup(uint32_t index){
 
 void _eepromInitCommands() {
 
-    terminalRegisterCommand(F("EEPROM"), [](Embedis* e) {
+    terminalRegisterCommand(F("EEPROM"), [](const terminal::CommandContext&) {
         infoMemory("EEPROM", SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE - settingsSize());
         eepromSectorsDebug();
         if (_eeprom_commit_count > 0) {
@@ -75,7 +75,7 @@ void _eepromInitCommands() {
         terminalOK();
     });
 
-    terminalRegisterCommand(F("EEPROM.COMMIT"), [](Embedis* e) {
+    terminalRegisterCommand(F("EEPROM.COMMIT"), [](const terminal::CommandContext&) {
         const bool res = _eepromCommit();
         if (res) {
             terminalOK();
@@ -84,24 +84,26 @@ void _eepromInitCommands() {
         }
     });
 
-    terminalRegisterCommand(F("EEPROM.DUMP"), [](Embedis* e) {
-        EEPROMr.dump(terminalSerial());
-        terminalOK();
+    terminalRegisterCommand(F("EEPROM.DUMP"), [](const terminal::CommandContext& ctx) {
+        // XXX: like Update::printError, dump only accepts Stream
+        //      this should be safe, since we expect read-only stream
+        EEPROMr.dump(reinterpret_cast<Stream&>(ctx.output));
+        terminalOK(ctx.output);
     });
 
-    terminalRegisterCommand(F("FLASH.DUMP"), [](Embedis* e) {
-        if (e->argc < 2) {
+    terminalRegisterCommand(F("FLASH.DUMP"), [](const terminal::CommandContext& ctx) {
+        if (ctx.argc < 2) {
             terminalError(F("Wrong arguments"));
             return;
         }
-        uint32_t sector = String(e->argv[1]).toInt();
+        uint32_t sector = ctx.argv[1].toInt();
         uint32_t max = ESP.getFlashChipSize() / SPI_FLASH_SEC_SIZE;
         if (sector >= max) {
             terminalError(F("Sector out of range"));
             return;
         }
-        EEPROMr.dump(terminalSerial(), sector);
-        terminalOK();
+        EEPROMr.dump(reinterpret_cast<Stream&>(ctx.output), sector);
+        terminalOK(ctx.output);
     });
 
 }

code/espurna/telnet.cpp

@@ -20,6 +20,7 @@ Updated to use WiFiServer and support reverse connections by Niek van der Maas <
 #if TELNET_SUPPORT
 
 #include <memory>
+#include <vector>
 
 #include "board.h"
 #include "ws.h"
@@ -107,6 +108,8 @@ void _telnetReverseMQTTCallback(unsigned int type, const char * topic, const cha
 
 #if TELNET_SERVER == TELNET_SERVER_WIFISERVER
 
+static std::vector<char> _telnet_data_buffer;
+
 void _telnetDisconnect(unsigned char clientId) {
     _telnetClients[clientId]->stop();
     _telnetClients[clientId] = nullptr;
@@ -370,11 +373,10 @@ void _telnetLoop() {
             } else {
                 // Read data from clients
                 while (_telnetClients[i] && _telnetClients[i]->available()) {
-                    char data[TERMINAL_BUFFER_SIZE];
                     size_t len = _telnetClients[i]->available();
-                    unsigned int r = _telnetClients[i]->readBytes(data, min(sizeof(data), len));
+                    unsigned int r = _telnetClients[i]->readBytes(_telnet_data_buffer.data(), min(_telnet_data_buffer.capacity(), len));
 
-                    _telnetData(i, data, r);
+                    _telnetData(i, _telnet_data_buffer.data(), r);
                 }
             }
         }
@@ -474,9 +476,10 @@ void _telnetConfigure() {
 void telnetSetup() {
 
     #if TELNET_SERVER == TELNET_SERVER_WIFISERVER
-        espurnaRegisterLoop(_telnetLoop);
+        _telnet_data_buffer.reserve(terminalCapacity());
         _telnetServer.setNoDelay(true);
         _telnetServer.begin();
+        espurnaRegisterLoop(_telnetLoop);
     #else
         _telnetServer.onClient([](void *s, AsyncClient* c) {
             _telnetNewClient(c);
@@ -497,17 +500,14 @@ void telnetSetup() {
         #endif
 
         #if TERMINAL_SUPPORT
-            terminalRegisterCommand(F("TELNET.REVERSE"), [](Embedis* e) {
-                if (e->argc < 3) {
+            terminalRegisterCommand(F("TELNET.REVERSE"), [](const terminal::CommandContext& ctx) {
+                if (ctx.argc < 3) {
                     terminalError(F("Wrong arguments. Usage: TELNET.REVERSE <host> <port>"));
                     return;
                 }
 
-                String host = String(e->argv[1]);
-                uint16_t port = String(e->argv[2]).toInt();
-
                 terminalOK();
-                _telnetReverse(host.c_str(), port);
+                _telnetReverse(ctx.argv[1].c_str(), ctx.argv[2].toInt());
             });
         #endif
     #endif

code/espurna/terminal.cpp

@@ -3,101 +3,209 @@
 TERMINAL MODULE
 
 Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
 
 */
 
-// (HACK) allow us to use internal lwip struct.
-// esp8266 re-defines enum values from tcp header... include them first
 #include "terminal.h"
 
 #if TERMINAL_SUPPORT
 
+#include "api.h"
+#include "debug.h"
 #include "settings.h"
 #include "system.h"
 #include "telnet.h"
 #include "utils.h"
+#include "mqtt.h"
 #include "wifi.h"
 #include "ws.h"
+
 #include "libs/URL.h"
-#include "libs/StreamInjector.h"
+#include "libs/StreamAdapter.h"
+#include "libs/PrintString.h"
+
+#include "web_asyncwebprint_impl.h"
 
+#include <algorithm>
 #include <vector>
+#include <utility>
+
+#include <Schedule.h>
 #include <Stream.h>
 
-StreamInjector _serial = StreamInjector(TERMINAL_BUFFER_SIZE);
-EmbedisWrap embedis(_serial, TERMINAL_BUFFER_SIZE);
+#if LWIP_VERSION_MAJOR != 1
 
-#if SERIAL_RX_ENABLED
-    char _serial_rx_buffer[TERMINAL_BUFFER_SIZE];
-    static unsigned char _serial_rx_pointer = 0;
-#endif // SERIAL_RX_ENABLED
+// not yet CONNECTING or LISTENING
+extern struct tcp_pcb *tcp_bound_pcbs;
+// accepting or sending data
+extern struct tcp_pcb *tcp_active_pcbs;
+// // TIME-WAIT status
+extern struct tcp_pcb *tcp_tw_pcbs;
 
-// -----------------------------------------------------------------------------
-// Commands
-// -----------------------------------------------------------------------------
+#endif
 
-void _terminalHelpCommand() {
+namespace {
 
-    // Get sorted list of commands
-    std::vector<String> commands;
-    unsigned char size = embedis.getCommandCount();
-    for (unsigned int i=0; i<size; i++) {
-
-        String command = embedis.getCommandName(i);
-        bool inserted = false;
-        for (unsigned char j=0; j<commands.size(); j++) {
-
-            // Check if we have to insert it before the current element
-            if (commands[j].compareTo(command) > 0) {
-                commands.insert(commands.begin() + j, command);
-                inserted = true;
-                break;
-            }
+// Based on libs/StreamInjector.h by Xose Pérez <xose dot perez at gmail dot com> (see git-log for more info)
+// Instead of custom write(uint8_t) callback, we provide writer implementation in-place
+
+struct TerminalIO final : public Stream {
 
+    TerminalIO(size_t capacity = 128) :
+        _buffer(new char[capacity]),
+        _capacity(capacity),
+        _write(0),
+        _read(0)
+    {}
+
+    ~TerminalIO() {
+        delete[] _buffer;
+    }
+
+    // ---------------------------------------------------------------------
+    // Injects data into the internal buffer so we can read() it
+    // ---------------------------------------------------------------------
+
+    size_t capacity() {
+        return _capacity;
+    }
+
+    size_t inject(char ch) {
+        _buffer[_write] = ch;
+        _write = (_write + 1) % _capacity;
+        return 1;
+    }
+
+    size_t inject(char *data, size_t len) {
+        for (size_t index = 0; index < len; ++index) {
+            inject(data[index]);
+        }
+        return len;
+    }
+
+    // ---------------------------------------------------------------------
+    // XXX: We are only supporting part of the Print & Stream interfaces
+    //      But, we need to be have all pure virtual methods implemented
+    // ---------------------------------------------------------------------
+
+    // Return data from the internal buffer
+    int available() override {
+        unsigned int bytes = 0;
+        if (_read > _write) {
+            bytes += (_write - _read + _capacity);
+        } else if (_read < _write) {
+            bytes += (_write - _read);
+        }
+        return bytes;
+    }
+
+    int peek() override {
+        int ch = -1;
+        if (_read != _write) {
+            ch = _buffer[_read];
+        }
+        return ch;
+    }
+
+    int read() override {
+        int ch = -1;
+        if (_read != _write) {
+            ch = _buffer[_read];
+            _read = (_read + 1) % _capacity;
         }
+        return ch;
+    }
 
-        // If we could not insert it, just push it at the end
-        if (!inserted) commands.push_back(command);
+    // {Stream,Print}::flush(), see:
+    // - https://github.com/esp8266/Arduino/blob/master/cores/esp8266/Print.h
+    // - https://github.com/espressif/arduino-esp32/blob/master/cores/esp32/Print.h
+    // - https://github.com/arduino/ArduinoCore-API/issues/102
+    // Old 2.3.0 expects flush() on Stream, latest puts in in Print
+    // We may have to cheat the system and implement everything as Stream to have it available.
+    void flush() override {
+        // Here, reset reader position so that we return -1 until we have new data
+        // writer flushing is implemented below, we don't need it here atm
+        _read = _write;
+    }
 
+    size_t write(const uint8_t* buffer, size_t size) override {
+    // Buffer data until we encounter line break, then flush via Raw debug method
+    // (which is supposed to 1-to-1 copy the data, without adding the timestamp)
+#if DEBUG_SUPPORT
+        if (!size) return 0;
+        if (buffer[size-1] == '\0') return 0;
+        if (_output.capacity() < (size + 2)) {
+            _output.reserve(_output.size() + size + 2);
+        }
+        _output.insert(_output.end(),
+            reinterpret_cast<const char*>(buffer),
+            reinterpret_cast<const char*>(buffer) + size
+        );
+        if (_output.end() != std::find(_output.begin(), _output.end(), '\n')) {
+            _output.push_back('\0');
+            debugSendRaw(_output.data());
+            _output.clear();
+        }
+#endif
+        return size;
     }
 
-    // Output the list
-    DEBUG_MSG_P(PSTR("Available commands:\n"));
-    for (unsigned char i=0; i<commands.size(); i++) {
-        DEBUG_MSG_P(PSTR("> %s\n"), (commands[i]).c_str());
+    size_t write(uint8_t ch) override {
+        uint8_t buffer[1] {ch};
+        return write(buffer, 1);
     }
 
-}
+    private:
 
-void _terminalKeysCommand() {
+#if DEBUG_SUPPORT
+    std::vector<char> _output;
+#endif
 
-    // Get sorted list of keys
-    auto keys = settingsKeys();
+    char * _buffer;
+    unsigned char _capacity;
+    unsigned char _write;
+    unsigned char _read;
 
-    // Write key-values
-    DEBUG_MSG_P(PSTR("Current settings:\n"));
-    for (unsigned int i=0; i<keys.size(); i++) {
-        const auto value = getSetting(keys[i]);
-        DEBUG_MSG_P(PSTR("> %s => \"%s\"\n"), (keys[i]).c_str(), value.c_str());
+};
+
+auto _io = TerminalIO(TERMINAL_SHARED_BUFFER_SIZE);
+terminal::Terminal _terminal(_io, _io.capacity());
+
+// TODO: re-evaluate how and why this is used
+#if SERIAL_RX_ENABLED
+
+constexpr size_t SerialRxBufferSize { 128u };
+char _serial_rx_buffer[SerialRxBufferSize];
+static unsigned char _serial_rx_pointer = 0;
+
+#endif // SERIAL_RX_ENABLED
+
+// -----------------------------------------------------------------------------
+// Commands
+// -----------------------------------------------------------------------------
+
+void _terminalHelpCommand(const terminal::CommandContext& ctx) {
+
+    // Get sorted list of commands
+    auto commands = _terminal.commandNames();
+    std::sort(commands.begin(), commands.end(), [](const String& rhs, const String& lhs) -> bool {
+        return lhs.compareTo(rhs) > 0;
+    });
+
+    // Output the list asap
+    ctx.output.print(F("Available commands:\n"));
+    for (auto& command : commands) {
+        ctx.output.printf("> %s\n", command.c_str());
     }
 
-    unsigned long freeEEPROM [[gnu::unused]] = SPI_FLASH_SEC_SIZE - settingsSize();
-    DEBUG_MSG_P(PSTR("Number of keys: %d\n"), keys.size());
-    DEBUG_MSG_P(PSTR("Current EEPROM sector: %u\n"), EEPROMr.current());
-    DEBUG_MSG_P(PSTR("Free EEPROM: %d bytes (%d%%)\n"), freeEEPROM, 100 * freeEEPROM / SPI_FLASH_SEC_SIZE);
+    terminalOK(ctx.output);
 
 }
 
 #if LWIP_VERSION_MAJOR != 1
 
-// not yet CONNECTING or LISTENING
-extern struct tcp_pcb *tcp_bound_pcbs;
-// accepting or sending data
-extern struct tcp_pcb *tcp_active_pcbs;
-// // TIME-WAIT status
-extern struct tcp_pcb *tcp_tw_pcbs;
-
-String _terminalPcbStateToString(const unsigned char state) {
+String _terminalPcbStateToString(unsigned char state) {
     switch (state) {
         case 0: return F("CLOSED");
         case 1: return F("LISTEN");
@@ -173,39 +281,32 @@ void _terminalDnsFound(const char* name, const ip_addr_t* result, void*) {
 
 #endif // LWIP_VERSION_MAJOR != 1
 
-void _terminalInitCommand() {
+void _terminalInitCommands() {
 
-    terminalRegisterCommand(F("COMMANDS"), [](Embedis* e) {
-        _terminalHelpCommand();
-        terminalOK();
-    });
+    terminalRegisterCommand(F("COMMANDS"), _terminalHelpCommand);
+    terminalRegisterCommand(F("HELP"), _terminalHelpCommand);
 
-    terminalRegisterCommand(F("ERASE.CONFIG"), [](Embedis* e) {
+    terminalRegisterCommand(F("ERASE.CONFIG"), [](const terminal::CommandContext&) {
         terminalOK();
         customResetReason(CUSTOM_RESET_TERMINAL);
         eraseSDKConfig();
         *((int*) 0) = 0; // see https://github.com/esp8266/Arduino/issues/1494
     });
 
-    terminalRegisterCommand(F("FACTORY.RESET"), [](Embedis* e) {
-        resetSettings();
-        terminalOK();
-    });
-
-    terminalRegisterCommand(F("GPIO"), [](Embedis* e) {
+    terminalRegisterCommand(F("GPIO"), [](const terminal::CommandContext& ctx) {
         int pin = -1;
 
-        if (e->argc < 2) {
+        if (ctx.argc < 2) {
             DEBUG_MSG("Printing all GPIO pins:\n");
         } else {
-            pin = String(e->argv[1]).toInt();
+            pin = ctx.argv[1].toInt();
             if (!gpioValid(pin)) {
                 terminalError(F("Invalid GPIO pin"));
                 return;
             }
 
-            if (e->argc > 2) {
-                bool state = String(e->argv[2]).toInt() == 1;
+            if (ctx.argc > 2) {
+                bool state = String(ctx.argv[2]).toInt() == 1;
                 digitalWrite(pin, state);
             }
         }
@@ -219,104 +320,46 @@ void _terminalInitCommand() {
         terminalOK();
     });
 
-    terminalRegisterCommand(F("HEAP"), [](Embedis* e) {
+    terminalRegisterCommand(F("HEAP"), [](const terminal::CommandContext&) {
         infoHeapStats();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("STACK"), [](Embedis* e) {
+    terminalRegisterCommand(F("STACK"), [](const terminal::CommandContext&) {
         infoMemory("Stack", CONT_STACKSIZE, getFreeStack());
         terminalOK();
     });
 
-    terminalRegisterCommand(F("HELP"), [](Embedis* e) {
-        _terminalHelpCommand();
-        terminalOK();
-    });
-
-    terminalRegisterCommand(F("INFO"), [](Embedis* e) {
+    terminalRegisterCommand(F("INFO"), [](const terminal::CommandContext&) {
         info();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("KEYS"), [](Embedis* e) {
-        _terminalKeysCommand();
-        terminalOK();
-    });
-
-    terminalRegisterCommand(F("GET"), [](Embedis* e) {
-        if (e->argc < 2) {
-            terminalError(F("Wrong arguments"));
-            return;
-        }
-
-        for (unsigned char i = 1; i < e->argc; i++) {
-            String key = String(e->argv[i]);
-            String value;
-            if (!Embedis::get(key, value)) {
-                const auto maybeDefault = settingsQueryDefaults(key);
-                if (maybeDefault.length()) {
-                    DEBUG_MSG_P(PSTR("> %s => %s (default)\n"), key.c_str(), maybeDefault.c_str());
-                } else {
-                    DEBUG_MSG_P(PSTR("> %s =>\n"), key.c_str());
-                }
-                continue;
-            }
-
-            DEBUG_MSG_P(PSTR("> %s => \"%s\"\n"), key.c_str(), value.c_str());
-        }
-
-        terminalOK();
-    });
-
-    terminalRegisterCommand(F("RELOAD"), [](Embedis* e) {
-        espurnaReload();
-        terminalOK();
-    });
-
-    terminalRegisterCommand(F("RESET"), [](Embedis* e) {
+    terminalRegisterCommand(F("RESET"), [](const terminal::CommandContext&) {
         terminalOK();
         deferredReset(100, CUSTOM_RESET_TERMINAL);
     });
 
-    terminalRegisterCommand(F("RESET.SAFE"), [](Embedis* e) {
+    terminalRegisterCommand(F("RESET.SAFE"), [](const terminal::CommandContext&) {
         systemStabilityCounter(SYSTEM_CHECK_MAX);
         terminalOK();
         deferredReset(100, CUSTOM_RESET_TERMINAL);
     });
 
-    terminalRegisterCommand(F("UPTIME"), [](Embedis* e) {
+    terminalRegisterCommand(F("UPTIME"), [](const terminal::CommandContext&) {
         infoUptime();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("CONFIG"), [](Embedis* e) {
-        DynamicJsonBuffer jsonBuffer(1024);
-        JsonObject& root = jsonBuffer.createObject();
-        settingsGetJson(root);
-        // XXX: replace with streaming
-        String output;
-        root.printTo(output);
-        DEBUG_MSG(output.c_str());
-        
-    });
-
-    #if not SETTINGS_AUTOSAVE
-        terminalRegisterCommand(F("SAVE"), [](Embedis* e) {
-            eepromCommit();
-            terminalOK();
-        });
-    #endif
-
     #if SECURE_CLIENT == SECURE_CLIENT_BEARSSL
-        terminalRegisterCommand(F("MFLN.PROBE"), [](Embedis* e) {
-            if (e->argc != 3) {
+        terminalRegisterCommand(F("MFLN.PROBE"), [](const terminal::CommandContext& ctx) {
+            if (ctx.argc != 3) {
                 terminalError(F("[url] [value]"));
                 return;
             }
 
-            URL _url(e->argv[1]);
-            uint16_t requested_mfln = atol(e->argv[2]);
+            URL _url(ctx.argv[1]);
+            uint16_t requested_mfln = atol(ctx.argv[2].c_str());
 
             auto client = std::make_unique<BearSSL::WiFiClientSecure>();
             client->setInsecure();
@@ -330,16 +373,16 @@ void _terminalInitCommand() {
     #endif
 
     #if LWIP_VERSION_MAJOR != 1
-        terminalRegisterCommand(F("HOST"), [](Embedis* e) {
-            if (e->argc != 2) {
+        terminalRegisterCommand(F("HOST"), [](const terminal::CommandContext& ctx) {
+            if (ctx.argc != 2) {
                 terminalError(F("HOST [hostname]"));
                 return;
             }
 
             ip_addr_t result;
-            auto error = dns_gethostbyname(e->argv[1], &result, _terminalDnsFound, nullptr);
+            auto error = dns_gethostbyname(ctx.argv[1].c_str(), &result, _terminalDnsFound, nullptr);
             if (error == ERR_OK) {
-                _terminalPrintDnsResult(e->argv[1], &result);
+                _terminalPrintDnsResult(ctx.argv[1].c_str(), &result);
                 terminalOK();
                 return;
             } else if (error != ERR_INPROGRESS) {
@@ -349,30 +392,56 @@ void _terminalInitCommand() {
 
         });
 
-        terminalRegisterCommand(F("NETSTAT"), [](Embedis*) {
+        terminalRegisterCommand(F("NETSTAT"), [](const terminal::CommandContext&) {
             _terminalPrintTcpPcbs();
         });
 
     #endif // LWIP_VERSION_MAJOR != 1
-    
+
 }
 
 void _terminalLoop() {
 
     #if DEBUG_SERIAL_SUPPORT
         while (DEBUG_PORT.available()) {
-            _serial.inject(DEBUG_PORT.read());
+            _io.inject(DEBUG_PORT.read());
         }
     #endif
 
-    embedis.process();
+    _terminal.process([](terminal::Terminal::Result result) {
+        bool out = false;
+        switch (result) {
+            case terminal::Terminal::Result::CommandNotFound:
+                terminalError(terminalDefaultStream(), F("Command not found"));
+                out = true;
+                break;
+            case terminal::Terminal::Result::BufferOverflow:
+                terminalError(terminalDefaultStream(), F("Command line buffer overflow"));
+                out = true;
+                break;
+            case terminal::Terminal::Result::Command:
+                out = true;
+                break;
+            case terminal::Terminal::Result::Pending:
+                out = false;
+                break;
+            case terminal::Terminal::Result::Error:
+                terminalError(terminalDefaultStream(), F("Unexpected error when parsing command line"));
+                out = false;
+                break;
+            case terminal::Terminal::Result::NoInput:
+                out = false;
+                break;
+        }
+        return out;
+    });
 
     #if SERIAL_RX_ENABLED
 
         while (SERIAL_RX_PORT.available() > 0) {
             char rc = SERIAL_RX_PORT.read();
             _serial_rx_buffer[_serial_rx_pointer++] = rc;
-            if ((_serial_rx_pointer == TERMINAL_BUFFER_SIZE) || (rc == 10)) {
+            if ((_serial_rx_pointer == SerialRxBufferSize) || (rc == 10)) {
                 terminalInject(_serial_rx_buffer, (size_t) _serial_rx_pointer);
                 _serial_rx_pointer = 0;
             }
@@ -382,52 +451,176 @@ void _terminalLoop() {
 
 }
 
+#if WEB_SUPPORT && TERMINAL_WEB_API_SUPPORT
+
+bool _terminalWebApiMatchPath(AsyncWebServerRequest* request) {
+    const String api_path = getSetting("termWebApiPath", TERMINAL_WEB_API_PATH);
+    return request->url().equals(api_path);
+}
+
+void _terminalWebApiSetup() {
+
+    webRequestRegister([](AsyncWebServerRequest* request) {
+        // continue to the next handler if path does not match
+        if (!_terminalWebApiMatchPath(request)) return false;
+
+        // return 'true' after this point, since we did handle the request
+        webLog(request);
+        if (!apiAuthenticate(request)) return true;
+
+        auto* cmd_param = request->getParam("line", (request->method() == HTTP_PUT));
+        if (!cmd_param) {
+            request->send(500);
+            return true;
+        }
+
+        auto cmd = cmd_param->value();
+        if (!cmd.length()) {
+            request->send(500);
+            return true;
+        }
+
+        if (!cmd.endsWith("\r\n") && !cmd.endsWith("\n")) {
+            cmd += '\n';
+        }
+
+        // TODO: batch requests? processLine() -> process(...)
+        AsyncWebPrint::scheduleFromRequest(request, [cmd](Print& print) {
+            StreamAdapter<const char*> stream(print, cmd.c_str(), cmd.c_str() + cmd.length() + 1);
+            terminal::Terminal handler(stream);
+            handler.processLine();
+        });
+
+        return true;
+    });
+
+}
+
+#endif // WEB_SUPPORT && TERMINAL_WEB_API_SUPPORT
+
+
+#if MQTT_SUPPORT && TERMINAL_MQTT_SUPPORT
+
+void _terminalMqttSetup() {
+
+    mqttRegister([](unsigned int type, const char * topic, const char * payload) {
+        if (type == MQTT_CONNECT_EVENT) {
+            mqttSubscribe(MQTT_TOPIC_CMD);
+            return;
+        }
+
+        if (type == MQTT_MESSAGE_EVENT) {
+            String t = mqttMagnitude((char *) topic);
+            if (!t.startsWith(MQTT_TOPIC_CMD)) return;
+            if (!strlen(payload)) return;
+
+            String cmd(payload);
+            if (!cmd.endsWith("\r\n") && !cmd.endsWith("\n")) {
+                cmd += '\n';
+            }
+
+            // TODO: unlike http handler, we have only one output stream
+            //       and **must** have a fixed-size output buffer
+            //       (wishlist: MQTT client does some magic and we don't buffer twice)
+            schedule_function([cmd]() {
+                PrintString buffer(TCP_MSS);
+                StreamAdapter<const char*> stream(buffer, cmd.c_str(), cmd.c_str() + cmd.length() + 1);
+
+                String out;
+                terminal::Terminal handler(stream);
+                switch (handler.processLine()) {
+                case terminal::Terminal::Result::CommandNotFound:
+                    out += F("Command not found");
+                    break;
+                case terminal::Terminal::Result::Command:
+                    out = std::move(buffer);
+                default:
+                    break;
+                }
+
+                if (out.length()) {
+                    mqttSendRaw(mqttTopic(MQTT_TOPIC_CMD, false).c_str(), out.c_str(), false);
+                }
+            });
+
+            return;
+        }
+    });
+
+}
+
+#endif // MQTT_SUPPORT && TERMINAL_MQTT_SUPPORT
+
+}
+
 // -----------------------------------------------------------------------------
 // Pubic API
 // -----------------------------------------------------------------------------
 
+Stream & terminalDefaultStream() {
+    return (Stream &) _io;
+}
+
+size_t terminalCapacity() {
+    return _io.capacity();
+}
+
 void terminalInject(void *data, size_t len) {
-    _serial.inject((char *) data, len);
+    _io.inject((char *) data, len);
 }
 
 void terminalInject(char ch) {
-    _serial.inject(ch);
+    _io.inject(ch);
 }
 
+void terminalRegisterCommand(const String& name, terminal::Terminal::CommandFunc func) {
+    terminal::Terminal::addCommand(name, func);
+};
 
-Stream & terminalSerial() {
-    return (Stream &) _serial;
+void terminalOK(Print& print) {
+    print.print(F("+OK\n"));
 }
 
-void terminalRegisterCommand(const String& name, embedis_command_f command) {
-    Embedis::command(name, command);
-};
+void terminalError(Print& print, const String& error) {
+    print.printf("-ERROR: %s\n", error.c_str());
+}
+
+void terminalOK(const terminal::CommandContext& ctx) {
+    terminalOK(ctx.output);
+}
+
+void terminalError(const terminal::CommandContext& ctx, const String& error) {
+    terminalError(ctx.output, error);
+}
 
 void terminalOK() {
-    DEBUG_MSG_P(PSTR("+OK\n"));
+    terminalOK(_io);
 }
 
 void terminalError(const String& error) {
-    DEBUG_MSG_P(PSTR("-ERROR: %s\n"), error.c_str());
+    terminalError(_io, error);
 }
 
 void terminalSetup() {
 
-    _serial.callback([](uint8_t ch) {
-        #if TELNET_SUPPORT
-            telnetWrite(ch);
-        #endif
-        #if DEBUG_SERIAL_SUPPORT
-            DEBUG_PORT.write(ch);
-        #endif
-    });
-
+    // Show DEBUG panel with input
     #if WEB_SUPPORT
         wsRegister()
             .onVisible([](JsonObject& root) { root["cmdVisible"] = 1; });
     #endif
 
-    _terminalInitCommand();
+    // Run terminal command and send back the result. Depends on the terminal command using ctx.output
+    #if WEB_SUPPORT && TERMINAL_WEB_API_SUPPORT
+        _terminalWebApiSetup();
+    #endif
+
+    // Similar to the above, but we allow only very small and in-place outputs.
+    #if MQTT_SUPPORT && TERMINAL_MQTT_SUPPORT
+        _terminalMqttSetup();
+    #endif
+
+    // Initialize default commands
+    _terminalInitCommands();
 
     #if SERIAL_RX_ENABLED
         SERIAL_RX_PORT.begin(SERIAL_RX_BAUDRATE);

code/espurna/terminal.h

@@ -12,17 +12,29 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 
 #if TERMINAL_SUPPORT
 
-#include "libs/EmbedisWrap.h"
+#include <functional>
+#include <memory>
+#include <unordered_map>
+#include <vector>
 
-using embedis_command_f = void (*)(Embedis*);
+#include "terminal_parsing.h"
+#include "terminal_commands.h"
 
 void terminalOK();
 void terminalError(const String& error);
 
-void terminalRegisterCommand(const String& name, embedis_command_f func);
+void terminalOK(Print&);
+void terminalError(Print&, const String& error);
+
+void terminalOK(const terminal::CommandContext&);
+void terminalError(const terminal::CommandContext&, const String&);
+
+void terminalRegisterCommand(const String& name, terminal::Terminal::CommandFunc func);
+
+size_t terminalCapacity();
 void terminalInject(void *data, size_t len);
 void terminalInject(char ch);
-Stream& terminalSerial();
+Stream& terminalDefaultStream();
 
 void terminalSetup();
 

code/espurna/terminal_commands.cpp

@@ -0,0 +1,93 @@
+/*
+
+Part of the TERMINAL MODULE
+
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+Heavily inspired by the Embedis design:
+- https://github.com/thingSoC/embedis
+
+*/
+
+#include <Arduino.h>
+
+#include "terminal_commands.h"
+
+#include <memory>
+
+namespace terminal {
+
+std::unordered_map<String, Terminal::CommandFunc,
+    parsing::LowercaseFnv1Hash<String>,
+    parsing::LowercaseEquals<String>> Terminal::commands;
+
+void Terminal::addCommand(const String& name, CommandFunc func) {
+    if (!func) return;
+    commands.emplace(std::make_pair(name, func));
+}
+
+size_t Terminal::commandsSize() {
+    return commands.size();
+}
+
+std::vector<String> Terminal::commandNames() {
+    std::vector<String> out;
+    out.reserve(commands.size());
+    for (auto& command : commands) {
+        out.push_back(command.first);
+    }
+    return out;
+}
+
+Terminal::Result Terminal::processLine() {
+
+    // Arduino stream API returns either `char` >= 0 or -1 on error
+    int c = -1;
+    while ((c = stream.read()) >= 0) {
+        if (buffer.size() >= (buffer_size - 1)) {
+            buffer.clear();
+            return Result::BufferOverflow;
+        }
+        buffer.push_back(c);
+        if (c == '\n') {
+            // in case we see \r\n, offset minus one and overwrite \r
+            auto end = buffer.end() - 1;
+            if (*(end - 1) == '\r') {
+                --end;
+            }
+            *end = '\0';
+
+            // parser should pick out at least one arg (command)
+            auto cmdline = parsing::parse_commandline(buffer.data());
+            buffer.clear();
+            if (cmdline.argc >= 1) {
+                auto command = commands.find(cmdline.argv[0]);
+                if (command == commands.end()) return Result::CommandNotFound;
+                (*command).second(CommandContext{std::move(cmdline.argv), cmdline.argc, stream});
+                return Result::Command;
+            }
+        }
+    }
+
+    // we need to notify about the fixable things
+    if (buffer.size() && (c < 0)) {
+        return Result::Pending;
+    } else if (!buffer.size() && (c < 0)) {
+        return Result::NoInput;
+    // ... and some unexpected conditions
+    } else {
+        return Result::Error;
+    }
+
+}
+
+bool Terminal::defaultProcessFunc(Result result) {
+    return (result != Result::Error) && (result != Result::NoInput);
+}
+
+void Terminal::process(ProcessFunc func) {
+    while (func(processLine())) {
+    }    
+}
+
+} // ns terminal

code/espurna/terminal_commands.h

@@ -0,0 +1,90 @@
+/*
+
+Part of the TERMINAL MODULE
+
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#pragma once
+
+#include <Arduino.h>
+
+#include "terminal_parsing.h"
+
+#include <unordered_map>
+#include <functional>
+#include <vector>
+
+namespace terminal {
+
+struct Terminal;
+
+// We need to be able to pass arbitrary Args structure into the command function
+// Like Embedis implementation, we only pass things that we actually use instead of complete obj instance
+struct CommandContext {
+    std::vector<String> argv;
+    size_t argc;
+    Print& output;
+};
+
+struct Terminal {
+
+    enum class Result {
+        Error,           // Genric error condition
+        Command,         // We successfully parsed the line and executed the callback specified via addCommand
+        CommandNotFound, // ... similar to the above, but command was never added via addCommand
+        BufferOverflow,  // Command line processing failed, no \r\n / \n before buffer was filled
+        Pending,         // We got something in the buffer, but can't yet do anything with it
+        NoInput          // We got nothing in the buffer and stream read() returns -1
+    };
+
+    using CommandFunc = void(*)(const CommandContext&);
+    using ProcessFunc = bool(*)(Result);
+
+    // stream      - see `stream` description below
+    // buffer_size - set internal limit for the total command line length
+    Terminal(Stream& stream, size_t buffer_size = 128) :
+        stream(stream),
+        buffer_size(buffer_size)
+    {
+        buffer.reserve(buffer_size);
+    }
+
+    static void addCommand(const String& name, CommandFunc func);
+    static size_t commandsSize();
+    static std::vector<String> commandNames();
+
+    // Try to process a single line (until either `\r\n` or just `\n`)
+    Result processLine();
+
+    // Calls processLine() repeatedly.
+    // Blocks until the stream no longer has any data available.
+    // `process_f` will return each individual processLine() Result,
+    // and we can either stop (false) or continue (true) the function.
+    void process(ProcessFunc = defaultProcessFunc);
+
+    private:
+
+    static bool defaultProcessFunc(Result);
+
+    // general input / output stream:
+    // - stream.read() should return user iput
+    // - stream.write() can be called from the command callback
+    // - stream.write() can be called by us to show error messages
+    Stream& stream;
+
+    // buffer for the input stream, fixed in size
+    std::vector<char> buffer;
+    const size_t buffer_size;
+
+    // TODO: every command is shared, instance should probably also have an
+    //       option to add 'private' commands list?
+    // Note: we can save ~2.5KB by using std::vector<std::pair<String, CommandFunc>>
+    //       https://github.com/xoseperez/espurna/pull/2247#issuecomment-633689741
+    static std::unordered_map<String, CommandFunc,
+        parsing::LowercaseFnv1Hash<String>,
+        parsing::LowercaseEquals<String>> commands;
+
+};
+}

code/espurna/terminal_parsing.cpp

@@ -0,0 +1,228 @@
+/*
+
+Part of the TERMINAL MODULE
+
+Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#include <vector>
+#include <cctype>
+
+#include "terminal_parsing.h"
+
+namespace terminal {
+namespace parsing {
+
+
+// c/p with minor modifications from redis / sds, so that we don't have to roll a custom parser
+// ref:
+// - https://github.com/antirez/sds/blob/master/sds.c
+// - https://github.com/antirez/redis/blob/unstable/src/networking.c
+//
+// Things are kept mostly the same, we are replacing Redis-specific things:
+// - sds structure -> String
+// - sds array -> std::vector<String>
+// - we return always return custom structure, nullptr can no longer be used
+//   to notify about the missing / unterminated / mismatching quotes
+// - hex_... function helpers types are changed
+
+// Original code is part of the SDSLib 2.0 -- A C dynamic strings library
+//  *
+//  * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
+//  * Copyright (c) 2015, Oran Agra
+//  * Copyright (c) 2015, Redis Labs, Inc
+//  * All rights reserved.
+//  *
+//  * Redistribution and use in source and binary forms, with or without
+//  * modification, are permitted provided that the following conditions are met:
+//  *
+//  *   * Redistributions of source code must retain the above copyright notice,
+//  *     this list of conditions and the following disclaimer.
+//  *   * Redistributions in binary form must reproduce the above copyright
+//  *     notice, this list of conditions and the following disclaimer in the
+//  *     documentation and/or other materials provided with the distribution.
+//  *   * Neither the name of Redis nor the names of its contributors may be used
+//  *     to endorse or promote products derived from this software without
+//  *     specific prior written permission.
+//  *
+//  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+//  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+//  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+//  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+//  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+//  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+//  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+//  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+//  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+//  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  * POSSIBILITY OF SUCH DAMAGE.
+
+// Helper functions to handle \xHH codes
+static bool is_hex_digit(char c) {
+    return (c >= '0' && c <= '9') \
+         ||(c >= 'a' && c <= 'f') \
+         ||(c >= 'A' && c <= 'F');
+}
+
+static char hex_digit_to_int(char c) {
+    switch (c) {
+        case '0': return 0;
+        case '1': return 1;
+        case '2': return 2;
+        case '3': return 3;
+        case '4': return 4;
+        case '5': return 5;
+        case '6': return 6;
+        case '7': return 7;
+        case '8': return 8;
+        case '9': return 9;
+        case 'a': case 'A': return 10;
+        case 'b': case 'B': return 11;
+        case 'c': case 'C': return 12;
+        case 'd': case 'D': return 13;
+        case 'e': case 'E': return 14;
+        case 'f': case 'F': return 15;
+        default: return 0;
+    }
+}
+
+// Our port of `sdssplitargs`
+CommandLine parse_commandline(const char *line) {
+    const char *p = line;
+
+    CommandLine result {{}, 0};
+    result.argv.reserve(4);
+
+    String current;
+
+    while(1) {
+        /* skip blanks */
+        while(*p && isspace(*p)) p++;
+        if (*p) {
+            /* get a token */
+            int inq=0;  /* set to 1 if we are in "quotes" */
+            int insq=0; /* set to 1 if we are in 'single quotes' */
+            int done=0;
+
+            while(!done) {
+                if (inq) {
+                    if (*p == '\\' && *(p+1) == 'x' &&
+                                             is_hex_digit(*(p+2)) &&
+                                             is_hex_digit(*(p+3)))
+                    {
+                        // XXX: make sure that we append `char` or `char[]`,
+                        // even with -funsigned-char this can accidentally append itoa conversion
+                        unsigned char byte = 
+                            (hex_digit_to_int(*(p+2))*16)+
+                            hex_digit_to_int(*(p+3));
+                        char buf[2] { static_cast<char>(byte), 0x00 };
+                        current += buf;
+                        p += 3;
+                    } else if (*p == '\\' && *(p+1)) {
+                        char c;
+
+                        p++;
+                        switch(*p) {
+                        case 'n': c = '\n'; break;
+                        case 'r': c = '\r'; break;
+                        case 't': c = '\t'; break;
+                        case 'b': c = '\b'; break;
+                        case 'a': c = '\a'; break;
+                        default: c = *p; break;
+                        }
+                        current += c;
+                    } else if (*p == '"') {
+                        /* closing quote must be followed by a space or
+                         * nothing at all. */
+                        if (*(p+1) && !isspace(*(p+1))) goto err;
+                        done=1;
+                    } else if (!*p) {
+                        /* unterminated quotes */
+                        goto err;
+                    } else {
+                        char buf[2] {*p, '\0'};
+                        current += buf;
+                    }
+                } else if (insq) {
+                    if (*p == '\\' && *(p+1) == '\'') {
+                        p++;
+                        current += '\'';
+                    } else if (*p == '\'') {
+                        /* closing quote must be followed by a space or
+                         * nothing at all. */
+                        if (*(p+1) && !isspace(*(p+1))) goto err;
+                        done=1;
+                    } else if (!*p) {
+                        /* unterminated quotes */
+                        goto err;
+                    } else {
+                        char buf[2] {*p, '\0'};
+                        current += buf;
+                    }
+                } else {
+                    switch(*p) {
+                    case ' ':
+                    case '\n':
+                    case '\r':
+                    case '\t':
+                    case '\0':
+                        done=1;
+                        break;
+                    case '"':
+                        inq=1;
+                        break;
+                    case '\'':
+                        insq=1;
+                        break;
+                    default: {
+                        char buf[2] {*p, '\0'};
+                        current += buf;
+                        break;
+                    }
+                    }
+                }
+                if (*p) p++;
+            }
+            /* add the token to the vector */
+            result.argv.emplace_back(std::move(current));
+            ++result.argc;
+        } else {
+            /* Even on empty input string return something not NULL. */
+            return result;
+        }
+    }
+
+err:
+    result.argc = 0;
+    result.argv.clear();
+    return result;
+}
+
+// Fowler–Noll–Vo hash function to hash command strings that treats input as lowercase
+// ref: https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
+template<>
+size_t LowercaseFnv1Hash<String>::operator()(const String& str) const {
+    constexpr uint32_t fnv_prime = 16777619u;
+    constexpr uint32_t fnv_basis = 2166136261u;
+
+    uint32_t hash = fnv_basis;
+    for (size_t idx = 0; idx < str.length(); ++idx) {
+        // TODO: String::operator[] is slightly slower here
+        //       does not happen with the std::string
+        hash = hash ^ static_cast<uint32_t>(tolower(str.c_str()[idx]));
+        hash = hash * fnv_prime;
+    }
+
+    return hash;
+}
+
+template<>
+bool LowercaseEquals<String>::operator()(const String& lhs, const String& rhs) const {
+    return lhs.equalsIgnoreCase(rhs);
+}
+
+
+} // namespace parsing
+} // namespace terminal

code/espurna/terminal_parsing.h

@@ -0,0 +1,45 @@
+/*
+
+Part of the TERMINAL MODULE
+
+Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#pragma once
+
+#include <Arduino.h>
+#include <vector>
+
+namespace terminal {
+namespace parsing {
+
+// Generic command line parser
+// - split each arg from the input line and put them into the argv array
+// - argc is expected to be equal to the argv
+struct CommandLine {
+    std::vector<String> argv;
+    size_t argc;
+};
+
+CommandLine parse_commandline(const char *line);
+
+// Fowler–Noll–Vo hash function to hash command strings that treats input as lowercase
+// ref: https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
+//
+// TODO: afaik, unordered_map should handle collisions (however rare they are in our case)
+//       if not, we can always roll static commands allocation and just match strings
+//       with LowercaseEquals (which is not that much slower)
+template <typename T>
+struct LowercaseFnv1Hash {
+    size_t operator()(const T& str) const;
+};
+
+template <typename T>
+struct LowercaseEquals {
+    bool operator()(const T& lhs, const T& rhs) const;
+};
+
+}
+}

code/espurna/tuya.cpp

@@ -510,7 +510,7 @@ namespace Tuya {
 
         #if TERMINAL_SUPPORT
 
-            terminalRegisterCommand(F("TUYA.SHOW"), [](Embedis* e) {
+            terminalRegisterCommand(F("TUYA.SHOW"), [](const terminal::CommandContext&) {
                 static const char fmt[] PROGMEM = "%12s%u => dp=%u value=%u\n";
                 showProduct();
 
@@ -525,7 +525,7 @@ namespace Tuya {
                 #endif
             });
 
-            terminalRegisterCommand(F("TUYA.SAVE"), [](Embedis* e) {
+            terminalRegisterCommand(F("TUYA.SAVE"), [](const terminal::CommandContext&) {
                 DEBUG_MSG_P(PSTR("[TUYA] Saving current configuration ...\n"));
                 for (unsigned char n=0; n < switchStates.size(); ++n) {
                     setSetting({"tuyaSwitch", n}, switchStates[n].dp);

code/espurna/web.cpp

@@ -10,6 +10,12 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 
 #if WEB_SUPPORT
 
+#include <algorithm>
+#include <functional>
+#include <memory>
+
+#include <Schedule.h>
+
 #include "system.h"
 #include "utils.h"
 #include "ntp.h"
@@ -43,6 +49,151 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #include "static/server.key.h"
 #endif // WEB_SSL_ENABLED
 
+
+AsyncWebPrint::AsyncWebPrint(const AsyncWebPrintConfig& config, AsyncWebServerRequest* request) :
+    mimeType(config.mimeType),
+    backlogCountMax(config.backlogCountMax),
+    backlogSizeMax(config.backlogSizeMax),
+    backlogTimeout(config.backlogTimeout),
+    _request(request),
+    _state(State::None)
+{}
+
+bool AsyncWebPrint::_addBuffer() {
+    if ((_buffers.size() + 1) > backlogCountMax) {
+        if (!_exhaustBuffers()) {
+            _state = State::Error;
+            return false;
+        }
+    }
+
+    // Note: c++17, emplace returns created object reference
+    //       c++11, we need to use .back()
+    _buffers.emplace_back(backlogSizeMax, 0);
+    _buffers.back().clear();
+
+    return true;
+}
+
+// Creates response object that will handle the data written into the Print& interface.
+//
+// This API expects a **very** careful approach to context switching between SYS and CONT:
+// - Returning RESPONSE_TRY_AGAIN before buffers are filled will result in invalid size marker being sent on the wire.
+//   HTTP client (curl, python requests etc., as discovered in testing) will then drop the connection
+// - Returning 0 will immediatly close the connection from our side
+// - Calling _prepareRequest() **before** _buffers are filled will result in returning 0
+// - Calling yield() / delay() while request AsyncWebPrint is active **may** trigger this callback out of sequence
+//   (e.g. Serial.print(..), DEBUG_MSG(...), or any other API trying to switch contexts)
+// - Receiving data (tcp ack from the previous packet) **will** trigger the callback when switching contexts.
+
+void AsyncWebPrint::_prepareRequest() {
+    _state = State::Sending;
+
+    auto *response = _request->beginChunkedResponse(mimeType, [this](uint8_t *buffer, size_t maxLen, size_t index) -> size_t {
+        switch (_state) {
+        case State::None:
+            return RESPONSE_TRY_AGAIN;
+        case State::Error:
+        case State::Done:
+            return 0;
+        case State::Sending:
+            break;
+        }
+
+        size_t written = 0;
+        while ((written < maxLen) && !_buffers.empty()) {
+            auto& chunk =_buffers.front();
+            auto have = maxLen - written;
+            if (chunk.size() > have) {
+                std::copy(chunk.data(), chunk.data() + have, buffer + written);
+                chunk.erase(chunk.begin(), chunk.begin() + have);
+                written += have;
+            } else {
+                std::copy(chunk.data(), chunk.data() + chunk.size(), buffer + written);
+                _buffers.pop_front();
+                written += chunk.size();
+            }
+        }
+
+
+        return written;
+    });
+
+    response->addHeader("Connection", "close");
+    _request->send(response);
+}
+
+void AsyncWebPrint::setState(State state) {
+    _state = state;
+}
+
+AsyncWebPrint::State AsyncWebPrint::getState() {
+    return _state;
+}
+
+size_t AsyncWebPrint::write(uint8_t b) {
+    const uint8_t tmp[1] {b};
+    return write(tmp, 1);
+}
+
+bool AsyncWebPrint::_exhaustBuffers() {
+    // XXX: espasyncwebserver will trigger write callback if we setup response too early
+    //      exploring code, callback handler responds to a special return value RESPONSE_TRY_AGAIN
+    //      but, it seemingly breaks chunked response logic
+    // XXX: this should be **the only place** that can trigger yield() while we stay in CONT
+    if (_state == State::None) {
+        _prepareRequest();
+    }
+
+    const auto start = millis();
+    do {
+        if (millis() - start > 5000) {
+            _buffers.clear();
+            break;
+        }
+        yield();
+    } while (!_buffers.empty());
+
+    return _buffers.empty();
+}
+
+void AsyncWebPrint::flush() {
+    _exhaustBuffers();
+    _state = State::Done;
+}
+
+size_t AsyncWebPrint::write(const uint8_t* data, size_t size) {
+    if (_state == State::Error) {
+        return 0;
+    }
+
+    size_t full_size = size;
+    auto* data_ptr = data;
+
+    while (size) {
+        if (_buffers.empty() && !_addBuffer()) {
+            full_size = 0;
+            break;
+        }
+        auto& current = _buffers.back();
+        const auto have = current.capacity() - current.size();
+        if (have >= size) {
+            current.insert(current.end(), data_ptr, data_ptr + size);
+            size = 0;
+        } else {
+            current.insert(current.end(), data_ptr, data_ptr + have);
+            if (!_addBuffer()) {
+                full_size = 0;
+                break;
+            }
+            data_ptr += have;
+            size -= have;
+        }
+    }
+
+    return full_size;
+}
+
 // -----------------------------------------------------------------------------
 
 AsyncWebServer * _server;

code/espurna/web.h

@@ -13,7 +13,10 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #if WEB_SUPPORT
 
 #include <functional>
+#include <list>
+#include <vector>
 
+#include <Print.h>
 #include <ESPAsyncTCP.h>
 #include <ESPAsyncWebServer.h>
 #include <Hash.h>
@@ -21,6 +24,65 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #include <AsyncJson.h>
 #include <ArduinoJson.h>
 
+struct AsyncWebPrintConfig {
+    const char* const mimeType;
+    const size_t backlogCountMax;
+    const size_t backlogSizeMax;
+    const decltype(millis()) backlogTimeout;
+};
+
+struct AsyncWebPrint : public Print {
+
+    enum class State {
+        None,
+        Sending,
+        Done,
+        Error
+    };
+
+    using BufferType = std::vector<uint8_t>;
+
+    // To be able to safely output data right from the request callback,
+    // we schedule a 'printer' task that will print into the request response buffer via AsyncChunkedResponse
+    // Note: implementation must be included in the header
+    template<typename CallbackType>
+    static void scheduleFromRequest(const AsyncWebPrintConfig& config, AsyncWebServerRequest*, CallbackType);
+
+    template<typename CallbackType>
+    static void scheduleFromRequest(AsyncWebServerRequest*, CallbackType);
+
+    State getState();
+    void setState(State);
+
+    // note: existing implementation only expects this to be available via AsyncWebPrint
+#if defined(ARDUINO_ESP8266_RELEASE_2_3_0)
+    void flush();
+#else
+    void flush() final override;
+#endif
+
+    size_t write(uint8_t) final override;
+    size_t write(const uint8_t *buffer, size_t size) final override;
+
+    const char* const mimeType;
+    const size_t backlogCountMax;
+    const size_t backlogSizeMax;
+    const decltype(millis()) backlogTimeout;
+
+    protected:
+
+    std::list<BufferType> _buffers;
+    AsyncWebServerRequest* const _request;
+    State _state;
+
+    AsyncWebPrint(const AsyncWebPrintConfig&, AsyncWebServerRequest* req);
+
+    bool _addBuffer();
+    bool _exhaustBuffers();
+    void _prepareRequest();
+
+};
+
 using web_body_callback_f = std::function<bool(AsyncWebServerRequest*, uint8_t* data, size_t len, size_t index, size_t total)>;
 using web_request_callback_f = std::function<bool(AsyncWebServerRequest*)>;
 

code/espurna/web_asyncwebprint_impl.h

@@ -0,0 +1,60 @@
+/*
+
+Part of the WEBSERVER MODULE
+
+Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
+
+*/
+
+#pragma once
+
+#include "web.h"
+#include "libs/TypeChecks.h"
+
+#include <Schedule.h>
+
+#if WEB_SUPPORT
+
+namespace asyncwebprint_traits {
+
+template <typename T>
+using print_callable_t = decltype(std::declval<T>()(std::declval<Print&>()));
+
+template <typename T>
+using is_print_callable = is_detected<print_callable_t, T>;
+
+}
+
+template<typename CallbackType>
+void AsyncWebPrint::scheduleFromRequest(const AsyncWebPrintConfig& config, AsyncWebServerRequest* request, CallbackType callback) {
+    static_assert(asyncwebprint_traits::is_print_callable<CallbackType>::value, "CallbackType needs to be a callable with void(Print&)");
+
+    // because of async nature of the server, we need to make sure we outlive 'request' object
+    auto print = std::shared_ptr<AsyncWebPrint>(new AsyncWebPrint(config, request));
+
+    // attach one ptr to onDisconnect capture, so we can detect disconnection before scheduled function runs
+    request->onDisconnect([print]() {
+        print->setState(AsyncWebPrint::State::Done);
+    });
+
+    // attach another capture to the scheduled function, so we execute as soon as we exit next loop()
+    schedule_function([callback, print]() {
+        if (State::None != print->getState()) return;
+        callback(*print.get());
+        print->flush();
+    });
+}
+
+constexpr AsyncWebPrintConfig AsyncWebPrintDefaults {
+    /*mimeType       =*/ "text/plain",
+    /*backlogCountMax=*/ 2,
+    /*backlogSizeMax= */ TCP_MSS,
+    /*backlogTimeout= */ 5000
+};
+
+template<typename CallbackType>
+void AsyncWebPrint::scheduleFromRequest(AsyncWebServerRequest* request, CallbackType callback) {
+    AsyncWebPrint::scheduleFromRequest(AsyncWebPrintDefaults, request, callback);
+}
+
+#endif

code/espurna/wifi.cpp

@@ -377,42 +377,42 @@ void _wifiDebugCallback(justwifi_messages_t code, char * parameter) {
 
 void _wifiInitCommands() {
 
-    terminalRegisterCommand(F("WIFI"), [](Embedis* e) {
+    terminalRegisterCommand(F("WIFI"), [](const terminal::CommandContext&) {
         wifiDebug();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("WIFI.RESET"), [](Embedis* e) {
+    terminalRegisterCommand(F("WIFI.RESET"), [](const terminal::CommandContext&) {
         _wifiConfigure();
         wifiDisconnect();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("WIFI.STA"), [](Embedis* e) {
+    terminalRegisterCommand(F("WIFI.STA"), [](const terminal::CommandContext&) {
         wifiStartSTA();
         terminalOK();
     });
 
-    terminalRegisterCommand(F("WIFI.AP"), [](Embedis* e) {
+    terminalRegisterCommand(F("WIFI.AP"), [](const terminal::CommandContext&) {
         wifiStartAP();
         terminalOK();
     });
 
     #if defined(JUSTWIFI_ENABLE_WPS)
-        terminalRegisterCommand(F("WIFI.WPS"), [](Embedis* e) {
+        terminalRegisterCommand(F("WIFI.WPS"), [](const terminal::CommandContext&) {
             wifiStartWPS();
             terminalOK();
         });
     #endif // defined(JUSTWIFI_ENABLE_WPS)
 
     #if defined(JUSTWIFI_ENABLE_SMARTCONFIG)
-        terminalRegisterCommand(F("WIFI.SMARTCONFIG"), [](Embedis* e) {
+        terminalRegisterCommand(F("WIFI.SMARTCONFIG"), [](const terminal::CommandContext&) {
             wifiStartSmartConfig();
             terminalOK();
         });
     #endif // defined(JUSTWIFI_ENABLE_SMARTCONFIG)
 
-    terminalRegisterCommand(F("WIFI.SCAN"), [](Embedis* e) {
+    terminalRegisterCommand(F("WIFI.SCAN"), [](const terminal::CommandContext&) {
         _wifiScan();
         terminalOK();
     });

code/espurna/wifi.h

@@ -17,6 +17,8 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #include <lwip/etharp.h>
 #endif
 
+// (HACK) allow us to use internal lwip struct.
+// esp8266 re-defines enum values from tcp header... include them first
 #define LWIP_INTERNAL
 #include <JustWifi.h>
 #include <Ticker.h>

code/espurna/ws_internal.h

@@ -106,6 +106,7 @@ struct ws_data_t {
 // -----------------------------------------------------------------------------
 
 using ws_debug_msg_t = std::pair<String, String>;
+using ws_debug_messages_t = std::vector<ws_debug_msg_t>;
 
 struct ws_debug_t {
 
@@ -139,6 +140,6 @@ struct ws_debug_t {
     bool flush;
     size_t current;
     const size_t capacity;
-    std::vector<ws_debug_msg_tpan>> messages;
+    ws_debug_messages_t messages;
 
 };

code/test/build/nondefault.h

@@ -11,3 +11,5 @@
 #define RPN_RULES_SUPPORT 1
 #define SSDP_SUPPORT 1
 #define UART_MQTT_SUPPORT 1
+#define TERMINAL_WEB_API_SUPPORT 1
+#define TERMINAL_MQTT_SUPPORT 1

code/test/platformio.ini

@@ -1,12 +1,20 @@
 [platformio]
 test_dir = unit
+src_dir = ../espurna
 
 [env:test]
 platform = native
 lib_compat_mode = off
+test_build_project_src = true
+src_filter =
+    +<../espurna/terminal_commands.cpp>
+    +<../espurna/terminal_parsing.cpp>
 lib_deps =
     StreamString
     https://github.com/bxparks/UnixHostDuino#d740398e
 build_flags =
+    -DMANUFACTURER="PLATFORMIO"
+    -DDEVICE="TEST"
     -std=gnu++11
+    -Os
     -I../espurna/

code/test/unit/terminal/terminal.cpp

@@ -0,0 +1,266 @@
+#include <unity.h>
+#include <Arduino.h>
+#include <StreamString.h>
+
+#include <terminal_commands.h>
+
+// TODO: should we just use std::function at this point?
+//       we don't actually benefit from having basic ptr functions in handler
+//       test would be simplified too, we would no longer need to have static vars
+
+// Got the idea from the Embedis test suite, set up a proxy for StreamString
+// Real terminal processing happens with ringbuffer'ed stream
+struct IOStreamString : public Stream {
+    StreamString in;
+    StreamString out;
+
+    size_t write(uint8_t ch) final override {
+        return in.write(ch);
+    }
+
+    int read() final override {
+        return out.read();
+    }
+
+    int available() final override {
+        return out.available();
+    }
+
+    int peek() final override {
+        return out.peek();
+    }
+
+    void flush() final override {
+        out.flush();
+    }
+};
+
+// We need to make sure that our changes to split_args actually worked
+
+void test_hex_codes() {
+
+    static bool abc_done = false;
+
+    terminal::Terminal::addCommand("abc", [](const terminal::CommandContext& ctx) {
+        TEST_ASSERT_EQUAL(2, ctx.argc);
+        TEST_ASSERT_EQUAL_STRING("abc", ctx.argv[0].c_str());
+        TEST_ASSERT_EQUAL_STRING("abc", ctx.argv[1].c_str());
+        abc_done = true;
+    });
+
+    IOStreamString str;
+    str.out += String("abc \"\x61\x62\x63\"\r\n");
+
+    terminal::Terminal handler(str);
+
+    TEST_ASSERT_EQUAL(
+        terminal::Terminal::Result::Command,
+        handler.processLine()
+    );
+    TEST_ASSERT(abc_done);
+}
+
+// Ensure that we can register multiple commands (at least 3, might want to test much more in the future?)
+// Ensure that registered commands can be called and they are called in order
+
+void test_multiple_commands() {
+
+    // set up counter to be chained between commands
+    static int command_calls = 0;
+
+    terminal::Terminal::addCommand("test1", [](const terminal::CommandContext& ctx) {
+        TEST_ASSERT_EQUAL_MESSAGE(1, ctx.argc, "Command without args should have argc == 1");
+        TEST_ASSERT_EQUAL(0, command_calls);
+        command_calls = 1;
+    });
+    terminal::Terminal::addCommand("test2", [](const terminal::CommandContext& ctx) {
+        TEST_ASSERT_EQUAL_MESSAGE(1, ctx.argc, "Command without args should have argc == 1");
+        TEST_ASSERT_EQUAL(1, command_calls);
+        command_calls = 2;
+    });
+    terminal::Terminal::addCommand("test3", [](const terminal::CommandContext& ctx) {
+        TEST_ASSERT_EQUAL_MESSAGE(1, ctx.argc, "Command without args should have argc == 1");
+        TEST_ASSERT_EQUAL(2, command_calls);
+        command_calls = 3;
+    });
+
+    IOStreamString str;
+    str.out += String("test1\r\ntest2\r\ntest3\r\n");
+
+    terminal::Terminal handler(str);
+
+    // each processing step only executes a single command
+    static int process_counter = 0;
+
+    handler.process([](terminal::Terminal::Result result) -> bool {
+        if (process_counter == 3) {
+            TEST_ASSERT_EQUAL(result, terminal::Terminal::Result::NoInput);
+            return false;
+        } else {
+            TEST_ASSERT_EQUAL(result, terminal::Terminal::Result::Command);
+            ++process_counter;
+            return true;
+        }
+        TEST_FAIL_MESSAGE("Should not be reached");
+        return false;
+    });
+    TEST_ASSERT_EQUAL(3, command_calls);
+    TEST_ASSERT_EQUAL(3, process_counter);
+
+}
+
+void test_command() {
+
+    static int counter = 0;
+
+    terminal::Terminal::addCommand("test.command", [](const terminal::CommandContext& ctx) {
+        TEST_ASSERT_EQUAL_MESSAGE(1, ctx.argc, "Command without args should have argc == 1");
+        ++counter;
+    });
+
+    IOStreamString str;
+    terminal::Terminal handler(str);
+
+    TEST_ASSERT_EQUAL_MESSAGE(
+        terminal::Terminal::Result::NoInput, handler.processLine(),
+        "We have not read anything yet"
+    );
+
+    str.out += String("test.command\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Command, handler.processLine());
+    TEST_ASSERT_EQUAL_MESSAGE(1, counter, "At this time `test.command` was called just once");
+
+    str.out += String("test.command");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Pending, handler.processLine());
+    TEST_ASSERT_EQUAL_MESSAGE(1, counter, "We are waiting for either \\r\\n or \\n, handler still has data buffered");
+
+    str.out += String("\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Command, handler.processLine());
+    TEST_ASSERT_EQUAL_MESSAGE(2, counter, "We should call `test.command` the second time");
+
+    str.out += String("test.command\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Command, handler.processLine());
+    TEST_ASSERT_EQUAL_MESSAGE(3, counter, "We should call `test.command` the third time, with just LF");
+
+}
+
+// Ensure that we can properly handle arguments
+
+void test_command_args() {
+
+    static bool waiting = false;
+
+    terminal::Terminal::addCommand("test.command.arg1", [](const terminal::CommandContext& ctx) {
+        TEST_ASSERT_EQUAL(2, ctx.argc);
+        waiting = false;
+    });
+
+    terminal::Terminal::addCommand("test.command.arg1_empty", [](const terminal::CommandContext& ctx) {
+        TEST_ASSERT_EQUAL(2, ctx.argc);
+        TEST_ASSERT(!ctx.argv[1].length());
+        waiting = false;
+    });
+
+    IOStreamString str;
+    terminal::Terminal handler(str);
+
+    waiting = true;
+    str.out += String("test.command.arg1 test\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Command, handler.processLine());
+    TEST_ASSERT(!waiting);
+
+    waiting = true;
+    str.out += String("test.command.arg1_empty \"\"\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Command, handler.processLine());
+    TEST_ASSERT(!waiting);
+
+}
+
+// Ensure that we return error when nothing was handled, but we kept feeding the processLine() with data
+
+void test_buffer() {
+
+    IOStreamString str;
+    str.out += String("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n");
+
+    terminal::Terminal handler(str, str.out.available() - 8);
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::BufferOverflow, handler.processLine());
+
+}
+
+// sdssplitargs returns nullptr when quotes are not terminated and empty char for an empty string. we treat it all the same
+
+void test_quotes() {
+
+    terminal::Terminal::addCommand("test.quotes", [](const terminal::CommandContext& ctx) {
+        for (auto& arg : ctx.argv) {
+            TEST_MESSAGE(arg.c_str());
+        }
+        TEST_FAIL_MESSAGE("`test.quotes` should not be called");
+    });
+
+    IOStreamString str;
+    terminal::Terminal handler(str);
+
+    str.out += String("test.quotes \"quote without a pair\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::NoInput, handler.processLine());
+
+    str.out += String("test.quotes 'quote without a pair\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::NoInput, handler.processLine());
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::NoInput, handler.processLine());
+
+}
+
+// we specify that commands lowercase == UPPERCASE, both with hashed values and with equality functions
+// (internal note: we use std::unordered_map at this time)
+
+void test_case_insensitive() {
+
+    terminal::Terminal::addCommand("test.lowercase1", [](const terminal::CommandContext& ctx) {
+        __asm__ volatile ("nop");
+    });
+    terminal::Terminal::addCommand("TEST.LOWERCASE1", [](const terminal::CommandContext& ctx) {
+        TEST_FAIL_MESSAGE("`test.lowercase1` was already registered, this should not be registered / called");
+    });
+
+    IOStreamString str;
+    terminal::Terminal handler(str);
+
+    str.out += String("TeSt.lOwErCaSe1\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Command, handler.processLine());
+    
+}
+
+// We can use command ctx.output to send something back into the stream
+
+void test_output() {
+
+    terminal::Terminal::addCommand("test.output", [](const terminal::CommandContext& ctx) {
+        if (ctx.argc != 2) return;
+        ctx.output.print(ctx.argv[1]);
+    });
+
+    IOStreamString str;
+    terminal::Terminal handler(str);
+
+    char match[] = "test1234567890";
+    str.out += String("test.output ") + String(match) + String("\r\n");
+    TEST_ASSERT_EQUAL(terminal::Terminal::Result::Command, handler.processLine());
+    TEST_ASSERT_EQUAL_STRING(match, str.in.c_str());
+
+}
+
+// When adding test functions, don't forget to add RUN_TEST(...) in the main()
+
+int main(int argc, char** argv) {
+    UNITY_BEGIN();
+    RUN_TEST(test_command);
+    RUN_TEST(test_command_args);
+    RUN_TEST(test_multiple_commands);
+    RUN_TEST(test_hex_codes);
+    RUN_TEST(test_buffer);
+    RUN_TEST(test_quotes);
+    RUN_TEST(test_case_insensitive);
+    RUN_TEST(test_output);
+    UNITY_END();
+}