web: prometheus metrics support (#2332)

- (experimental) provide generic way to read magnitude values
- expose /api/metrics with values formatted specifically for prometheus, with relay and sensor data
- small tweaks to sensor init

Example config:
```
scrape_configs:
  - job_name: 'espurna'
    metrics_path: '/api/metrics'
    params:
      apikey: ['apikeyapikey']
    static_configs:
      - targets: ['espurna-blabla.lan:80']
```
Where ESPurna side has
```
apiKey => "apikeyapikey"
apiEnabled => "1"
```

code/espurna/api.h

@@ -46,6 +46,9 @@ struct Api {
 
     Api() = delete;
 
+    // TODO:
+    // - bind to multiple paths, dispatch specific path in the callback
+    // - allow index to be passed through path argument (/{arg1}/{arg2} syntax, for example)
     Api(const String& path_, Type type_, unsigned char arg_, BasicHandler get_, BasicHandler put_ = nullptr) :
         path(path_),
         type(type_),

code/espurna/board.cpp

@@ -81,6 +81,12 @@ PROGMEM const char espurna_modules[] =
     #if NTP_SUPPORT
         "NTP "
     #endif
+    #if PROMETHEUS_SUPPORT
+        "METRICS "
+    #endif
+    #if RELAY_SUPPORT
+        "RELAY "
+    #endif
     #if RFM69_SUPPORT
         "RFM69 "
     #endif

code/espurna/config/dependencies.h

@@ -209,7 +209,6 @@
 
 //------------------------------------------------------------------------------
 // We should always set MQTT_MAX_PACKET_SIZE
-//
 
 #if MQTT_LIBRARY == MQTT_LIBRARY_PUBSUBCLIENT
 #if not defined(MQTT_MAX_PACKET_SIZE)
@@ -225,3 +224,11 @@
 #undef BME680_SUPPORT
 #define BME680_SUPPORT 0
 #endif
+
+//------------------------------------------------------------------------------
+// Prometheus needs web server + request handler API
+
+#if PROMETHEUS_SUPPORT
+#undef WEB_SUPPORT
+#define WEB_SUPPORT 1
+#endif

code/espurna/config/general.h

@@ -1787,6 +1787,14 @@
 #define MCP23S08_SUPPORT            0
 #endif
 
+//--------------------------------------------------------------------------------
+// Support prometheus metrics export
+//--------------------------------------------------------------------------------
+
+#ifndef PROMETHEUS_SUPPORT
+#define PROMETHEUS_SUPPORT          0
+#endif
+
 // =============================================================================
 // Configuration helpers
 // =============================================================================

code/espurna/config/sensors.h

@@ -1244,6 +1244,11 @@
 // MAX6675
 // Enable support by passing MAX6675_SUPPORT=1 build flag
 //------------------------------------------------------------------------------
+
+#ifndef MAX6675_SUPPORT
+#define MAX6675_SUPPORT                              0
+#endif
+
 #ifndef MAX6675_CS_PIN
 #define MAX6675_CS_PIN                               13
 #endif

code/espurna/main.cpp

@@ -61,6 +61,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include "web.h"
 #include "ws.h"
 #include "mcp23s08.h"
+#include "prometheus.h"
 
 std::vector<void_callback_f> _loop_callbacks;
 std::vector<void_callback_f> _reload_callbacks;
@@ -187,13 +188,18 @@ void setup() {
     #endif
 
     // Multiple modules depend on the generic 'API' services
-    #if API_SUPPORT || TERMINAL_WEB_API_SUPPORT
+    #if API_SUPPORT || TERMINAL_WEB_API_SUPPORT || PROMETHEUS_SUPPORT
         apiCommonSetup();
     #endif
+
     #if API_SUPPORT
         apiSetup();
     #endif
 
+    #if PROMETHEUS_SUPPORT
+        prometheusSetup();
+    #endif
+
     // Hardware GPIO expander, needs to be available for modules down below
     #if MCP23S08_SUPPORT
         MCP23S08Setup();

code/espurna/prometheus.cpp

@@ -0,0 +1,67 @@
+/*
+
+PROMETHEUS METRICS MODULE
+
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#include "espurna.h"
+
+#if WEB_SUPPORT && PROMETHEUS_SUPPORT
+
+#include "prometheus.h"
+
+#include "api.h"
+#include "relay.h"
+#include "sensor.h"
+#include "web.h"
+
+void _prometheusRequestHandler(AsyncWebServerRequest* request) {
+    static_assert(RELAY_SUPPORT || SENSOR_SUPPORT, "");
+
+    // TODO: Add more stuff?
+    // Note: Response 'stream' backing buffer is customizable. Default is 1460 bytes (see ESPAsyncWebServer.h)
+    //       In case printf overflows, memory of CurrentSize+N{overflow} will be allocated to replace
+    //       the existing buffer. Previous buffer will be copied into the new and destroyed after that.
+    AsyncResponseStream *response = request->beginResponseStream("text/plain");
+
+    #if RELAY_SUPPORT
+        for (unsigned char index = 0; index < relayCount(); ++index) {
+            response->printf("relay%u %d\n", index, static_cast<int>(relayStatus(index)));
+        }
+    #endif
+
+    #if SENSOR_SUPPORT
+        char buffer[64] { 0 };
+        for (unsigned char index = 0; index < magnitudeCount(); ++index) {
+            String topic(magnitudeTopicIndex(index));
+            topic.replace("/", "");
+
+            magnitudeFormat(magnitudeValue(index), buffer, sizeof(buffer));
+            response->printf("%s %s\n", topic.c_str(), buffer);
+        }
+    #endif
+
+    response->write('\n');
+
+    request->send(response);
+}
+
+bool _prometheusRequestCallback(AsyncWebServerRequest* request) {
+    if (request->url().equals(F("/api/metrics"))) {
+        webLog(request);
+        if (apiAuthenticate(request)) {
+            _prometheusRequestHandler(request);
+        }
+        return true;
+    }
+
+    return false;
+}
+
+void prometheusSetup() {
+    webRequestRegister(_prometheusRequestCallback);
+}
+
+#endif // PROMETHEUS_SUPPORT

code/espurna/prometheus.h

@@ -0,0 +1,11 @@
+/*
+
+PROMETHEUS METRICS MODULE
+
+Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+
+*/
+
+#pragma once
+
+void prometheusSetup();

code/espurna/sensor.cpp

@@ -10,9 +10,6 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 
 #if SENSOR_SUPPORT
 
-#include <vector>
-#include <float.h>
-
 #include "api.h"
 #include "broker.h"
 #include "domoticz.h"
@@ -25,6 +22,11 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #include "rtcmem.h"
 #include "ws.h"
 
+#include <cfloat>
+#include <cmath>
+#include <limits>
+#include <vector>
+
 //--------------------------------------------------------------------------------
 
 // TODO: namespace { ... } ? sensor ctors need to work though
@@ -215,6 +217,7 @@ struct sensor_magnitude_t {
 
     private:
 
+    constexpr static double _unset = std::numeric_limits<double>::quiet_NaN();
     static unsigned char _counts[MAGNITUDE_MAX];
 
     public:
@@ -223,27 +226,28 @@ struct sensor_magnitude_t {
         return _counts[type];
     }
 
-    sensor_magnitude_t();
+    sensor_magnitude_t() = default;
     sensor_magnitude_t(unsigned char slot, unsigned char index_local, unsigned char type, sensor::Unit units, BaseSensor* sensor);
 
-    BaseSensor * sensor;        // Sensor object
-    BaseFilter * filter;        // Filter object
+    BaseSensor * sensor { nullptr }; // Sensor object
+    BaseFilter * filter { nullptr }; // Filter object
+
+    unsigned char slot { 0u }; // Sensor slot # taken by the magnitude, used to access the measurement
+    unsigned char type { MAGNITUDE_NONE }; // Type of measurement, returned by the BaseSensor::type(slot)
 
-    unsigned char slot;         // Sensor slot # taken by the magnitude, used to access the measurement
-    unsigned char type;         // Type of measurement, returned by the BaseSensor::type(slot)
+    unsigned char index_local { 0u };  // N'th magnitude of it's type, local to the sensor
+    unsigned char index_global { 0u }; // ... and across all of the active sensors
 
-    unsigned char index_local;  // N'th magnitude of it's type, local to the sensor
-    unsigned char index_global; // ... and across all of the active sensors
+    sensor::Unit units { sensor::Unit::None }; // Units of measurement
+    unsigned char decimals { 0u }; // Number of decimals in textual representation
 
-    sensor::Unit units;         // Units of measurement
-    unsigned char decimals;     // Number of decimals in textual representation
+    double last { _unset };     // Last raw value from sensor (unfiltered)
+    double reported { _unset }; // Last reported value
+    double min_change { 0.0 };  // Minimum value change to report
+    double max_change { 0.0 };  // Maximum value change to report
+    double correction { 0.0 };  // Value correction (applied when processing)
 
-    double last;                // Last raw value from sensor (unfiltered)
-    double reported;            // Last reported value
-    double min_change;          // Minimum value change to report
-    double max_change;          // Maximum value change to report
-    double correction;          // Value correction (applied when processing)
-    double zero_threshold;      // Reset value to zero when below threshold (applied when reading)
+    double zero_threshold { _unset }; // Reset value to zero when below threshold (applied when reading)
 
 };
 
@@ -485,36 +489,13 @@ unsigned char _sensor_report_every = SENSOR_REPORT_EVERY;
 // Private
 // -----------------------------------------------------------------------------
 
-sensor_magnitude_t::sensor_magnitude_t() :
-    sensor(nullptr),
-    filter(nullptr),
-    slot(0),
-    type(0),
-    index_local(0),
-    index_global(0),
-    units(sensor::Unit::None),
-    decimals(0),
-    last(0.0),
-    reported(0.0),
-    min_change(0.0),
-    max_change(0.0),
-    correction(0.0)
-{}
-
 sensor_magnitude_t::sensor_magnitude_t(unsigned char slot, unsigned char index_local, unsigned char type, sensor::Unit units, BaseSensor* sensor) :
     sensor(sensor),
-    filter(nullptr),
     slot(slot),
     type(type),
     index_local(index_local),
     index_global(_counts[type]),
-    units(units),
-    decimals(0),
-    last(0.0),
-    reported(0.0),
-    min_change(0.0),
-    max_change(0.0),
-    correction(0.0)
+    units(units)
 {
     ++_counts[type];
 
@@ -2566,11 +2547,36 @@ unsigned char magnitudeType(unsigned char index) {
     return MAGNITUDE_NONE;
 }
 
-double magnitudeValue(unsigned char index) {
-    if (index < _magnitudes.size()) {
-        return _sensor_realtime ? _magnitudes[index].last : _magnitudes[index].reported;
+double sensor::Value::get() {
+    return _sensor_realtime ? last : reported;
+}
+
+sensor::Value magnitudeValue(unsigned char index) {
+    sensor::Value result;
+
+    if (index >= _magnitudes.size()) {
+        result.last = std::numeric_limits<double>::quiet_NaN(),
+        result.reported = std::numeric_limits<double>::quiet_NaN(),
+        result.decimals = 0u;
+        return result;
     }
-    return DBL_MIN;
+
+    auto& magnitude = _magnitudes[index];
+    result.last = magnitude.last;
+    result.reported = magnitude.reported;
+    result.decimals = magnitude.decimals;
+
+    return result;
+}
+
+void magnitudeFormat(const sensor::Value& value, char* out, size_t) {
+    // TODO: 'size' does not do anything, since dtostrf used here is expected to be 'sane', but
+    //       it does not allow any size arguments besides for digits after the decimal point
+    dtostrf(
+        _sensor_realtime ? value.last : value.reported,
+        1, value.decimals,
+        out
+    );
 }
 
 unsigned char magnitudeIndex(unsigned char index) {

code/espurna/sensor.h

@@ -127,6 +127,16 @@ struct Energy {
     Ws ws;
 };
 
+struct Value {
+    constexpr static size_t BufferSize { 33u };
+
+    double get();
+
+    double last;
+    double reported;
+    unsigned char decimals;
+};
+
 }
 
 BrokerDeclare(SensorReadBroker, void(const String&, unsigned char, double, const char*));
@@ -140,7 +150,9 @@ unsigned char magnitudeIndex(unsigned char index);
 String magnitudeTopicIndex(unsigned char index);
 
 unsigned char magnitudeCount();
-double magnitudeValue(unsigned char index);
+
+sensor::Value magnitudeValue(unsigned char index);
+void magnitudeFormat(const sensor::Value& value, char* output, size_t size);
 
 // XXX: without param name it is kind of vague what exactly unsigned char is
 //      consider adding stronger param type e.g. enum class

code/espurna/thermostat.cpp

@@ -16,6 +16,9 @@ Copyright (C) 2017 by Dmitry Blinov <dblinov76 at gmail dot com>
 #include "mqtt.h"
 #include "ws.h"
 
+#include <limits>
+#include <cmath>
+
 const char* NAME_THERMOSTAT_ENABLED     = "thermostatEnabled";
 const char* NAME_THERMOSTAT_MODE        = "thermostatMode";
 const char* NAME_TEMP_RANGE_MIN         = "tempRangeMin";
@@ -371,35 +374,28 @@ void updateCounters() {
 }
 
 //------------------------------------------------------------------------------
+double _getLocalValue(const char* description, unsigned char type) {
+#if SENSOR_SUPPORT
+    for (unsigned char index = 0; index < magnitudeCount(); ++index) {
+        if (magnitudeType(index) != type) continue;
+        auto value = magnitudeValue(index);
+
+        char tmp_str[16];
+        magnitudeFormat(value, tmp_str, sizeof(tmp_str));
+        DEBUG_MSG_P(PSTR("[THERMOSTAT] %s: %s\n"), description, tmp_str);
+
+        return value.get();
+    }
+#endif
+  return std::numeric_limits<double>::quiet_NaN();
+}
+
 double getLocalTemperature() {
-  #if SENSOR_SUPPORT
-      for (byte i=0; i<magnitudeCount(); i++) {
-          if (magnitudeType(i) == MAGNITUDE_TEMPERATURE) {
-              double temp = magnitudeValue(i);
-              char tmp_str[16];
-              dtostrf(temp, 1, 1, tmp_str);
-              DEBUG_MSG_P(PSTR("[THERMOSTAT] getLocalTemperature temp: %s\n"), tmp_str);
-              return temp > -0.1 && temp < 0.1 ? DBL_MIN : temp;
-          }
-      }
-  #endif
-  return DBL_MIN;
+    return _getLocalValue("getLocalTemperature", MAGNITUDE_TEMPERATURE);
 }
 
-//------------------------------------------------------------------------------
 double getLocalHumidity() {
-  #if SENSOR_SUPPORT
-      for (byte i=0; i<magnitudeCount(); i++) {
-          if (magnitudeType(i) == MAGNITUDE_HUMIDITY) {
-              double hum = magnitudeValue(i);
-              char tmp_str[16];
-              dtostrf(hum, 1, 0, tmp_str);
-              DEBUG_MSG_P(PSTR("[THERMOSTAT] getLocalHumidity hum: %s\%\n"), tmp_str);
-              return hum > -0.1 && hum < 0.1 ? DBL_MIN : hum;
-          }
-      }
-  #endif
-  return DBL_MIN;
+    return _getLocalValue("getLocalHumidity", MAGNITUDE_HUMIDITY);
 }
 
 //------------------------------------------------------------------------------
@@ -428,7 +424,7 @@ void thermostatLoop(void) {
       _thermostat.temperature_source = temp_remote;
       DEBUG_MSG_P(PSTR("[THERMOSTAT] setup thermostat by remote temperature\n"));
       checkTempAndAdjustRelay(_remote_temp.temp);
-    } else if (getLocalTemperature() != DBL_MIN) {
+    } else if (!std::isnan(getLocalTemperature())) {
       // we have local temp
       _thermostat.temperature_source = temp_local;
       DEBUG_MSG_P(PSTR("[THERMOSTAT] setup thermostat by local temperature\n"));
@@ -602,7 +598,7 @@ void display_local_temp() {
   String local_temp_title = String("Local      t");
   display.drawString(0, 32, local_temp_title);
 
-  String local_temp_vol = String("= ") + (getLocalTemperature() != DBL_MIN ? String(getLocalTemperature(), 1) : String("?")) + "°";
+  String local_temp_vol = String("= ") + (!std::isnan(getLocalTemperature()) ? String(getLocalTemperature(), 1) : String("?")) + "°";
   display.drawString(75, 32, local_temp_vol);
 
   _display_need_refresh = true;
@@ -619,7 +615,7 @@ void display_local_humidity() {
   String local_hum_title = String("Local      h ");
   display.drawString(0, 48, local_hum_title);
 
-  String local_hum_vol = String("= ") + (getLocalHumidity() != DBL_MIN ? String(getLocalHumidity(), 0) : String("?")) + "%";
+  String local_hum_vol = String("= ") + (!std::isnan(getLocalHumidity()) ? String(getLocalHumidity(), 0) : String("?")) + "%";
   display.drawString(75, 48, local_hum_vol);
 
   _display_need_refresh = true;

code/espurna/web.cpp

@@ -469,10 +469,11 @@ void _onRequest(AsyncWebServerRequest *request){
 
     if (!_onAPModeRequest(request)) return;
 
-    // Send request to subscribers
-    for (unsigned char i = 0; i < _web_request_callbacks.size(); i++) {
-        bool response = (_web_request_callbacks[i])(request);
-        if (response) return;
+    // Send request to subscribers, break when request is 'handled' by the callback
+    for (auto& callback : _web_request_callbacks) {
+        if (callback(request)) {
+            return;
+        }
     }
 
     // No subscriber handled the request, return a 404 with implicit "Connection: close"

code/test/build/nondefault.h

@@ -1,3 +1,4 @@
+#define SENSOR_SUPPORT 1
 #define INFLUXDB_SUPPORT 1
 #define KINGART_CURTAIN_SUPPORT 1
 #define LLMNR_SUPPORT 1
@@ -11,5 +12,6 @@
 #define UART_MQTT_SUPPORT 1
 #define TERMINAL_WEB_API_SUPPORT 1
 #define TERMINAL_MQTT_SUPPORT 1
+#define PROMETHEUS_SUPPORT 1
 #define RFB_SUPPORT 1
 #define RFB_PROVIDER RFB_PROVIDER_RCSWITCH