Browse Source
Rework settings (#2282)
* wip based on early draft. todo benchmarking * fixup eraser, assume keys are unique * fix cursor copy, test removal at random * small benchmark via permutations. todo lambdas and novirtual * fix empty condition / reset * overwrite optimizations, fix move offsets overflows * ...erase using 0xff instead of 0 * test the theory with code, different length kv were bugged * try to check for out-of-bounds writes / reads * style * trying to fix mover again * clarify length, defend against reading len on edge * fix uncommited rewind change * prove space assumptions * more concise traces, fix move condition (agrh!!!) * slightly more internal knowledge (estimates API?) * make sure cursor is only valid within the range * ensure 0 does not blow things * go back up * cursor comments * comments * rewrite writes through cursor * in del too * estimate kv storage requirements, return available size * move raw erase / move into a method, allow ::set to avoid scanning storage twice * refactor naming, use in code * amend storage slicing test * fix crash handler offsets, cleanup configuration * start -> begin * eeprom readiness * dependencies * unused * SPI_FLASH constants for older Core * vtables -> templates * less include dependencies * gcov help, move estimate outside of the class * writer position can never match, use begin + offset * tweak save_crash to trigger only once in a serious crash * doh, header function should be inline * foreach api, tweak structs for public api * use test helper class * when not using foreach, move cursor reset closer to the loop using read_kv * coverage comments, fix typo in tests decltype * ensure set() does not break with offset * make codacy happy againmcspr-patch-1
Max Prokhorov
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1484 additions and 274 deletions
Unified View
Diff Options
-
+1 -1code/espurna/board.cpp
-
+1 -0code/espurna/config/all.h
-
+0 -11code/espurna/config/general.h
-
+162 -81code/espurna/crash.cpp
-
+21 -8code/espurna/crash.h
-
+1 -1code/espurna/debug.cpp
-
+1 -1code/espurna/espurna.h
-
+1 -1code/espurna/gpio.cpp
-
+0 -11code/espurna/relay.cpp
-
+49 -135code/espurna/settings.cpp
-
+35 -7code/espurna/settings.h
-
+674 -0code/espurna/settings_embedis.h
-
+12 -6code/espurna/storage_eeprom.cpp
-
+13 -0code/espurna/storage_eeprom.h
-
+1 -1code/espurna/system.cpp
-
+1 -2code/espurna/telnet.cpp
-
+1 -1code/espurna/terminal.cpp
-
+2 -3code/espurna/utils.cpp
-
+2 -3code/espurna/web.cpp
-
+0 -1code/platformio.ini
-
+19 -0code/test/platformio.ini
-
+487 -0code/test/unit/settings/main.cpp
+ 1
- 1
code/espurna/board.cpp
View File
+ 1
- 0
code/espurna/config/all.h
View File
+ 0
- 11
code/espurna/config/general.h
View File
+ 162
- 81
code/espurna/crash.cpp
View File
+ 21
- 8
code/espurna/crash.h
View File
+ 1
- 1
code/espurna/debug.cpp
View File
+ 1
- 1
code/espurna/espurna.h
View File
+ 1
- 1
code/espurna/gpio.cpp
View File
+ 0
- 11
code/espurna/relay.cpp
View File
+ 49
- 135
code/espurna/settings.cpp
View File
+ 35
- 7
code/espurna/settings.h
View File
+ 674
- 0
code/espurna/settings_embedis.h
View File
| @ -0,0 +1,674 @@ | |||||
| /* | |||||
| Part of the SETTINGS MODULE | |||||
| Copyright (C) 2020 by Maxim Prokhorov <prokhorov dot max at outlook dot com> | |||||
| Reimplementation of the Embedis storage format: | |||||
| - https://github.com/thingSoC/embedis | |||||
| */ | |||||
| #pragma once | |||||
| #include <Arduino.h> | |||||
| #include <algorithm> | |||||
| #include <memory> | |||||
| #include <vector> | |||||
| #include "libs/TypeChecks.h" | |||||
| namespace settings { | |||||
| namespace embedis { | |||||
| // 'optional' type for byte range | |||||
| struct ValueResult { | |||||
| operator bool() { | |||||
| return result; | |||||
| } | |||||
| bool result { false }; | |||||
| String value; | |||||
| }; | |||||
| // We can't save empty keys but can save empty values as 0x00 0x00 0x00 0x00 | |||||
| // total sum is: | |||||
| // - 2 bytes gap at the end (will be re-used by the next value length byte) | |||||
| // - 4 bytes to store length of 2 values (stored as big-endian) | |||||
| // - N bytes of values themselves | |||||
| inline size_t estimate(const String& key, const String& value) { | |||||
| if (!key.length()) { | |||||
| return 0; | |||||
| } | |||||
| const auto key_len = key.length(); | |||||
| const auto value_len = value.length(); | |||||
| return (4 + key_len + ((value_len > 0) ? value_len : 2)); | |||||
| } | |||||
| // Note: KeyValueStore is templated to avoid having to provide RawStorageBase via virtual inheritance. | |||||
| template <typename RawStorageBase> | |||||
| class KeyValueStore { | |||||
| // ----------------------------------------------------------------------------------- | |||||
| // Notice: we can only use sfinae checks with the current compiler version | |||||
| // TODO: provide actual benchmark comparison with 'lambda'-list-as-vtable (old Embedis style) | |||||
| // and vtable approach (write(), read() and commit() as pure virtual) | |||||
| // TODO: consider overrides for bulk operations like move (see ::del method) | |||||
| template <typename T> | |||||
| using storage_can_write_t = decltype(std::declval<T>().write( | |||||
| std::declval<uint16_t>(), std::declval<uint8_t>())); | |||||
| template <typename T> | |||||
| using storage_can_write = is_detected<storage_can_write_t, T>; | |||||
| template <typename T> | |||||
| using storage_can_read_t = decltype(std::declval<T>().read(std::declval<uint16_t>())); | |||||
| template <typename T> | |||||
| using storage_can_read = is_detected<storage_can_read_t, T>; | |||||
| template <typename T> | |||||
| using storage_can_commit_t = decltype(std::declval<T>().commit()); | |||||
| template <typename T> | |||||
| using storage_can_commit = is_detected<storage_can_commit_t, T>; | |||||
| static_assert( | |||||
| (storage_can_commit<RawStorageBase>{} && | |||||
| storage_can_read<RawStorageBase>{} && | |||||
| storage_can_write<RawStorageBase>{}), | |||||
| "Storage class must implement read(index), write(index, byte) and commit()" | |||||
| ); | |||||
| // ----------------------------------------------------------------------------------- | |||||
| protected: | |||||
| // Tracking state of the parser inside of _raw_read() | |||||
| enum class State { | |||||
| Begin, | |||||
| End, | |||||
| LenByte1, | |||||
| LenByte2, | |||||
| EmptyValue, | |||||
| Value | |||||
| }; | |||||
| // Pointer to the region of data that we are using | |||||
| // | |||||
| // XXX: It does not matter right now, but we **will** overflow position when using sizes >= (2^16) - 1 | |||||
| // Note: Implementation is also in the header b/c c++ won't allow us | |||||
| // to have a plain member (not a ptr or ref) of unknown size. | |||||
| // Note: There was a considiration to implement this as 'stashing iterator' to be compatible with stl algorithms. | |||||
| // In such implementation, we would store intermediate index and allow the user to receive a `value_proxy`, | |||||
| // temporary returned by `value_proxy& operator*()' that is bound to Cursor instance. | |||||
| // This **will** cause problems with 'reverse_iterator' or anything like it, as it expects reference to | |||||
| // outlive the iterator object (specifically, result of `return *--tmp`, where `tmp` is created inside of a function block) | |||||
| struct Cursor { | |||||
| Cursor(RawStorageBase& storage, uint16_t position_, uint16_t begin_, uint16_t end_) : | |||||
| position(position_), | |||||
| begin(begin_), | |||||
| end(end_), | |||||
| _storage(storage) | |||||
| {} | |||||
| Cursor(RawStorageBase& storage, uint16_t begin_, uint16_t end_) : | |||||
| Cursor(storage, 0, begin_, end_) | |||||
| {} | |||||
| explicit Cursor(RawStorageBase& storage) : | |||||
| Cursor(storage, 0, 0, 0) | |||||
| {} | |||||
| static Cursor merge(RawStorageBase& storage, const Cursor& key, const Cursor& value) { | |||||
| return Cursor(storage, key.begin, value.end); | |||||
| } | |||||
| static Cursor fromEnd(RawStorageBase& storage, uint16_t begin, uint16_t end) { | |||||
| return Cursor(storage, end - begin, begin, end); | |||||
| } | |||||
| Cursor() = delete; | |||||
| void reset(uint16_t begin_, uint16_t end_) { | |||||
| position = 0; | |||||
| begin = begin_; | |||||
| end = end_; | |||||
| } | |||||
| uint8_t read() { | |||||
| return _storage.read(begin + position); | |||||
| } | |||||
| void write(uint8_t value) { | |||||
| _storage.write(begin + position, value); | |||||
| } | |||||
| void resetBeginning() { | |||||
| position = 0; | |||||
| } | |||||
| void resetEnd() { | |||||
| position = end - begin; | |||||
| } | |||||
| size_t size() { | |||||
| return (end - begin); | |||||
| } | |||||
| bool inRange(uint16_t position_) { | |||||
| return (position_ < (end - begin)); | |||||
| } | |||||
| operator bool() { | |||||
| return inRange(position); | |||||
| } | |||||
| uint8_t operator[](size_t position_) const { | |||||
| return _storage.read(begin + position_); | |||||
| } | |||||
| bool operator ==(const Cursor& other) { | |||||
| return (begin == other.begin) && (end == other.end); | |||||
| } | |||||
| bool operator !=(const Cursor& other) { | |||||
| return !(*this == other); | |||||
| } | |||||
| Cursor& operator++() { | |||||
| ++position; | |||||
| return *this; | |||||
| } | |||||
| Cursor operator++(int) { | |||||
| Cursor other(*this); | |||||
| ++*this; | |||||
| return other; | |||||
| } | |||||
| Cursor& operator--() { | |||||
| --position; | |||||
| return *this; | |||||
| } | |||||
| Cursor operator--(int) { | |||||
| Cursor other(*this); | |||||
| --*this; | |||||
| return other; | |||||
| } | |||||
| uint16_t position; | |||||
| uint16_t begin; | |||||
| uint16_t end; | |||||
| private: | |||||
| RawStorageBase& _storage; | |||||
| }; | |||||
| public: | |||||
| // Store value location in a more reasonable forward-iterator-style manner | |||||
| // Allows us to skip string creation when just searching for specific values | |||||
| // XXX: be cautious that cursor positions **will** break when underlying storage changes | |||||
| struct ReadResult { | |||||
| friend class KeyValueStore<RawStorageBase>; | |||||
| ReadResult(const Cursor& cursor_) : | |||||
| length(0), | |||||
| cursor(cursor_), | |||||
| result(false) | |||||
| {} | |||||
| ReadResult(RawStorageBase& storage) : | |||||
| length(0), | |||||
| cursor(storage), | |||||
| result(false) | |||||
| {} | |||||
| operator bool() { | |||||
| return result; | |||||
| } | |||||
| String read() { | |||||
| String out; | |||||
| out.reserve(length); | |||||
| if (!length) { | |||||
| return out; | |||||
| } | |||||
| decltype(length) index = 0; | |||||
| cursor.resetBeginning(); | |||||
| while (index < length) { | |||||
| out += static_cast<char>(cursor.read()); | |||||
| ++cursor; | |||||
| ++index; | |||||
| } | |||||
| return out; | |||||
| } | |||||
| uint16_t length; | |||||
| private: | |||||
| Cursor cursor; | |||||
| bool result; | |||||
| }; | |||||
| // Internal storage consists of sequences of <byte-range><length> | |||||
| struct KeyValueResult { | |||||
| operator bool() { | |||||
| return (key) && (value) && (key.length > 0); | |||||
| } | |||||
| bool operator !() { | |||||
| return !(static_cast<bool>(*this)); | |||||
| } | |||||
| template <typename T = ReadResult> | |||||
| KeyValueResult(T&& key_, T&& value_) : | |||||
| key(std::forward<T>(key_)), | |||||
| value(std::forward<T>(value_)) | |||||
| {} | |||||
| KeyValueResult(RawStorageBase& storage) : | |||||
| key(storage), | |||||
| value(storage) | |||||
| {} | |||||
| ReadResult key; | |||||
| ReadResult value; | |||||
| }; | |||||
| // one and only possible constructor, simply move the class object into the | |||||
| // member variable to avoid forcing the user of the API to keep 2 objects alive. | |||||
| KeyValueStore(RawStorageBase&& storage, uint16_t begin, uint16_t end) : | |||||
| _storage(std::move(storage)), | |||||
| _cursor(_storage, begin, end), | |||||
| _state(State::Begin) | |||||
| {} | |||||
| // Try to find the matching key. Datastructure that we use does not specify | |||||
| // any value 'type' inside of it. We expect 'key' to be the first non-empty string, | |||||
| // 'value' can be empty. | |||||
| ValueResult get(const String& key) { | |||||
| return _get(key, true); | |||||
| } | |||||
| bool has(const String& key) { | |||||
| return static_cast<bool>(_get(key, false)); | |||||
| } | |||||
| // We going be using this pattern all the time here, because we need 2 consecutive **valid** ranges | |||||
| // TODO: expose _read_kv() and _cursor_reset_end() so we can have 'break' here? | |||||
| // perhaps as a wrapper object, allow something like next() and seekBegin() | |||||
| template <typename CallbackType> | |||||
| void foreach(CallbackType callback) { | |||||
| _cursor_reset_end(); | |||||
| do { | |||||
| auto kv = _read_kv(); | |||||
| if (!kv) { | |||||
| break; | |||||
| } | |||||
| callback(std::move(kv)); | |||||
| } while (_state != State::End); | |||||
| } | |||||
| // read every key into a vector | |||||
| std::vector<String> keys() { | |||||
| std::vector<String> out; | |||||
| out.reserve(count()); | |||||
| foreach([&](KeyValueResult&& kv) { | |||||
| out.push_back(kv.key.read()); | |||||
| }); | |||||
| return out; | |||||
| } | |||||
| // set or update key with value contents. ensure 'key' isn't empty, 'value' can be empty | |||||
| bool set(const String& key, const String& value) { | |||||
| // ref. 'estimate()' implementation in regards to the storage calculation | |||||
| auto need = estimate(key, value); | |||||
| if (!need) { | |||||
| return false; | |||||
| } | |||||
| auto key_len = key.length(); | |||||
| auto value_len = value.length(); | |||||
| Cursor to_erase(_storage); | |||||
| bool need_erase = false; | |||||
| auto start_pos = _cursor_reset_end(); | |||||
| do { | |||||
| auto kv = _read_kv(); | |||||
| if (!kv) { | |||||
| break; | |||||
| } | |||||
| start_pos = kv.value.cursor.begin; | |||||
| // in the very special case we can match the existing key | |||||
| if ((kv.key.length == key_len) && (kv.key.read() == key)) { | |||||
| if (kv.value.length == value.length()) { | |||||
| if (kv.value.read() == value) { | |||||
| return true; | |||||
| } | |||||
| start_pos = kv.key.cursor.end; | |||||
| break; | |||||
| } | |||||
| // but we may need to write over it, when contents are different | |||||
| to_erase.reset(kv.value.cursor.begin, kv.key.cursor.end); | |||||
| need_erase = true; | |||||
| } | |||||
| } while (_state != State::End); | |||||
| if (need_erase) { | |||||
| _raw_erase(start_pos, to_erase); | |||||
| start_pos += to_erase.size(); | |||||
| } | |||||
| // we should only insert when possition is still within possible size | |||||
| if (start_pos && (start_pos >= need)) { | |||||
| auto writer = Cursor::fromEnd(_storage, start_pos - need, start_pos); | |||||
| // put the length of the value as 2 bytes and then write the data | |||||
| (--writer).write(key_len & 0xff); | |||||
| (--writer).write((key_len >> 8) & 0xff); | |||||
| while (key_len--) { | |||||
| (--writer).write(key[key_len]); | |||||
| } | |||||
| (--writer).write(value_len & 0xff); | |||||
| (--writer).write((value_len >> 8) & 0xff); | |||||
| if (value_len) { | |||||
| while (value_len--) { | |||||
| (--writer).write(value[value_len]); | |||||
| } | |||||
| } else { | |||||
| (--writer).write(0); | |||||
| (--writer).write(0); | |||||
| } | |||||
| // we also need to pad the space *after* the value | |||||
| // but, only when we still have some space left | |||||
| if ((start_pos - need) >= 2) { | |||||
| _cursor_set_position(writer.begin - _cursor.begin); | |||||
| auto next_kv = _read_kv(); | |||||
| if (!next_kv) { | |||||
| auto padding = Cursor::fromEnd(_storage, writer.begin - 2, writer.begin); | |||||
| (--padding).write(0); | |||||
| (--padding).write(0); | |||||
| } | |||||
| } | |||||
| _storage.commit(); | |||||
| return true; | |||||
| } | |||||
| return false; | |||||
| } | |||||
| // remove key from the storage. will check that 'key' argument isn't empty | |||||
| bool del(const String& key) { | |||||
| size_t key_len = key.length(); | |||||
| if (!key_len) { | |||||
| return false; | |||||
| } | |||||
| // Removes key from the storage by overwriting the key with left-most data | |||||
| size_t start_pos = _cursor_reset_end() - 1; | |||||
| auto to_erase = Cursor::fromEnd(_storage, _cursor.begin, _cursor.end); | |||||
| // we should only compare strings of equal length. | |||||
| // when matching, record { value ... key } range + 4 bytes for length | |||||
| // continue searching for the leftmost boundary | |||||
| foreach([&](KeyValueResult&& kv) { | |||||
| start_pos = kv.value.cursor.begin; | |||||
| if (!to_erase && (kv.key.length == key_len) && (kv.key.read() == key)) { | |||||
| to_erase.reset(kv.value.cursor.begin, kv.key.cursor.end); | |||||
| } | |||||
| }); | |||||
| if (!to_erase) { | |||||
| return false; | |||||
| } | |||||
| _raw_erase(start_pos, to_erase); | |||||
| return true; | |||||
| } | |||||
| // Simply count key-value pairs that we could parse | |||||
| size_t count() { | |||||
| size_t result = 0; | |||||
| foreach([&result](KeyValueResult&&) { | |||||
| ++result; | |||||
| }); | |||||
| return result; | |||||
| } | |||||
| // Do exactly the same thing as 'keys' does, but return the amount | |||||
| // of bytes to the left of the last kv | |||||
| size_t available() { | |||||
| size_t result = _cursor.size(); | |||||
| foreach([&result](KeyValueResult&& kv) { | |||||
| result -= kv.key.cursor.size(); | |||||
| result -= kv.value.cursor.size(); | |||||
| }); | |||||
| return result; | |||||
| } | |||||
| // How much bytes can be used is directly read from the cursor, based on begin and end values | |||||
| size_t size() { | |||||
| return _cursor.size(); | |||||
| } | |||||
| protected: | |||||
| // Try to find the matching key. Datastructure that we use does not specify | |||||
| // any value 'type' inside of it. We expect 'key' to be the first non-empty string, | |||||
| // 'value' can be empty. | |||||
| // To implement has(), allow to skip reading the value | |||||
| ValueResult _get(const String& key, bool read_value) { | |||||
| ValueResult out; | |||||
| auto len = key.length(); | |||||
| _cursor_reset_end(); | |||||
| do { | |||||
| auto kv = _read_kv(); | |||||
| if (!kv) { | |||||
| break; | |||||
| } | |||||
| // no point in comparing keys when length does not match | |||||
| // (and we also don't want to allocate the string) | |||||
| if (kv.key.length != len) { | |||||
| continue; | |||||
| } | |||||
| auto key_result = kv.key.read(); | |||||
| if (key_result == key) { | |||||
| if (read_value) { | |||||
| out.value = kv.value.read(); | |||||
| } | |||||
| out.result = true; | |||||
| break; | |||||
| } | |||||
| } while (_state != State::End); | |||||
| return out; | |||||
| } | |||||
| // Place cursor at the `end` and resets the parser to expect length byte | |||||
| uint16_t _cursor_reset_end() { | |||||
| _cursor.resetEnd(); | |||||
| _state = State::Begin; | |||||
| return _cursor.end; | |||||
| } | |||||
| uint16_t _cursor_set_position(uint16_t position) { | |||||
| _state = State::Begin; | |||||
| _cursor.position = position; | |||||
| return _cursor.position; | |||||
| } | |||||
| // implementation quirk is that `Cursor::operator=` won't work because of the `RawStorageBase&` member | |||||
| // right now, just construct in place and assume that compiler will inline things | |||||
| // on one hand, we can implement it. but, we can't specifically | |||||
| KeyValueResult _read_kv() { | |||||
| auto key = _raw_read(); | |||||
| if (!key || !key.length) { | |||||
| return {_storage}; | |||||
| } | |||||
| auto value = _raw_read(); | |||||
| return {key, value}; | |||||
| }; | |||||
| void _raw_erase(size_t start_pos, Cursor& to_erase) { | |||||
| // we either end up to the left or to the right of the boundary | |||||
| if (start_pos < to_erase.begin) { | |||||
| auto from = Cursor::fromEnd(_storage, start_pos, to_erase.begin); | |||||
| auto to = Cursor::fromEnd(_storage, start_pos + to_erase.size(), to_erase.end); | |||||
| while (--from && --to) { | |||||
| to.write(from.read()); | |||||
| from.write(0xff); | |||||
| }; | |||||
| } else { | |||||
| // just null the length bytes, since we at the last key | |||||
| to_erase.resetEnd(); | |||||
| (--to_erase).write(0); | |||||
| (--to_erase).write(0); | |||||
| } | |||||
| _storage.commit(); | |||||
| } | |||||
| // Returns Cursor to the region that holds the data | |||||
| // Result object does not hold any data, we need to explicitly request read() | |||||
| // | |||||
| // Cursor object is always expected to point to something, e.g. minimum: | |||||
| // 0x01 0x00 0x01 | |||||
| // data len2 len1 | |||||
| // Position will be 0, end will be 4. Total length is 3, data length is 1 | |||||
| // | |||||
| // Note the distinction between real length and data length. For example, | |||||
| // special-case for empty 'value' (as 'key' can never be empty and will be rejected): | |||||
| // 0x00 0x00 0x00 0x00 | |||||
| // data data len2 len1 | |||||
| // Position will be 0, end will be 5. Total length is 4, data length is 0 | |||||
| ReadResult _raw_read() { | |||||
| uint16_t len = 0; | |||||
| ReadResult out(_storage); | |||||
| do { | |||||
| // storage is written right-to-left, cursor is always decreasing | |||||
| switch (_state) { | |||||
| case State::Begin: | |||||
| if (_cursor.position > 2) { | |||||
| --_cursor; | |||||
| _state = State::LenByte1; | |||||
| } else { | |||||
| _state = State::End; | |||||
| } | |||||
| continue; | |||||
| // len is 16 bit uint (bigendian) | |||||
| // special case is 0, which is valid and should be returned when encountered | |||||
| // another special case is 0xffff, meaning we just hit an empty space | |||||
| case State::LenByte1: | |||||
| len = _cursor.read(); | |||||
| --_cursor; | |||||
| _state = State::LenByte2; | |||||
| break; | |||||
| case State::LenByte2: | |||||
| { | |||||
| uint8_t len2 = _cursor.read(); | |||||
| --_cursor; | |||||
| if ((0 == len) && (0 == len2)) { | |||||
| len = 2; | |||||
| _state = State::EmptyValue; | |||||
| } else if ((0xff == len) && (0xff == len2)) { | |||||
| _state = State::End; | |||||
| } else { | |||||
| len |= len2 << 8; | |||||
| _state = State::Value; | |||||
| } | |||||
| break; | |||||
| } | |||||
| case State::EmptyValue: | |||||
| case State::Value: { | |||||
| uint16_t left = len; | |||||
| // ensure we don't go out-of-bounds | |||||
| switch (_state) { | |||||
| case State::Value: | |||||
| while (_cursor && --left) { | |||||
| --_cursor; | |||||
| } | |||||
| break; | |||||
| // ...and only read 0's | |||||
| case State::EmptyValue: | |||||
| while (_cursor && (_cursor.read() == 0) && --left) { | |||||
| --_cursor; | |||||
| } | |||||
| break; | |||||
| default: | |||||
| break; | |||||
| } | |||||
| if (left) { | |||||
| _state = State::End; | |||||
| break; | |||||
| } | |||||
| // set the resulting cursor as [pos:len+2) | |||||
| out.result = true; | |||||
| out.length = (_state == State::EmptyValue) ? 0 : len; | |||||
| out.cursor.reset( | |||||
| _cursor.begin + _cursor.position, | |||||
| _cursor.begin + _cursor.position + len + 2 | |||||
| ); | |||||
| _state = State::Begin; | |||||
| goto return_result; | |||||
| } | |||||
| case State::End: | |||||
| default: | |||||
| break; | |||||
| } | |||||
| } while (_state != State::End); | |||||
| return_result: | |||||
| return out; | |||||
| } | |||||
| RawStorageBase _storage; | |||||
| Cursor _cursor; | |||||
| State _state { State::Begin }; | |||||
| }; | |||||
| } // namespace embedis | |||||
| } // namespace settings | |||||
+ 12
- 6
code/espurna/storage_eeprom.cpp
View File
+ 13
- 0
code/espurna/storage_eeprom.h
View File
+ 1
- 1
code/espurna/system.cpp
View File
+ 1
- 2
code/espurna/telnet.cpp
View File
+ 1
- 1
code/espurna/terminal.cpp
View File
+ 2
- 3
code/espurna/utils.cpp
View File
+ 2
- 3
code/espurna/web.cpp
View File
+ 0
- 1
code/platformio.ini
View File
+ 19
- 0
code/test/platformio.ini
View File
+ 487
- 0
code/test/unit/settings/main.cpp
View File
| @ -0,0 +1,487 @@ | |||||
| #include <unity.h> | |||||
| #include <Arduino.h> | |||||
| #pragma GCC diagnostic warning "-Wall" | |||||
| #pragma GCC diagnostic warning "-Wextra" | |||||
| #pragma GCC diagnostic warning "-Wstrict-aliasing" | |||||
| #pragma GCC diagnostic warning "-Wpointer-arith" | |||||
| #pragma GCC diagnostic warning "-Wstrict-overflow=5" | |||||
| #include <settings_embedis.h> | |||||
| #include <array> | |||||
| #include <algorithm> | |||||
| #include <numeric> | |||||
| namespace settings { | |||||
| namespace embedis { | |||||
| template <typename T> | |||||
| struct StaticArrayStorage { | |||||
| explicit StaticArrayStorage(T& blob) : | |||||
| _blob(blob), | |||||
| _size(blob.size()) | |||||
| {} | |||||
| uint8_t read(size_t index) { | |||||
| TEST_ASSERT_LESS_THAN(_size, index); | |||||
| return _blob[index]; | |||||
| } | |||||
| void write(size_t index, uint8_t value) { | |||||
| TEST_ASSERT_LESS_THAN(_size, index); | |||||
| _blob[index] = value; | |||||
| } | |||||
| void commit() { | |||||
| } | |||||
| T& _blob; | |||||
| const size_t _size; | |||||
| }; | |||||
| } // namespace embedis | |||||
| } // namespace settings | |||||
| template <size_t Size> | |||||
| struct StorageHandler { | |||||
| using array_type = std::array<uint8_t, Size>; | |||||
| using storage_type = settings::embedis::StaticArrayStorage<array_type>; | |||||
| using kvs_type = settings::embedis::KeyValueStore<storage_type>; | |||||
| StorageHandler() : | |||||
| kvs(std::move(storage_type{blob}), 0, Size) | |||||
| { | |||||
| blob.fill(0xff); | |||||
| } | |||||
| array_type blob; | |||||
| kvs_type kvs; | |||||
| }; | |||||
| // generate stuff depending on the mode | |||||
| // - Indexed: key1:val1, key2:val2, ... | |||||
| // - IncreasingLength: k:v, kk:vv, ... | |||||
| struct TestSequentialKvGenerator { | |||||
| using kv = std::pair<String, String>; | |||||
| enum class Mode { | |||||
| Indexed, | |||||
| IncreasingLength | |||||
| }; | |||||
| TestSequentialKvGenerator() = default; | |||||
| explicit TestSequentialKvGenerator(Mode mode) : | |||||
| _mode(mode) | |||||
| {} | |||||
| const kv& next() { | |||||
| auto index = _index++; | |||||
| _current.first = ""; | |||||
| _current.second = ""; | |||||
| switch (_mode) { | |||||
| case Mode::Indexed: | |||||
| _current.first = String("key") + String(index); | |||||
| _current.second = String("val") + String(index); | |||||
| break; | |||||
| case Mode::IncreasingLength: { | |||||
| size_t sizes = _index; | |||||
| _current.first.reserve(sizes); | |||||
| _current.second.reserve(sizes); | |||||
| do { | |||||
| _current.first += "k"; | |||||
| _current.second += "v"; | |||||
| } while (--sizes); | |||||
| break; | |||||
| } | |||||
| } | |||||
| TEST_ASSERT(_last.first != _current.first); | |||||
| TEST_ASSERT(_last.second != _current.second); | |||||
| return (_last = _current); | |||||
| } | |||||
| std::vector<kv> make(size_t size) {; | |||||
| std::vector<kv> res; | |||||
| for (size_t index = 0; index < size; ++index) { | |||||
| res.push_back(next()); | |||||
| } | |||||
| return res; | |||||
| } | |||||
| kv _current; | |||||
| kv _last; | |||||
| Mode _mode { Mode::Indexed }; | |||||
| size_t _index { 0 }; | |||||
| }; | |||||
| // ---------------------------------------------------------------------------- | |||||
| using TestStorageHandler = StorageHandler<1024>; | |||||
| template <typename T> | |||||
| void check_kv(T& instance, const String& key, const String& value) { | |||||
| auto result = instance.kvs.get(key); | |||||
| TEST_ASSERT_MESSAGE(static_cast<bool>(result), key.c_str()); | |||||
| TEST_ASSERT(result.value.length()); | |||||
| TEST_ASSERT_EQUAL_STRING(value.c_str(), result.value.c_str()); | |||||
| }; | |||||
| void test_sizes() { | |||||
| // empty storage is still manageble, it just does not work :) | |||||
| { | |||||
| StorageHandler<0> empty; | |||||
| TEST_ASSERT_EQUAL(0, empty.kvs.count()); | |||||
| TEST_ASSERT_FALSE(empty.kvs.set("cannot", "happen")); | |||||
| TEST_ASSERT_FALSE(static_cast<bool>(empty.kvs.get("cannot"))); | |||||
| } | |||||
| // some hard-coded estimates to notify us about internal changes | |||||
| { | |||||
| StorageHandler<16> instance; | |||||
| TEST_ASSERT_EQUAL(0, instance.kvs.count()); | |||||
| TEST_ASSERT_EQUAL(16, instance.kvs.available()); | |||||
| TEST_ASSERT_EQUAL(0, settings::embedis::estimate("", "123456")); | |||||
| TEST_ASSERT_EQUAL(16, settings::embedis::estimate("123456", "123456")); | |||||
| TEST_ASSERT_EQUAL(10, settings::embedis::estimate("123", "123")); | |||||
| TEST_ASSERT_EQUAL(9, settings::embedis::estimate("345", "")); | |||||
| } | |||||
| } | |||||
| void test_longkey() { | |||||
| TestStorageHandler instance; | |||||
| const auto estimate = instance.kvs.size() - 6; | |||||
| String key; | |||||
| key.reserve(estimate); | |||||
| for (size_t n = 0; n < estimate; ++n) { | |||||
| key += 'a'; | |||||
| } | |||||
| TEST_ASSERT(instance.kvs.set(key, "")); | |||||
| auto result = instance.kvs.get(key); | |||||
| TEST_ASSERT(static_cast<bool>(result)); | |||||
| } | |||||
| void test_perseverance() { | |||||
| // ensure we can handle setting the same key | |||||
| using storage_type = StorageHandler<128>; | |||||
| using blob_type = decltype(std::declval<storage_type>().blob); | |||||
| // xxx: implementation detail? | |||||
| // can we avoid blob modification when value is the same as the existing one | |||||
| { | |||||
| storage_type instance; | |||||
| blob_type original(instance.blob); | |||||
| TEST_ASSERT(instance.kvs.set("key", "value")); | |||||
| TEST_ASSERT(instance.kvs.set("another", "keyvalue")); | |||||
| TEST_ASSERT(original != instance.blob); | |||||
| blob_type snapshot(instance.blob); | |||||
| TEST_ASSERT(instance.kvs.set("key", "value")); | |||||
| TEST_ASSERT(snapshot == instance.blob); | |||||
| } | |||||
| // xxx: pointless implementation detail? | |||||
| // can we re-use existing 'value' storage and avoid data-shift | |||||
| { | |||||
| storage_type instance; | |||||
| blob_type original(instance.blob); | |||||
| // insert in a specific order, change middle | |||||
| TEST_ASSERT(instance.kvs.set("aaa", "bbb")); | |||||
| TEST_ASSERT(instance.kvs.set("cccc", "dd")); | |||||
| TEST_ASSERT(instance.kvs.set("ee", "fffff")); | |||||
| TEST_ASSERT(instance.kvs.set("cccc", "ff")); | |||||
| TEST_ASSERT(original != instance.blob); | |||||
| blob_type before(instance.blob); | |||||
| // purge, insert again with updated values | |||||
| TEST_ASSERT(instance.kvs.del("aaa")); | |||||
| TEST_ASSERT(instance.kvs.del("cccc")); | |||||
| TEST_ASSERT(instance.kvs.del("ee")); | |||||
| TEST_ASSERT(instance.kvs.set("aaa", "bbb")); | |||||
| TEST_ASSERT(instance.kvs.set("cccc", "ff")); | |||||
| TEST_ASSERT(instance.kvs.set("ee", "fffff")); | |||||
| blob_type after(instance.blob); | |||||
| TEST_ASSERT(original != before); | |||||
| TEST_ASSERT(original != after); | |||||
| TEST_ASSERT(before == after); | |||||
| } | |||||
| } | |||||
| template <size_t Size> | |||||
| struct test_overflow_runner { | |||||
| void operator ()() const { | |||||
| StorageHandler<Size> instance; | |||||
| TEST_ASSERT(instance.kvs.set("a", "b")); | |||||
| TEST_ASSERT(instance.kvs.set("c", "d")); | |||||
| TEST_ASSERT_EQUAL(2, instance.kvs.count()); | |||||
| TEST_ASSERT_FALSE(instance.kvs.set("e", "f")); | |||||
| TEST_ASSERT(instance.kvs.del("a")); | |||||
| TEST_ASSERT_EQUAL(1, instance.kvs.count()); | |||||
| TEST_ASSERT(instance.kvs.set("e", "f")); | |||||
| TEST_ASSERT_EQUAL(2, instance.kvs.count()); | |||||
| check_kv(instance, "e", "f"); | |||||
| check_kv(instance, "c", "d"); | |||||
| } | |||||
| }; | |||||
| void test_overflow() { | |||||
| // slightly more that available, but we cannot fit the key | |||||
| test_overflow_runner<16>(); | |||||
| // no more space | |||||
| test_overflow_runner<12>(); | |||||
| } | |||||
| void test_small_gaps() { | |||||
| // ensure we can intemix empty and non-empty values | |||||
| TestStorageHandler instance; | |||||
| TEST_ASSERT(instance.kvs.set("key", "value")); | |||||
| TEST_ASSERT(instance.kvs.set("empty", "")); | |||||
| TEST_ASSERT(instance.kvs.set("empty_again", "")); | |||||
| TEST_ASSERT(instance.kvs.set("finally", "avalue")); | |||||
| auto check_empty = [&instance](const String& key) { | |||||
| auto result = instance.kvs.get(key); | |||||
| TEST_ASSERT(static_cast<bool>(result)); | |||||
| TEST_ASSERT_FALSE(result.value.length()); | |||||
| }; | |||||
| check_empty("empty_again"); | |||||
| check_empty("empty"); | |||||
| check_empty("empty_again"); | |||||
| check_empty("empty"); | |||||
| auto check_value = [&instance](const String& key, const String& value) { | |||||
| auto result = instance.kvs.get(key); | |||||
| TEST_ASSERT(static_cast<bool>(result)); | |||||
| TEST_ASSERT(result.value.length()); | |||||
| TEST_ASSERT_EQUAL_STRING(value.c_str(), result.value.c_str()); | |||||
| }; | |||||
| check_value("finally", "avalue"); | |||||
| check_value("key", "value"); | |||||
| } | |||||
| void test_remove_randomized() { | |||||
| // ensure we can remove keys in any order | |||||
| // 8 seems like a good number to stop on, 9 will spend ~10seconds | |||||
| // TODO: seems like a good start benchmarking read / write performance? | |||||
| constexpr size_t KeysNumber = 8; | |||||
| TestSequentialKvGenerator generator(TestSequentialKvGenerator::Mode::IncreasingLength); | |||||
| auto kvs = generator.make(KeysNumber); | |||||
| // generate indexes array to allow us to reference keys at random | |||||
| TestStorageHandler instance; | |||||
| std::array<size_t, KeysNumber> indexes; | |||||
| std::iota(indexes.begin(), indexes.end(), 0); | |||||
| // - insert keys sequentially | |||||
| // - remove keys based on the order provided by next_permutation() | |||||
| size_t index = 0; | |||||
| do { | |||||
| TEST_ASSERT(0 == instance.kvs.count()); | |||||
| for (auto& kv : kvs) { | |||||
| TEST_ASSERT(instance.kvs.set(kv.first, kv.second)); | |||||
| } | |||||
| for (auto index : indexes) { | |||||
| auto key = kvs[index].first; | |||||
| TEST_ASSERT(static_cast<bool>(instance.kvs.get(key))); | |||||
| TEST_ASSERT(instance.kvs.del(key)); | |||||
| TEST_ASSERT_FALSE(static_cast<bool>(instance.kvs.get(key))); | |||||
| } | |||||
| index++; | |||||
| } while (std::next_permutation(indexes.begin(), indexes.end())); | |||||
| String message("- keys: "); | |||||
| message += KeysNumber; | |||||
| message += ", permutations: "; | |||||
| message += index; | |||||
| TEST_MESSAGE(message.c_str()); | |||||
| } | |||||
| void test_basic() { | |||||
| TestStorageHandler instance; | |||||
| constexpr size_t KeysNumber = 5; | |||||
| // ensure insert works | |||||
| TestSequentialKvGenerator generator; | |||||
| auto kvs = generator.make(KeysNumber); | |||||
| for (auto& kv : kvs) { | |||||
| instance.kvs.set(kv.first, kv.second); | |||||
| } | |||||
| // and we can retrieve keys back | |||||
| for (auto& kv : kvs) { | |||||
| auto result = instance.kvs.get(kv.first); | |||||
| TEST_ASSERT(static_cast<bool>(result)); | |||||
| TEST_ASSERT_EQUAL_STRING(kv.second.c_str(), result.value.c_str()); | |||||
| } | |||||
| } | |||||
| void test_storage() { | |||||
| constexpr size_t Size = 32; | |||||
| StorageHandler<Size> instance; | |||||
| // empty keys are invalid | |||||
| TEST_ASSERT_FALSE(instance.kvs.set("", "value1")); | |||||
| TEST_ASSERT_FALSE(instance.kvs.del("")); | |||||
| // ...and both keys are not yet set | |||||
| TEST_ASSERT_FALSE(instance.kvs.del("key1")); | |||||
| TEST_ASSERT_FALSE(instance.kvs.del("key2")); | |||||
| // some different ways to set keys | |||||
| TEST_ASSERT(instance.kvs.set("key1", "value0")); | |||||
| TEST_ASSERT_EQUAL(1, instance.kvs.count()); | |||||
| TEST_ASSERT(instance.kvs.set("key1", "value1")); | |||||
| TEST_ASSERT_EQUAL(1, instance.kvs.count()); | |||||
| TEST_ASSERT(instance.kvs.set("key2", "value_old")); | |||||
| TEST_ASSERT_EQUAL(2, instance.kvs.count()); | |||||
| TEST_ASSERT(instance.kvs.set("key2", "value2")); | |||||
| TEST_ASSERT_EQUAL(2, instance.kvs.count()); | |||||
| auto kvsize = settings::embedis::estimate("key1", "value1"); | |||||
| TEST_ASSERT_EQUAL((Size - (2 * kvsize)), instance.kvs.available()); | |||||
| // checking keys one by one by using a separate kvs object, | |||||
| // working on the same underlying data-store | |||||
| using storage_type = decltype(instance)::storage_type; | |||||
| using kvs_type = decltype(instance)::kvs_type; | |||||
| // - ensure we can operate with storage offsets | |||||
| // - test for internal length optimization that will overwrite the key in-place | |||||
| // - make sure we did not break the storage above | |||||
| // storage_type accepts reference to the blob, so we can seamlessly use the same | |||||
| // underlying data storage and share it between kvs instances | |||||
| { | |||||
| kvs_type slice(storage_type(instance.blob), (Size - kvsize), Size); | |||||
| TEST_ASSERT_EQUAL(1, slice.count()); | |||||
| TEST_ASSERT_EQUAL(kvsize, slice.size()); | |||||
| TEST_ASSERT_EQUAL(0, slice.available()); | |||||
| auto result = slice.get("key1"); | |||||
| TEST_ASSERT(static_cast<bool>(result)); | |||||
| TEST_ASSERT_EQUAL_STRING("value1", result.value.c_str()); | |||||
| } | |||||
| // ensure that right offset also works | |||||
| { | |||||
| kvs_type slice(storage_type(instance.blob), 0, (Size - kvsize)); | |||||
| TEST_ASSERT_EQUAL(1, slice.count()); | |||||
| TEST_ASSERT_EQUAL((Size - kvsize), slice.size()); | |||||
| TEST_ASSERT_EQUAL((Size - kvsize - kvsize), slice.available()); | |||||
| auto result = slice.get("key2"); | |||||
| TEST_ASSERT(static_cast<bool>(result)); | |||||
| TEST_ASSERT_EQUAL_STRING("value2", result.value.c_str()); | |||||
| } | |||||
| // ensure offset does not introduce offset bugs | |||||
| // for instance, test in-place key overwrite by moving left boundary 2 bytes to the right | |||||
| { | |||||
| const auto available = instance.kvs.available(); | |||||
| const auto offset = 2; | |||||
| TEST_ASSERT_GREATER_OR_EQUAL(offset, available); | |||||
| kvs_type slice(storage_type(instance.blob), offset, Size); | |||||
| TEST_ASSERT_EQUAL(2, slice.count()); | |||||
| auto key1 = slice.get("key1"); | |||||
| TEST_ASSERT(static_cast<bool>(key1)); | |||||
| String updated(key1.value); | |||||
| for (size_t index = 0; index < key1.value.length(); ++index) { | |||||
| updated[index] = 'A'; | |||||
| } | |||||
| TEST_ASSERT(slice.set("key1", updated)); | |||||
| TEST_ASSERT(slice.set("key2", updated)); | |||||
| TEST_ASSERT_EQUAL(2, slice.count()); | |||||
| auto check_key1 = slice.get("key1"); | |||||
| TEST_ASSERT(static_cast<bool>(check_key1)); | |||||
| TEST_ASSERT_EQUAL_STRING(updated.c_str(), check_key1.value.c_str()); | |||||
| auto check_key2 = slice.get("key2"); | |||||
| TEST_ASSERT(static_cast<bool>(check_key2)); | |||||
| TEST_ASSERT_EQUAL_STRING(updated.c_str(), check_key2.value.c_str()); | |||||
| TEST_ASSERT_EQUAL(available - offset, slice.available()); | |||||
| } | |||||
| } | |||||
| void test_keys_iterator() { | |||||
| constexpr size_t Size = 32; | |||||
| StorageHandler<Size> instance; | |||||
| TEST_ASSERT_EQUAL(Size, instance.kvs.available()); | |||||
| TEST_ASSERT_EQUAL(Size, instance.kvs.size()); | |||||
| TEST_ASSERT(instance.kvs.set("key", "value")); | |||||
| TEST_ASSERT(instance.kvs.set("another", "thing")); | |||||
| // ensure we get the same order of keys when iterating via foreach | |||||
| std::vector<String> keys; | |||||
| instance.kvs.foreach([&keys](decltype(instance)::kvs_type::KeyValueResult&& kv) { | |||||
| keys.push_back(kv.key.read()); | |||||
| }); | |||||
| TEST_ASSERT(instance.kvs.keys() == keys); | |||||
| TEST_ASSERT_EQUAL(2, keys.size()); | |||||
| TEST_ASSERT_EQUAL(2, instance.kvs.count()); | |||||
| TEST_ASSERT_EQUAL_STRING("key", keys[0].c_str()); | |||||
| TEST_ASSERT_EQUAL_STRING("another", keys[1].c_str()); | |||||
| } | |||||
| int main(int argc, char** argv) { | |||||
| UNITY_BEGIN(); | |||||
| RUN_TEST(test_storage); | |||||
| RUN_TEST(test_keys_iterator); | |||||
| RUN_TEST(test_basic); | |||||
| RUN_TEST(test_remove_randomized); | |||||
| RUN_TEST(test_small_gaps); | |||||
| RUN_TEST(test_overflow); | |||||
| RUN_TEST(test_perseverance); | |||||
| RUN_TEST(test_longkey); | |||||
| RUN_TEST(test_sizes); | |||||
| return UNITY_END(); | |||||
| } | |||||