diff --git a/docs/source/Controller/C016.rst b/docs/source/Controller/C016.rst index 83c2bd32a..c47eee29d 100644 --- a/docs/source/Controller/C016.rst +++ b/docs/source/Controller/C016.rst @@ -63,8 +63,29 @@ Data Delivery The controller can deliver the data to: +- JavaScript to process the data inside the browser. See the ``dump5.htm`` file in the ``misc`` folder. - Upload bin files to some server (HTTP post?) (TODO) - Provide a sample to any connected controller (TODO) - Do nothing and let some extern host pull the data from the node. (TODO) -- JavaScript to process the data inside the browser. (TODO) - Feed it to some plugin (e.g. a display to show a chart) (TODO) + + +Fetch and Decode data in the browser +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``dump5.htm`` (in ``misc`` directory) file should be uploaded to the file system of the ESP. +When there are cache controller bin files present on the file system, +open this htm file from the file browser in ESPEasy into a new tab in your browser. + +This presents a large button "Fetch cache files". +When pressed, the JavaScript in this htm file will fetch JSON information from +ESPEasy describing the column names and the binary cache files present on the file system. + +Those bin files will be fetched and decoded in the browser. +When done, a "Download" button will be presented which generates and downloads a new CSV file. + +This file can be opened in any spreadsheet program. +LibreNMS has proven to be the easiest to parse the column separators and make the best guess on the data types in each cell. + + + diff --git a/docs/source/Reference/RTC.rst b/docs/source/Reference/RTC.rst index 9b7dca611..7bdfa5888 100644 --- a/docs/source/Reference/RTC.rst +++ b/docs/source/Reference/RTC.rst @@ -13,6 +13,9 @@ ESP32 does also have RTC memory, but that's organised a bit different. RTC layout ESPEasy ------------------ +ESP8266 +^^^^^^^ + On the ESP82xx the RTC memory is addressable per 32 bit. In total, there is 768 bytes (192 addressable blocks). @@ -26,6 +29,18 @@ In total, there is 768 bytes (192 addressable blocks). * 132 .. 191 (240 bytes) Cache Controller (C016) data 6 blocks per sample => max 10 samples +ESP32 +^^^^^ + +On ESP32, the compiler determines where an object is stored in RTC. +Thus data stored in RTC may appear corrupt to a newly flashed build if the addresses where an object is stored may have changed. + +Structures stored in RTC: + +* RTC Struct +* UserVar (task values) + + RTC Struct ---------- diff --git a/misc/CacheController/Readme.txt b/misc/CacheController/Readme.txt new file mode 100644 index 000000000..6785bcc93 --- /dev/null +++ b/misc/CacheController/Readme.txt @@ -0,0 +1,18 @@ +Fetch and decode bin files from the cache controller +**************************************************** + +The dump5.htm file should be uploaded to the file system of the ESP. +When there are cache controller bin files present on the file system, +open this htm file from the file browser in ESPEasy into a new tab in your browser. + +This presents a large button "Fetch cache files". +When pressed, the JavaScript in this htm file will fetch JSON information from +ESPEasy describing the column names and the binary cache files present on the file system. + +Those bin files will be fetched and decoded in the browser. +When done, a "Download" button will be presented which generates and downloads a new CSV file. + +This file can be opened in any spreadsheet program. +LibreNMS has proven to be the easiest to parse the column separators and make the best guess on the data types in each cell. + + diff --git a/misc/CacheController/dump5.htm b/misc/CacheController/dump5.htm new file mode 100644 index 000000000..a43f5d280 --- /dev/null +++ b/misc/CacheController/dump5.htm @@ -0,0 +1,263 @@ + + + + + + + + +

ESPeasy cache to CSV

+
+ + + +
+
0%
+
+
+ + + \ No newline at end of file diff --git a/src/src/CustomBuild/define_plugin_sets.h b/src/src/CustomBuild/define_plugin_sets.h index f022da2ab..42fd020ab 100644 --- a/src/src/CustomBuild/define_plugin_sets.h +++ b/src/src/CustomBuild/define_plugin_sets.h @@ -1408,9 +1408,7 @@ To create/register a plugin, you have to : #endif #endif #ifndef USES_C016 - #ifndef ESP32 // Not implemented yet for ESP32 - #define USES_C016 // Cache controller - #endif + #define USES_C016 // Cache controller #endif #ifndef USES_C018 #define USES_C018 // TTN RN2483 @@ -1437,13 +1435,6 @@ To create/register a plugin, you have to : // #undef USES_P075 // Nextion // #undef USES_P078 // Eastron Modbus Energy meters (doesn't work yet on ESP32) // #undef USES_P082 // GPS - - #ifdef USES_C016 - // Cache controller uses RTC memory which we do not yet support on ESP32. - #undef USES_C016 // Cache controller - #endif - - #endif diff --git a/src/src/DataStructs/RTCStruct.h b/src/src/DataStructs/RTCStruct.h index 58affc6a9..69f8dddf7 100644 --- a/src/src/DataStructs/RTCStruct.h +++ b/src/src/DataStructs/RTCStruct.h @@ -8,7 +8,15 @@ #define RTC_BASE_USERVAR 74 #define RTC_BASE_CACHE 124 -#define RTC_CACHE_DATA_SIZE 240 +#ifdef ESP8266 +#define RTC_CACHE_DATA_SIZE 240 // 10 elements +#endif +#ifdef ESP32 +// TODO TD-er: ESP32 can store much more samples in its RTC +// However we must make sure the data can be flushed on demand or else +// one may have to wait for a long time to be able to read the data from the filesystem +#define RTC_CACHE_DATA_SIZE 240 // 10 elements +#endif #define CACHE_FILE_MAX_SIZE 24000 /*********************************************************************************************\ diff --git a/src/src/DataStructs/RTC_cache_handler_struct.cpp b/src/src/DataStructs/RTC_cache_handler_struct.cpp index e3e98b256..e53b0fe16 100644 --- a/src/src/DataStructs/RTC_cache_handler_struct.cpp +++ b/src/src/DataStructs/RTC_cache_handler_struct.cpp @@ -1,14 +1,30 @@ #include "RTC_cache_handler_struct.h" #include "../../ESPEasy_common.h" -#include "RTCStruct.h" +#include "../DataStructs/RTCStruct.h" #include "../Helpers/CRC_functions.h" #include "../Helpers/ESPEasy_Storage.h" +#include "../ESPEasyCore/ESPEasy_Log.h" + #ifdef ESP8266 #include #endif +#ifdef ESP32 + #include + + // For ESP32 the RTC mapped structure may not be a member of an object, + // but must be declared 'static' + // This also means we can only have a single instance of this + // RTC_cache_handler_struct. + RTC_NOINIT_ATTR RTC_cache_struct RTC_cache; + RTC_NOINIT_ATTR uint8_t RTC_cache_data[RTC_CACHE_DATA_SIZE]; +#endif + + + + /********************************************************************************************\ RTC located cache \*********************************************************************************************/ @@ -122,7 +138,14 @@ bool RTC_cache_handler_struct::flush() { size_t filesize = fw.size(); int bytesWriten = fw.write(&RTC_cache_data[0], RTC_cache.writePos); - if ((bytesWriten < RTC_cache.writePos) || (fw.size() == filesize)) { + delay(0); + fw.flush(); + #ifdef RTC_STRUCT_DEBUG + addLog(LOG_LEVEL_INFO, F("RTC : flush RTC cache")); + #endif // ifdef RTC_STRUCT_DEBUG + + + if ((bytesWriten < RTC_cache.writePos) /*|| (fw.size() == filesize)*/) { #ifdef RTC_STRUCT_DEBUG String log = F("RTC : error writing file. Size before: "); log += filesize; @@ -140,11 +163,6 @@ bool RTC_cache_handler_struct::flush() { } return false; } - delay(0); - fw.flush(); - #ifdef RTC_STRUCT_DEBUG - addLog(LOG_LEVEL_INFO, F("RTC : flush RTC cache")); - #endif // ifdef RTC_STRUCT_DEBUG initRTCcache_data(); clearRTCcacheData(); saveRTCcache(); @@ -203,10 +221,13 @@ String RTC_cache_handler_struct::getPeekCacheFileName(bool& islast) { bool RTC_cache_handler_struct::deleteOldestCacheBlock() { if (updateRTC_filenameCounters()) { - if (RTC_cache.readFileNr != RTC_cache.writeFileNr) { + const int nrCacheFiles = RTC_cache.writeFileNr - RTC_cache.readFileNr; + + if (nrCacheFiles > 1) { // read and write file nr are not the same file, remove the read file nr. String fname = createCacheFilename(RTC_cache.readFileNr); + writeerror = false; if (tryDeleteFile(fname)) { #ifdef RTC_STRUCT_DEBUG String log = F("RTC : Removed file from FS: "); @@ -214,7 +235,6 @@ bool RTC_cache_handler_struct::deleteOldestCacheBlock() { addLog(LOG_LEVEL_INFO, String(log)); #endif // ifdef RTC_STRUCT_DEBUG updateRTC_filenameCounters(); - writeerror = false; return true; } } @@ -224,28 +244,27 @@ bool RTC_cache_handler_struct::deleteOldestCacheBlock() { bool RTC_cache_handler_struct::loadMetaData() { - #if defined(ESP32) - return false; - #else // if defined(ESP32) + // No need to load on ESP32, as the data is already allocated to the RTC memory by the compiler + #ifdef ESP8266 if (!system_rtc_mem_read(RTC_BASE_CACHE, (byte *)&RTC_cache, sizeof(RTC_cache))) { return false; } + #endif return RTC_cache.checksumMetadata == calc_CRC32((byte *)&RTC_cache, sizeof(RTC_cache) - sizeof(uint32_t)); - #endif // if defined(ESP32) } bool RTC_cache_handler_struct::loadData() { - #if defined(ESP32) - return false; - #else // if defined(ESP32) initRTCcache_data(); + // No need to load on ESP32, as the data is already allocated to the RTC memory by the compiler + #ifdef ESP8266 if (!system_rtc_mem_read(RTC_BASE_CACHE + (sizeof(RTC_cache) / 4), (byte *)&RTC_cache_data[0], RTC_CACHE_DATA_SIZE)) { return false; } + #endif if (RTC_cache.checksumData != getDataChecksum()) { # ifdef RTC_STRUCT_DEBUG @@ -254,7 +273,6 @@ bool RTC_cache_handler_struct::loadData() return false; } return RTC_cache.checksumData == getDataChecksum(); - #endif // if defined(ESP32) } bool RTC_cache_handler_struct::saveRTCcache() { @@ -263,12 +281,13 @@ bool RTC_cache_handler_struct::saveRTCcache() { bool RTC_cache_handler_struct::saveRTCcache(unsigned int startOffset, size_t nrBytes) { - #if defined(ESP32) - return false; - #else // if defined(ESP32) RTC_cache.checksumData = getDataChecksum(); RTC_cache.checksumMetadata = calc_CRC32((byte *)&RTC_cache, sizeof(RTC_cache) - sizeof(uint32_t)); + #ifdef ESP32 + return true; + #endif + #ifdef ESP8266 if (!system_rtc_mem_write(RTC_BASE_CACHE, (byte *)&RTC_cache, sizeof(RTC_cache)) || !loadMetaData()) { # ifdef RTC_STRUCT_DEBUG @@ -293,26 +312,30 @@ bool RTC_cache_handler_struct::saveRTCcache(unsigned int startOffset, size_t nrB # endif // ifdef RTC_STRUCT_DEBUG } return true; - #endif // if defined(ESP32) + #endif } uint32_t RTC_cache_handler_struct::getDataChecksum() { initRTCcache_data(); + /* size_t dataLength = RTC_cache.writePos; if (dataLength > RTC_CACHE_DATA_SIZE) { // Is this allowed to happen? dataLength = RTC_CACHE_DATA_SIZE; } + */ // Only compute the checksum over the number of samples stored. return calc_CRC32((byte *)&RTC_cache_data[0], /*dataLength*/ RTC_CACHE_DATA_SIZE); } void RTC_cache_handler_struct::initRTCcache_data() { + #ifdef ESP8266 if (RTC_cache_data.size() != RTC_CACHE_DATA_SIZE) { RTC_cache_data.resize(RTC_CACHE_DATA_SIZE); } + #endif if (RTC_cache.writeFileNr == 0) { // RTC value not reliable diff --git a/src/src/DataStructs/RTC_cache_handler_struct.h b/src/src/DataStructs/RTC_cache_handler_struct.h index fb2ddc67c..c9f541e2e 100644 --- a/src/src/DataStructs/RTC_cache_handler_struct.h +++ b/src/src/DataStructs/RTC_cache_handler_struct.h @@ -23,6 +23,8 @@ #define CACHE_STORAGE_BEHIND_SPIFFS 3 +#define RTC_STRUCT_DEBUG + /********************************************************************************************\ RTC located cache \*********************************************************************************************/ @@ -79,8 +81,10 @@ private: size_t nrBytes); #endif // ifdef RTC_STRUCT_DEBUG +#ifdef ESP8266 RTC_cache_struct RTC_cache; std::vectorRTC_cache_data; +#endif File fw; File fr; File fp; diff --git a/src/src/Helpers/ESPEasyRTC.cpp b/src/src/Helpers/ESPEasyRTC.cpp index e709dcd32..2ea4aa5f9 100644 --- a/src/src/Helpers/ESPEasyRTC.cpp +++ b/src/src/Helpers/ESPEasyRTC.cpp @@ -83,15 +83,19 @@ -// #define RTC_STRUCT_DEBUG +//#define RTC_STRUCT_DEBUG #ifdef ESP32 +constexpr size_t UserVar_nrelements = VARS_PER_TASK * TASKS_MAX; + + // Since the global UserVar and RTC objects are defined "extern", they cannot be located in the RTC memory. // Thus we have to keep a copy here. -RTC_NOINIT_ATTR float UserVar_RTC[VARS_PER_TASK * TASKS_MAX]; RTC_NOINIT_ATTR RTCStruct RTC_tmp; +RTC_NOINIT_ATTR float UserVar_RTC[UserVar_nrelements]; +RTC_NOINIT_ATTR uint32_t UserVar_checksum; #endif @@ -163,9 +167,10 @@ bool saveUserVarToRTC() // ESP8266 has the RTC struct stored in memory which we must actively fetch // ESP32 Uses a temp structure which is mapped to the RTC address range. #if defined(ESP32) - for (size_t i = 0; i < VARS_PER_TASK * TASKS_MAX; ++i) { + for (size_t i = 0; i < UserVar_nrelements; ++i) { UserVar_RTC[i] = UserVar[i]; } + UserVar_checksum = calc_CRC32((byte *)(&UserVar[0]), UserVar_nrelements * sizeof(float)); return true; #endif @@ -188,10 +193,13 @@ bool readUserVarFromRTC() // ESP8266 has the RTC struct stored in memory which we must actively fetch // ESP32 Uses a temp structure which is mapped to the RTC address range. #if defined(ESP32) - for (size_t i = 0; i < VARS_PER_TASK * TASKS_MAX; ++i) { - UserVar[i] = UserVar_RTC[i]; + if (calc_CRC32((byte *)(&UserVar_RTC[0]), UserVar_nrelements * sizeof(float)) == UserVar_checksum) { + for (size_t i = 0; i < UserVar_nrelements; ++i) { + UserVar[i] = UserVar_RTC[i]; + } + return true; } - return true; + return false; #endif #ifdef ESP8266 diff --git a/src/src/Helpers/ESPEasy_Storage.cpp b/src/src/Helpers/ESPEasy_Storage.cpp index f1756d378..9f2176e06 100644 --- a/src/src/Helpers/ESPEasy_Storage.cpp +++ b/src/src/Helpers/ESPEasy_Storage.cpp @@ -1396,9 +1396,9 @@ String createCacheFilename(unsigned int count) { #ifdef ESP32 fname = '/'; #endif // ifdef ESP32 - fname += "cache_"; + fname += F("cache_"); fname += String(count); - fname += ".bin"; + fname += F(".bin"); return fname; } @@ -1446,22 +1446,27 @@ bool getCacheFileCounters(uint16_t& lowest, uint16_t& highest, size_t& filesizeH } #endif // ESP8266 #ifdef ESP32 - File root = ESPEASY_FS.open(F("/cache")); + File root = ESPEASY_FS.open(F("/")); File file = root.openNextFile(); while (file) { if (!file.isDirectory()) { - int count = getCacheFileCountFromFilename(file.name()); + const String fname(file.name()); + if (fname.startsWith(F("/cache")) || fname.startsWith(F("cache"))) { + int count = getCacheFileCountFromFilename(fname); - if (count >= 0) { - if (lowest > count) { - lowest = count; - } + if (count >= 0) { + if (lowest > count) { + lowest = count; + } - if (highest < count) { - highest = count; - filesizeHighest = file.size(); + if (highest < count) { + highest = count; + filesizeHighest = file.size(); + } + } else { + addLog(LOG_LEVEL_INFO, String(F("RTC : Cannot get count from: ")) + fname); } } }