Michel2/ESPEasy
1
0
Fork 0
mirror of https://github.com/letscontrolit/ESPEasy.git synced 2026-09-12 01:24:04 +00:00
Code Issues Releases Activity

[Cache controller] Make Cache controller work on ESP32

Browse Source 
ESP32 does have less space on the file system than an ESP8266 with 4M flash.
So it is less useful on ESP32 with only 4M flash.

Still it does work :)
This commit is contained in:
TD-er
2021-05-12 17:29:41 +02:00
parent 7de0329f19
commit e3ea3020d8
10 changed files with 406 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/source/Controller/C016.rst
+22 -1
View File
@@ -63,8 +63,29 @@ Data Delivery
The controller can deliver the data to:
- JavaScript to process the data inside the browser. See the ``dump5.htm`` file in the ``misc`` folder.
- Upload bin files to some server (HTTP post?) (TODO)
- Provide a sample to any connected controller (TODO)
- Do nothing and let some extern host pull the data from the node. (TODO)
- JavaScript to process the data inside the browser. (TODO)
- Feed it to some plugin (e.g. a display to show a chart) (TODO)
Fetch and Decode data in the browser
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The ``dump5.htm`` (in ``misc`` directory) file should be uploaded to the file system of the ESP.
When there are cache controller bin files present on the file system,
open this htm file from the file browser in ESPEasy into a new tab in your browser.
This presents a large button "Fetch cache files".
When pressed, the JavaScript in this htm file will fetch JSON information from
ESPEasy describing the column names and the binary cache files present on the file system.
Those bin files will be fetched and decoded in the browser.
When done, a "Download" button will be presented which generates and downloads a new CSV file.
This file can be opened in any spreadsheet program.
LibreNMS has proven to be the easiest to parse the column separators and make the best guess on the data types in each cell.
docs/source/Reference/RTC.rst
+15
View File
@@ -13,6 +13,9 @@ ESP32 does also have RTC memory, but that's organised a bit different.
RTC layout ESPEasy
------------------
ESP8266
^^^^^^^
On the ESP82xx the RTC memory is addressable per 32 bit.
In total, there is 768 bytes (192 addressable blocks).
@@ -26,6 +29,18 @@ In total, there is 768 bytes (192 addressable blocks).
* 132 .. 191 (240 bytes) Cache Controller (C016) data 6 blocks per sample => max 10 samples
ESP32
^^^^^
On ESP32, the compiler determines where an object is stored in RTC.
Thus data stored in RTC may appear corrupt to a newly flashed build if the addresses where an object is stored may have changed.
Structures stored in RTC:
* RTC Struct
* UserVar (task values)
RTC Struct
----------
misc/CacheController/Readme.txt
+18
View File
@@ -0,0 +1,18 @@
Fetch and decode bin files from the cache controller
****************************************************
The dump5.htm file should be uploaded to the file system of the ESP.
When there are cache controller bin files present on the file system,
open this htm file from the file browser in ESPEasy into a new tab in your browser.
This presents a large button "Fetch cache files".
When pressed, the JavaScript in this htm file will fetch JSON information from
ESPEasy describing the column names and the binary cache files present on the file system.
Those bin files will be fetched and decoded in the browser.
When done, a "Download" button will be presented which generates and downloads a new CSV file.
This file can be opened in any spreadsheet program.
LibreNMS has proven to be the easiest to parse the column separators and make the best guess on the data types in each cell.
misc/CacheController/dump5.htm
+263
View File
@@ -0,0 +1,263 @@
<html>
<script src="https://cdn.jsdelivr.net/npm/lodash@4.17.11/lodash.min.js"></script>
<script>
const TASKS_MAX = 12;
const VARS_PER_TASK = 4;
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
class DataParser {
constructor(data) {
this.view = new DataView(data);
this.offset = 0;
this.bitbyte = 0;
this.bitbytepos = 7;
}
pad(nr) {
while (this.offset % nr) {
this.offset++;
}
}
bit(signed = false, write = false, val) {
if (this.bitbytepos === 7) {
if (!write) {
this.bitbyte = this.byte();
this.bitbytepos = 0;
} else {
this.byte(signed, write, this.bitbyte);
}
}
if (!write) {
return (this.bitbyte >> this.bitbytepos++) & 1;
} else {
this.bitbyte = val ? (this.bitbyte | (1 << this.bitbytepos++)) : (this.bitbyte & ~(1 << this.bitbytepos++));
}
}
byte(signed = false, write = false, val) {
this.pad(1);
const fn = `${write ? 'set' : 'get'}${signed ? 'Int8' : 'Uint8'}`;
const res = this.view[fn](this.offset, val);
this.offset += 1;
return res;
}
int16(signed = false, write = false, val) {
this.pad(2);
let fn = signed ? 'Int16' : 'Uint16';
const res = write ? this.view[`set${fn}`](this.offset, val, true) : this.view[`get${fn}`](this.offset, true);
this.offset += 2;
return res;
}
int32(signed = false, write = false, val) {
this.pad(4);
let fn = signed ? 'Int32' : 'Uint32';
const res = write ? this.view[`set${fn}`](this.offset, val, true) : this.view[`get${fn}`](this.offset, true);
this.offset += 4;
return res;
}
float(signed = false, write = false, val) {
this.pad(4);
const res = write ? this.view.setFloat32(this.offset, val, true) : this.view.getFloat32(this.offset, true);
this.offset += 4;
return res;
}
bytes(nr, signed = false, write = false, vals) {
const res = [];
for (var x = 0; x < nr; x++) {
res.push(this.byte(signed, write, vals ? vals[x] : null));
}
return res;
}
ints(nr, signed = false, write = false, vals) {
const res = [];
for (var x = 0; x < nr; x++) {
res.push(this.int16(signed, write, vals ? vals[x] : null));
}
return res;
}
longs(nr, signed = false, write = false, vals) {
const res = [];
for (var x = 0; x < nr; x++) {
res.push(this.int32(signed, write, vals ? vals[x] : null));
}
return res;
}
floats(nr, signed = false, write = false, vals) {
const res = [];
for (var x = 0; x < nr; x++) {
res.push(this.float(write, vals ? vals[x] : null));
}
return res;
}
string(nr, signed = false, write = false, val) {
if (write) {
for (var i = 0; i < nr; ++i) {
var code = val.charCodeAt(i) || '\0';
this.byte(false, true, code);
}
} else {
const res = this.bytes(nr);
return String.fromCharCode.apply(null, res).replace(/\x00/g, '');
}
}
}
const parseConfig = (data, config, start) => {
const p = new DataParser(data);
if (start) p.offset = start;
const result = {};
config.map(value => {
const prop = value.length ? value.length : value.signed;
_.set(result, value.prop, p[value.type](prop, value.signed));
});
return result;
}
/*
const fileFormat = [
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].values`, type:'floats', length: VARS_PER_TASK })),
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].timestamp`, type: 'longs', signed: false })),
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].taskIndex`, type: 'byte' })),
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].controllerIndex`, type: 'byte' })),
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].sensorType`, type: 'byte' })),
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].valueCount`, type: 'byte' })),
];
*/
const fileFormat = [
{ prop: 'values', type:'floats', length: VARS_PER_TASK },
{ prop: 'timestamp', type: 'int32', signed: false },
{ prop: 'taskIndex', type: 'byte' },
{ prop: 'controllerIndex', type: 'byte' },
{ prop: 'sensorType', type: 'byte' },
{ prop: 'valueCount', type: 'byte' },
];
/*
loadConfig = () => {
return fetch('http://192.168.1.182/cache_json').then(response => response.arrayBuffer()).then(async response => {
document.body.innerText = parseConfig(response, fileFormat);
});
}
*/
loadConfig = async () => {
const floatvalues = {};
const info = await fetch('/cache_json').then(response => response.json());
let csv = info.columns.join(';') + '\n';
for (var j = 0; j < (VARS_PER_TASK * TASKS_MAX); j++) {
// TODO make "unused" value configurable
floatvalues[j] = 0;
}
// TODO must also read partial files (< 24k)
var maxFileNr = info.files.length;
for (var filenr = 0; filenr < maxFileNr; filenr++) {
var elem = document.getElementById("bar");
var width = Math.round(100.0 * (filenr / (maxFileNr - 1 )));
elem.style.width = width + '%';
elem.innerHTML = width * 1 + '%';
const binary = await fetch(info.files[filenr]).then(response => response.arrayBuffer()).then(async response => {
const samples = {};
var arrayLength = Math.floor(response.byteLength / 24);
//1000;//samples.length;
[...Array(arrayLength)].map((x, i) => {
samples[i] = parseConfig(response, fileFormat, 24 * i);
});
// TODO Fetch number of samples.
for (var i = 0; i < arrayLength; i++) {
var floatIndex = VARS_PER_TASK * samples[i].taskIndex;
samples[i].values.forEach(item => {
floatvalues[floatIndex] = item;
floatIndex++;
});
// TODO quick fix to remove damaged samples due to writing to closed files.
if (samples[i].timestamp > 1500000000) {
const utc_date = new Date(samples[i].timestamp * 1000);
csv += samples[i].timestamp + ';' + utc_date.toISOString() + ';' + samples[i].taskIndex;
for (var j = 0; j < (VARS_PER_TASK * TASKS_MAX); j++) {
csv += ';' + floatvalues[j];
}
csv += '\n';
}
}
await sleep(100); // Wait to prevent the ESPeasy node from rebooting.
});
}
// document.body.innerText = csv;
// document.body.innerHTML = `<a href='data:text/plain;charset=utf-8,${encodeURIComponent(csv)}' download='test.csv'>click me</a>`;
const a = document.createElement('a');
const aText = document.createTextNode('Download');
a.href = window.URL.createObjectURL(new Blob([csv]), {type: 'text/csv'});
a.download = 'test.csv';
a.appendChild(aText);
a.classList.add("button");
document.getElementById("downloadLink").appendChild(a);
// document.body.appendChild(a);
}
</script>
<style>
#progress {
width: 100%;
background-color: #ddd;
}
#bar {
width: 0%;
height: 30px;
background-color: #4CAF50;
text-align: center;
line-height: 30px;
color: white;
}
h1, h2 {
font-size: 16pt;
margin: 8px 0;
}
h1, h2 {
color: #07D;
}
* {
box-sizing: border-box;
font-family: sans-serif;
font-size: 12pt;
margin: 0;
padding: 0;
}
.button {
background-color: #07D;
border: none;
border-radius: 4px;
color: #FFF;
margin: 4px;
padding: 4px 16px;
}
</style>
<body>
<h2>ESPeasy cache to CSV</h2>
<BR>
<button class="button" type="button" onclick="loadConfig()">Fetch cache files</button>
<div id="progress">
<div id="bar">0%</div>
</div>
<BR>
<p id="downloadLink"></p>
</body>
</html>
src/src/CustomBuild/define_plugin_sets.h
+1 -10
View File
@@ -1408,9 +1408,7 @@ To create/register a plugin, you have to :
#endif
#endif
#ifndef USES_C016
#ifndef ESP32 // Not implemented yet for ESP32
#define USES_C016 // Cache controller
#endif
#define USES_C016 // Cache controller
#endif
#ifndef USES_C018
#define USES_C018 // TTN RN2483
@@ -1437,13 +1435,6 @@ To create/register a plugin, you have to :
// #undef USES_P075 // Nextion
// #undef USES_P078 // Eastron Modbus Energy meters (doesn't work yet on ESP32)
// #undef USES_P082 // GPS
#ifdef USES_C016
// Cache controller uses RTC memory which we do not yet support on ESP32.
#undef USES_C016 // Cache controller
#endif
#endif
src/src/DataStructs/RTCStruct.h
+9 -1
View File
@@ -8,7 +8,15 @@
#define RTC_BASE_USERVAR 74
#define RTC_BASE_CACHE 124
#define RTC_CACHE_DATA_SIZE 240
#ifdef ESP8266
#define RTC_CACHE_DATA_SIZE 240 // 10 elements
#endif
#ifdef ESP32
// TODO TD-er: ESP32 can store much more samples in its RTC
// However we must make sure the data can be flushed on demand or else
// one may have to wait for a long time to be able to read the data from the filesystem
#define RTC_CACHE_DATA_SIZE 240 // 10 elements
#endif
#define CACHE_FILE_MAX_SIZE 24000
/*********************************************************************************************\
src/src/DataStructs/RTC_cache_handler_struct.cpp
+44 -21
View File
@@ -1,14 +1,30 @@
#include "RTC_cache_handler_struct.h"
#include "../../ESPEasy_common.h"
#include "RTCStruct.h"
#include "../DataStructs/RTCStruct.h"
#include "../Helpers/CRC_functions.h"
#include "../Helpers/ESPEasy_Storage.h"
#include "../ESPEasyCore/ESPEasy_Log.h"
#ifdef ESP8266
#include <user_interface.h>
#endif
#ifdef ESP32
#include <soc/rtc.h>
// For ESP32 the RTC mapped structure may not be a member of an object,
// but must be declared 'static'
// This also means we can only have a single instance of this
// RTC_cache_handler_struct.
RTC_NOINIT_ATTR RTC_cache_struct RTC_cache;
RTC_NOINIT_ATTR uint8_t RTC_cache_data[RTC_CACHE_DATA_SIZE];
#endif
/********************************************************************************************\
RTC located cache
\*********************************************************************************************/
@@ -122,7 +138,14 @@ bool RTC_cache_handler_struct::flush() {
size_t filesize = fw.size();
int bytesWriten = fw.write(&RTC_cache_data[0], RTC_cache.writePos);
if ((bytesWriten < RTC_cache.writePos) || (fw.size() == filesize)) {
delay(0);
fw.flush();
#ifdef RTC_STRUCT_DEBUG
addLog(LOG_LEVEL_INFO, F("RTC : flush RTC cache"));
#endif // ifdef RTC_STRUCT_DEBUG
if ((bytesWriten < RTC_cache.writePos) /*|| (fw.size() == filesize)*/) {
#ifdef RTC_STRUCT_DEBUG
String log = F("RTC : error writing file. Size before: ");
log += filesize;
@@ -140,11 +163,6 @@ bool RTC_cache_handler_struct::flush() {
}
return false;
}
delay(0);
fw.flush();
#ifdef RTC_STRUCT_DEBUG
addLog(LOG_LEVEL_INFO, F("RTC : flush RTC cache"));
#endif // ifdef RTC_STRUCT_DEBUG
initRTCcache_data();
clearRTCcacheData();
saveRTCcache();
@@ -203,10 +221,13 @@ String RTC_cache_handler_struct::getPeekCacheFileName(bool& islast) {
bool RTC_cache_handler_struct::deleteOldestCacheBlock() {
if (updateRTC_filenameCounters()) {
if (RTC_cache.readFileNr != RTC_cache.writeFileNr) {
const int nrCacheFiles = RTC_cache.writeFileNr - RTC_cache.readFileNr;
if (nrCacheFiles > 1) {
// read and write file nr are not the same file, remove the read file nr.
String fname = createCacheFilename(RTC_cache.readFileNr);
writeerror = false;
if (tryDeleteFile(fname)) {
#ifdef RTC_STRUCT_DEBUG
String log = F("RTC : Removed file from FS: ");
@@ -214,7 +235,6 @@ bool RTC_cache_handler_struct::deleteOldestCacheBlock() {
addLog(LOG_LEVEL_INFO, String(log));
#endif // ifdef RTC_STRUCT_DEBUG
updateRTC_filenameCounters();
writeerror = false;
return true;
}
}
@@ -224,28 +244,27 @@ bool RTC_cache_handler_struct::deleteOldestCacheBlock() {
bool RTC_cache_handler_struct::loadMetaData()
{
#if defined(ESP32)
return false;
#else // if defined(ESP32)
// No need to load on ESP32, as the data is already allocated to the RTC memory by the compiler
#ifdef ESP8266
if (!system_rtc_mem_read(RTC_BASE_CACHE, (byte *)&RTC_cache, sizeof(RTC_cache))) {
return false;
}
#endif
return RTC_cache.checksumMetadata == calc_CRC32((byte *)&RTC_cache, sizeof(RTC_cache) - sizeof(uint32_t));
#endif // if defined(ESP32)
}
bool RTC_cache_handler_struct::loadData()
{
#if defined(ESP32)
return false;
#else // if defined(ESP32)
initRTCcache_data();
// No need to load on ESP32, as the data is already allocated to the RTC memory by the compiler
#ifdef ESP8266
if (!system_rtc_mem_read(RTC_BASE_CACHE + (sizeof(RTC_cache) / 4), (byte *)&RTC_cache_data[0], RTC_CACHE_DATA_SIZE)) {
return false;
}
#endif
if (RTC_cache.checksumData != getDataChecksum()) {
# ifdef RTC_STRUCT_DEBUG
@@ -254,7 +273,6 @@ bool RTC_cache_handler_struct::loadData()
return false;
}
return RTC_cache.checksumData == getDataChecksum();
#endif // if defined(ESP32)
}
bool RTC_cache_handler_struct::saveRTCcache() {
@@ -263,12 +281,13 @@ bool RTC_cache_handler_struct::saveRTCcache() {
bool RTC_cache_handler_struct::saveRTCcache(unsigned int startOffset, size_t nrBytes)
{
#if defined(ESP32)
return false;
#else // if defined(ESP32)
RTC_cache.checksumData = getDataChecksum();
RTC_cache.checksumMetadata = calc_CRC32((byte *)&RTC_cache, sizeof(RTC_cache) - sizeof(uint32_t));
#ifdef ESP32
return true;
#endif
#ifdef ESP8266
if (!system_rtc_mem_write(RTC_BASE_CACHE, (byte *)&RTC_cache, sizeof(RTC_cache)) || !loadMetaData())
{
# ifdef RTC_STRUCT_DEBUG
@@ -293,26 +312,30 @@ bool RTC_cache_handler_struct::saveRTCcache(unsigned int startOffset, size_t nrB
# endif // ifdef RTC_STRUCT_DEBUG
}
return true;
#endif // if defined(ESP32)
#endif
}
uint32_t RTC_cache_handler_struct::getDataChecksum() {
initRTCcache_data();
/*
size_t dataLength = RTC_cache.writePos;
if (dataLength > RTC_CACHE_DATA_SIZE) {
// Is this allowed to happen?
dataLength = RTC_CACHE_DATA_SIZE;
}
*/
// Only compute the checksum over the number of samples stored.
return calc_CRC32((byte *)&RTC_cache_data[0], /*dataLength*/ RTC_CACHE_DATA_SIZE);
}
void RTC_cache_handler_struct::initRTCcache_data() {
#ifdef ESP8266
if (RTC_cache_data.size() != RTC_CACHE_DATA_SIZE) {
RTC_cache_data.resize(RTC_CACHE_DATA_SIZE);
}
#endif
if (RTC_cache.writeFileNr == 0) {
// RTC value not reliable
src/src/DataStructs/RTC_cache_handler_struct.h
+4
View File
@@ -23,6 +23,8 @@
#define CACHE_STORAGE_BEHIND_SPIFFS 3
#define RTC_STRUCT_DEBUG
/********************************************************************************************\
RTC located cache
\*********************************************************************************************/
@@ -79,8 +81,10 @@ private:
size_t nrBytes);
#endif // ifdef RTC_STRUCT_DEBUG
#ifdef ESP8266
RTC_cache_struct RTC_cache;
std::vector<uint8_t>RTC_cache_data;
#endif
File fw;
File fr;
File fp;
src/src/Helpers/ESPEasyRTC.cpp
+14 -6
View File
@@ -83,15 +83,19 @@
// #define RTC_STRUCT_DEBUG
//#define RTC_STRUCT_DEBUG
#ifdef ESP32
constexpr size_t UserVar_nrelements = VARS_PER_TASK * TASKS_MAX;
// Since the global UserVar and RTC objects are defined "extern", they cannot be located in the RTC memory.
// Thus we have to keep a copy here.
RTC_NOINIT_ATTR float UserVar_RTC[VARS_PER_TASK * TASKS_MAX];
RTC_NOINIT_ATTR RTCStruct RTC_tmp;
RTC_NOINIT_ATTR float UserVar_RTC[UserVar_nrelements];
RTC_NOINIT_ATTR uint32_t UserVar_checksum;
#endif
@@ -163,9 +167,10 @@ bool saveUserVarToRTC()
// ESP8266 has the RTC struct stored in memory which we must actively fetch
// ESP32 Uses a temp structure which is mapped to the RTC address range.
#if defined(ESP32)
for (size_t i = 0; i < VARS_PER_TASK * TASKS_MAX; ++i) {
for (size_t i = 0; i < UserVar_nrelements; ++i) {
UserVar_RTC[i] = UserVar[i];
}
UserVar_checksum = calc_CRC32((byte *)(&UserVar[0]), UserVar_nrelements * sizeof(float));
return true;
#endif
@@ -188,10 +193,13 @@ bool readUserVarFromRTC()
// ESP8266 has the RTC struct stored in memory which we must actively fetch
// ESP32 Uses a temp structure which is mapped to the RTC address range.
#if defined(ESP32)
for (size_t i = 0; i < VARS_PER_TASK * TASKS_MAX; ++i) {
UserVar[i] = UserVar_RTC[i];
if (calc_CRC32((byte *)(&UserVar_RTC[0]), UserVar_nrelements * sizeof(float)) == UserVar_checksum) {
for (size_t i = 0; i < UserVar_nrelements; ++i) {
UserVar[i] = UserVar_RTC[i];
}
return true;
}
return true;
return false;
#endif
#ifdef ESP8266
src/src/Helpers/ESPEasy_Storage.cpp
+16 -11
View File
@@ -1396,9 +1396,9 @@ String createCacheFilename(unsigned int count) {
#ifdef ESP32
fname = '/';
#endif // ifdef ESP32
fname += "cache_";
fname += F("cache_");
fname += String(count);
fname += ".bin";
fname += F(".bin");
return fname;
}
@@ -1446,22 +1446,27 @@ bool getCacheFileCounters(uint16_t& lowest, uint16_t& highest, size_t& filesizeH
}
#endif // ESP8266
#ifdef ESP32
File root = ESPEASY_FS.open(F("/cache"));
File root = ESPEASY_FS.open(F("/"));
File file = root.openNextFile();
while (file)
{
if (!file.isDirectory()) {
int count = getCacheFileCountFromFilename(file.name());
const String fname(file.name());
if (fname.startsWith(F("/cache")) || fname.startsWith(F("cache"))) {
int count = getCacheFileCountFromFilename(fname);
if (count >= 0) {
if (lowest > count) {
lowest = count;
}
if (count >= 0) {
if (lowest > count) {
lowest = count;
}
if (highest < count) {
highest = count;
filesizeHighest = file.size();
if (highest < count) {
highest = count;
filesizeHighest = file.size();
}
} else {
addLog(LOG_LEVEL_INFO, String(F("RTC : Cannot get count from: ")) + fname);
}
}
}