mirror of
https://github.com/letscontrolit/ESPEasy.git
synced 2026-07-27 19:57:38 +00:00
[Cache Reader] Store/fetch Plugin ID per sample
This commit is contained in:
TD-er
@@ -0,0 +1,277 @@
|
||||
|
||||
<html>
|
||||
<script src="https://cdn.jsdelivr.net/npm/lodash@4.17.11/lodash.min.js"></script>
|
||||
<script>
|
||||
const TASKS_MAX = 12;
|
||||
const VARS_PER_TASK = 4;
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
class DataParser {
|
||||
constructor(data) {
|
||||
this.view = new DataView(data);
|
||||
this.offset = 0;
|
||||
this.bitbyte = 0;
|
||||
this.bitbytepos = 7;
|
||||
}
|
||||
|
||||
pad(nr) {
|
||||
while (this.offset % nr) {
|
||||
this.offset++;
|
||||
}
|
||||
}
|
||||
|
||||
bit(signed = false, write = false, val) {
|
||||
if (this.bitbytepos === 7) {
|
||||
if (!write) {
|
||||
this.bitbyte = this.byte();
|
||||
this.bitbytepos = 0;
|
||||
} else {
|
||||
this.byte(signed, write, this.bitbyte);
|
||||
}
|
||||
}
|
||||
if (!write) {
|
||||
return (this.bitbyte >> this.bitbytepos++) & 1;
|
||||
} else {
|
||||
this.bitbyte = val ? (this.bitbyte | (1 << this.bitbytepos++)) : (this.bitbyte & ~(1 << this.bitbytepos++));
|
||||
}
|
||||
}
|
||||
|
||||
byte(signed = false, write = false, val) {
|
||||
this.pad(1);
|
||||
const fn = `${write ? 'set' : 'get'}${signed ? 'Int8' : 'Uint8'}`;
|
||||
const res = this.view[fn](this.offset, val);
|
||||
this.offset += 1;
|
||||
return res;
|
||||
}
|
||||
|
||||
int16(signed = false, write = false, val) {
|
||||
this.pad(2);
|
||||
let fn = signed ? 'Int16' : 'Uint16';
|
||||
const res = write ? this.view[`set${fn}`](this.offset, val, true) : this.view[`get${fn}`](this.offset, true);
|
||||
this.offset += 2;
|
||||
return res;
|
||||
}
|
||||
|
||||
int32(signed = false, write = false, val) {
|
||||
this.pad(4);
|
||||
let fn = signed ? 'Int32' : 'Uint32';
|
||||
const res = write ? this.view[`set${fn}`](this.offset, val, true) : this.view[`get${fn}`](this.offset, true);
|
||||
this.offset += 4;
|
||||
return res;
|
||||
}
|
||||
float(signed = false, write = false, val) {
|
||||
this.pad(4);
|
||||
const res = write ? this.view.setFloat32(this.offset, val, true) : this.view.getFloat32(this.offset, true);
|
||||
this.offset += 4;
|
||||
return res;
|
||||
}
|
||||
bytes(nr, signed = false, write = false, vals) {
|
||||
const res = [];
|
||||
for (var x = 0; x < nr; x++) {
|
||||
res.push(this.byte(signed, write, vals ? vals[x] : null));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
ints(nr, signed = false, write = false, vals) {
|
||||
const res = [];
|
||||
for (var x = 0; x < nr; x++) {
|
||||
res.push(this.int16(signed, write, vals ? vals[x] : null));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
longs(nr, signed = false, write = false, vals) {
|
||||
const res = [];
|
||||
for (var x = 0; x < nr; x++) {
|
||||
res.push(this.int32(signed, write, vals ? vals[x] : null));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
floats(nr, signed = false, write = false, vals) {
|
||||
const res = [];
|
||||
for (var x = 0; x < nr; x++) {
|
||||
res.push(this.float(write, vals ? vals[x] : null));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
string(nr, signed = false, write = false, val) {
|
||||
if (write) {
|
||||
for (var i = 0; i < nr; ++i) {
|
||||
var code = val.charCodeAt(i) || '\0';
|
||||
this.byte(false, true, code);
|
||||
}
|
||||
} else {
|
||||
const res = this.bytes(nr);
|
||||
return String.fromCharCode.apply(null, res).replace(/\x00/g, '');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const parseConfig = (data, config, start) => {
|
||||
const p = new DataParser(data);
|
||||
if (start) p.offset = start;
|
||||
const result = {};
|
||||
config.map(value => {
|
||||
const prop = value.length ? value.length : value.signed;
|
||||
_.set(result, value.prop, p[value.type](prop, value.signed));
|
||||
});
|
||||
return result;
|
||||
}
|
||||
/*
|
||||
const fileFormat = [
|
||||
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].values`, type:'floats', length: VARS_PER_TASK })),
|
||||
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].timestamp`, type: 'longs', signed: false })),
|
||||
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].taskIndex`, type: 'byte' })),
|
||||
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].controllerIndex`, type: 'byte' })),
|
||||
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].sensorType`, type: 'byte' })),
|
||||
[...Array(1000)].map((x, i) => ({ prop: `samples[${i}].valueCount`, type: 'byte' })),
|
||||
];
|
||||
*/
|
||||
const fileFormat = [
|
||||
{ prop: 'values', type:'floats', length: VARS_PER_TASK },
|
||||
{ prop: 'timestamp', type: 'int32', signed: false },
|
||||
{ prop: 'taskIndex', type: 'byte' },
|
||||
{ prop: 'pluginID', type: 'byte' },
|
||||
{ prop: 'sensorType', type: 'byte' },
|
||||
{ prop: 'valueCount', type: 'byte' },
|
||||
];
|
||||
|
||||
/*
|
||||
loadConfig = () => {
|
||||
return fetch('http://192.168.1.182/cache_json').then(response => response.arrayBuffer()).then(async response => {
|
||||
document.body.innerText = parseConfig(response, fileFormat);
|
||||
});
|
||||
}
|
||||
*/
|
||||
|
||||
loadConfig = async () => {
|
||||
const floatvalues = {};
|
||||
const info = await fetch('/cache_json?pluginID=1').then(response => response.json());
|
||||
let separator = ',';
|
||||
if ("separator" in info) {
|
||||
separator = info.separator;
|
||||
}
|
||||
let csv = info.columns.join(separator) + '\n';
|
||||
for (var j = 0; j < (VARS_PER_TASK * TASKS_MAX); j++) {
|
||||
// TODO make "unused" value configurable
|
||||
floatvalues[j] = 0;
|
||||
}
|
||||
let hasPluginId = info.columns.includes("plugin ID");
|
||||
|
||||
// TODO must also read partial files (< 24k)
|
||||
var maxFileNr = info.files.length;
|
||||
for (var filenr = 0; filenr < maxFileNr; filenr++) {
|
||||
var elem = document.getElementById("bar");
|
||||
var width = Math.round(100.0 * (filenr / (maxFileNr - 1 )));
|
||||
elem.style.width = width + '%';
|
||||
elem.innerHTML = width * 1 + '%';
|
||||
const binary = await fetch(info.files[filenr]).then(response => response.arrayBuffer()).then(async response => {
|
||||
const samples = {};
|
||||
var arrayLength = Math.floor(response.byteLength / 24);
|
||||
//1000;//samples.length;
|
||||
|
||||
[...Array(arrayLength)].map((x, i) => {
|
||||
samples[i] = parseConfig(response, fileFormat, 24 * i);
|
||||
});
|
||||
|
||||
// TODO Fetch number of samples.
|
||||
for (var i = 0; i < arrayLength; i++) {
|
||||
var floatIndex = VARS_PER_TASK * samples[i].taskIndex;
|
||||
samples[i].values.forEach(item => {
|
||||
floatvalues[floatIndex] = item;
|
||||
floatIndex++;
|
||||
});
|
||||
// TODO quick fix to remove damaged samples due to writing to closed files.
|
||||
//if (samples[i].timestamp > 1500000000) {
|
||||
const utc_date = new Date(samples[i].timestamp * 1000);
|
||||
csv += samples[i].timestamp + separator + utc_date.toISOString() + separator + samples[i].taskIndex;
|
||||
if (hasPluginId) {
|
||||
csv += separator;
|
||||
if ("pluginID" in info) {
|
||||
csv += info.pluginID[samples[i].taskIndex];
|
||||
} else {
|
||||
csv += samples[i].pluginID;
|
||||
}
|
||||
}
|
||||
|
||||
for (var j = 0; j < (VARS_PER_TASK * TASKS_MAX); j++) {
|
||||
csv += separator + floatvalues[j];
|
||||
}
|
||||
csv += '\n';
|
||||
//}
|
||||
}
|
||||
await sleep(100); // Wait to prevent the ESPeasy node from rebooting.
|
||||
});
|
||||
}
|
||||
// document.body.innerText = csv;
|
||||
// document.body.innerHTML = `<a href='data:text/plain;charset=utf-8,${encodeURIComponent(csv)}' download='test.csv'>click me</a>`;
|
||||
|
||||
const a = document.createElement('a');
|
||||
const aText = document.createTextNode('Download');
|
||||
a.href = window.URL.createObjectURL(new Blob([csv]), {type: 'text/csv'});
|
||||
a.download = 'test.csv';
|
||||
a.appendChild(aText);
|
||||
a.classList.add("button");
|
||||
document.getElementById("downloadLink").appendChild(a);
|
||||
// document.body.appendChild(a);
|
||||
}
|
||||
</script>
|
||||
|
||||
<style>
|
||||
#progress {
|
||||
width: 100%;
|
||||
background-color: #ddd;
|
||||
}
|
||||
|
||||
#bar {
|
||||
width: 0%;
|
||||
height: 30px;
|
||||
background-color: #4CAF50;
|
||||
text-align: center;
|
||||
line-height: 30px;
|
||||
color: white;
|
||||
}
|
||||
|
||||
h1, h2 {
|
||||
font-size: 16pt;
|
||||
margin: 8px 0;
|
||||
}
|
||||
|
||||
h1, h2 {
|
||||
color: #07D;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
font-family: sans-serif;
|
||||
font-size: 12pt;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.button {
|
||||
background-color: #07D;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
color: #FFF;
|
||||
margin: 4px;
|
||||
padding: 4px 16px;
|
||||
}
|
||||
|
||||
</style>
|
||||
<body>
|
||||
|
||||
<h2>ESPeasy cache to CSV</h2>
|
||||
<BR>
|
||||
|
||||
<button class="button" type="button" onclick="loadConfig()">Fetch cache files</button>
|
||||
|
||||
<div id="progress">
|
||||
<div id="bar">0%</div>
|
||||
</div>
|
||||
<BR>
|
||||
<p id="downloadLink"></p>
|
||||
</body>
|
||||
</html>
|
||||
@@ -117,6 +117,10 @@ bool CPlugin_016(CPlugin::Function function, struct EventStruct *event, String&
|
||||
event,
|
||||
valueCount,
|
||||
C016_allowLocalSystemTime ? node_time.now() : node_time.getUnixTime());
|
||||
|
||||
// It makes no sense to keep the controller index when storing it.
|
||||
// re-purpose it to store the pluginID
|
||||
element.setPluginID_insteadOf_controller_idx();
|
||||
success = ControllerCache.write(reinterpret_cast<const uint8_t *>(&element), sizeof(element));
|
||||
|
||||
/*
|
||||
|
||||
@@ -146,7 +146,7 @@ boolean Plugin_146(uint8_t function, struct EventStruct *event, String& string)
|
||||
addFormCheckBox(F("Append 'bin' to topic"), F("appendbintopic"), P146_GET_APPEND_BINARY_TOPIC);
|
||||
addFormCheckBox(F("Send ReadPos"), F("sendreadpos"), P146_GET_SEND_READ_POS);
|
||||
addFormNumericBox(F("Minimal Send Interval"), F("minsendinterval"), P146_MINIMAL_SEND_INTERVAL, 0, 1000);
|
||||
addFormNumericBox(F("Max Message Size"), F("maxmsgsize"), P146_MQTT_MESSAGE_LENGTH, 0, MQTT_MAX_PACKET_SIZE - 200);
|
||||
addFormNumericBox(F("Max Message Size"), F("maxmsgsize"), P146_MQTT_MESSAGE_LENGTH, sizeof(C016_queue_element) + 16, MQTT_MAX_PACKET_SIZE - 200);
|
||||
|
||||
addFormSubHeader(F("Non MQTT Output Options"));
|
||||
addFormCheckBox(F("Send Timestamp"), F("sendtimestamp"), P146_GET_SEND_TIMESTAMP);
|
||||
|
||||
@@ -70,4 +70,8 @@ bool C016_queue_element::isDuplicate(const C016_queue_element& other) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
void C016_queue_element::setPluginID_insteadOf_controller_idx() {
|
||||
controller_idx = getPluginID_from_TaskIndex(TaskIndex);
|
||||
}
|
||||
|
||||
#endif // ifdef USES_C016
|
||||
|
||||
@@ -41,6 +41,10 @@ public:
|
||||
|
||||
const UnitMessageCount_t* getUnitMessageCount() const { return nullptr; }
|
||||
|
||||
// It makes no sense to keep the controller index when storing it.
|
||||
// re-purpose it to store the pluginID
|
||||
void setPluginID_insteadOf_controller_idx();
|
||||
|
||||
float values[VARS_PER_TASK] = { 0 };
|
||||
unsigned long _timestamp = 0; // Unix timestamp
|
||||
taskIndex_t TaskIndex = INVALID_TASK_INDEX;
|
||||
|
||||
@@ -46,11 +46,13 @@ uint32_t P146_data_struct::sendBinaryInBulk(taskIndex_t P146_TaskIndex, uint32_t
|
||||
bool done = false;
|
||||
|
||||
for (int chunk = 0; chunk < nrChunks && !done; ++chunk) {
|
||||
uint8_t chunkdata[chunkSize] = { 0 };
|
||||
C016_queue_element element;
|
||||
|
||||
if (ControllerCache.peek(chunkdata, chunkSize))
|
||||
if (ControllerCache.peek(reinterpret_cast<uint8_t *>(&element), chunkSize))
|
||||
{
|
||||
message += formatToHex_array(chunkdata, chunkSize);
|
||||
// It makes no sense to keep the controller index when storing it.
|
||||
element.setPluginID_insteadOf_controller_idx();
|
||||
message += formatToHex_array(reinterpret_cast<const uint8_t *>(&element), chunkSize);
|
||||
message += ';';
|
||||
} else {
|
||||
done = true;
|
||||
@@ -62,7 +64,8 @@ uint32_t P146_data_struct::sendBinaryInBulk(taskIndex_t P146_TaskIndex, uint32_t
|
||||
return 0;
|
||||
}
|
||||
messageLength = message.length();
|
||||
String topic = F("CULreader/upload");
|
||||
String topic = concat(F("tracker_v2/%sysname%_%unit%/") , getTaskDeviceName(P146_TaskIndex)) + F("/upload");
|
||||
topic = parseTemplate(topic);
|
||||
|
||||
if (MQTTpublish(enabledMqttController, P146_TaskIndex, std::move(topic), std::move(message), false)) {
|
||||
return messageLength;
|
||||
|
||||
@@ -100,6 +100,10 @@ void handle_cache_json() {
|
||||
addHtml(to_json_value(F("UTC timestamp")));
|
||||
addHtml(',');
|
||||
addHtml(to_json_value(F("task index")));
|
||||
if (hasArg(F("pluginID"))) {
|
||||
addHtml(',');
|
||||
addHtml(to_json_value(F("plugin ID")));
|
||||
}
|
||||
|
||||
for (taskIndex_t i = 0; i < TASKS_MAX; ++i) {
|
||||
for (int j = 0; j < VARS_PER_TASK; ++j) {
|
||||
@@ -128,6 +132,15 @@ void handle_cache_json() {
|
||||
}
|
||||
}
|
||||
addHtml(F("],\n"));
|
||||
addHtml(F("\"pluginID\": ["));
|
||||
for (taskIndex_t taskIndex = 0; validTaskIndex(taskIndex); ++taskIndex) {
|
||||
if (taskIndex != 0) {
|
||||
addHtml(',');
|
||||
}
|
||||
addHtmlInt(getPluginID_from_TaskIndex(taskIndex));
|
||||
}
|
||||
addHtml(F("],\n"));
|
||||
stream_next_json_object_value(F("separator"), F(";"));
|
||||
stream_last_json_object_value(F("nrfiles"), filenr);
|
||||
addHtml('\n');
|
||||
TXBuffer.endStream();
|
||||
|
||||
Reference in New Issue
Block a user