From 4ee133fdaf5a64f8a817d9b12ef8bad103bfecb2 Mon Sep 17 00:00:00 2001 From: TD-er Date: Tue, 17 Jan 2023 15:50:20 +0100 Subject: [PATCH] [Cleanup] Group timingstats & fix delete of cache controller files --- src/_P146_CacheControllerReader.ino | 7 +- src/src/Commands/Tasks.cpp | 8 ++ src/src/DataStructs/Caches.cpp | 26 +++--- src/src/DataStructs/Caches.h | 4 + src/src/DataStructs/ESPEasyControllerCache.h | 2 + .../DataStructs/ESPeasyControllerCache.cpp | 6 ++ .../DataStructs/RTC_cache_handler_struct.cpp | 14 +++- .../DataStructs/RTC_cache_handler_struct.h | 3 + src/src/DataStructs/TimingStats.cpp | 2 +- src/src/DataStructs/TimingStats.h | 81 +++++++++++-------- src/src/ESPEasyCore/Controller.cpp | 11 +-- src/src/Helpers/ESPEasyStatistics.cpp | 43 ---------- src/src/Helpers/ESPEasy_Storage.cpp | 24 +++++- src/src/Helpers/ESPEasy_Storage.h | 2 + src/src/Helpers/Rules_calculate.cpp | 3 + 15 files changed, 137 insertions(+), 99 deletions(-) diff --git a/src/_P146_CacheControllerReader.ino b/src/_P146_CacheControllerReader.ino index c17e88763..edff9e620 100644 --- a/src/_P146_CacheControllerReader.ino +++ b/src/_P146_CacheControllerReader.ino @@ -102,8 +102,8 @@ boolean Plugin_146(uint8_t function, struct EventStruct *event, String& string) P146_MQTT_MESSAGE_LENGTH = 800; String strings[P146_Nlines]; - strings[P146_TaskInfoTopicIndex] = F("tracker_v2/%sysname%_%unit%/%tskname%/upload_meta"); - strings[P146_PublishTopicIndex] = F("tracker_v2/%sysname%_%unit%/%tskname%/upload"); + strings[P146_TaskInfoTopicIndex] = F("%sysname%_%unit%/%tskname%/upload_meta"); + strings[P146_PublishTopicIndex] = F("%sysname%_%unit%/%tskname%/upload"); SaveCustomTaskSettings(event->TaskIndex, strings, P146_Nlines, 0); @@ -114,6 +114,9 @@ boolean Plugin_146(uint8_t function, struct EventStruct *event, String& string) case PLUGIN_INIT: { + // Init the controller cache handler, just in case the cache controller may not be enabled + ControllerCache.init(); + // Restore the last position from RTC when rebooting. ControllerCache.setPeekFilePos( P146_TASKVALUE_FILENR, diff --git a/src/src/Commands/Tasks.cpp b/src/src/Commands/Tasks.cpp index 51397db1b..1d004f0e0 100644 --- a/src/src/Commands/Tasks.cpp +++ b/src/src/Commands/Tasks.cpp @@ -6,6 +6,8 @@ #include "../Commands/Common.h" +#include "../DataStructs/TimingStats.h" + #include "../ESPEasyCore/Controller.h" #include "../ESPEasyCore/Serial.h" @@ -190,7 +192,10 @@ const __FlashStringHelper * Command_Task_ValueSetAndRun(struct EventStruct *even const __FlashStringHelper * returnvalue = taskValueSet(event, Line, taskIndex, success); if (success) { + START_TIMER; SensorSendTask(taskIndex); + STOP_TIMER(SENSOR_SEND_TASK); + return return_command_success(); } return returnvalue; @@ -208,7 +213,10 @@ const __FlashStringHelper * Command_Task_Run(struct EventStruct *event, const ch return F("TASK_NOT_ENABLED"); } + START_TIMER; SensorSendTask(taskIndex); + STOP_TIMER(SENSOR_SEND_TASK); + return return_command_success(); } diff --git a/src/src/DataStructs/Caches.cpp b/src/src/DataStructs/Caches.cpp index 839de6c3d..393577055 100644 --- a/src/src/DataStructs/Caches.cpp +++ b/src/src/DataStructs/Caches.cpp @@ -144,19 +144,23 @@ String Caches::getTaskDeviceValueName(taskIndex_t TaskIndex, uint8_t rel_index) return EMPTY_STRING; } +bool Caches::hasFormula(taskIndex_t TaskIndex) +{ + if (validTaskIndex(TaskIndex)) { + // Just a quick test to see if we do have a formula present. + // Task Formula are not used very often, so we will probably almost always have to return an empty string. + auto it = getExtraTaskSettings(TaskIndex); + + if (it != extraTaskSettings_cache.end()) { + return it->second.hasFormula; + } + } + return false; +} + String Caches::getTaskDeviceFormula(taskIndex_t TaskIndex, uint8_t rel_index) { - if (validTaskIndex(TaskIndex) && (rel_index < VARS_PER_TASK)) { - { - // Just a quick test to see if we do have a formula present. - // Task Formula are not used very often, so we will probably almost always have to return an empty string. - auto it = getExtraTaskSettings(TaskIndex); - - if (it != extraTaskSettings_cache.end()) { - if (!it->second.hasFormula) { return EMPTY_STRING; } - } - } - + if (rel_index < VARS_PER_TASK && hasFormula(TaskIndex)) { LoadTaskSettings(TaskIndex); return ExtraTaskSettings.TaskDeviceFormula[rel_index]; } diff --git a/src/src/DataStructs/Caches.h b/src/src/DataStructs/Caches.h index 2f713f55b..bb321a11a 100644 --- a/src/src/DataStructs/Caches.h +++ b/src/src/DataStructs/Caches.h @@ -67,6 +67,10 @@ struct Caches { String getTaskDeviceValueName(taskIndex_t TaskIndex, uint8_t rel_index); + // Check to see if at least one of the taskvalues has a non-empty formula field. + bool hasFormula(taskIndex_t TaskIndex); + + String getTaskDeviceFormula(taskIndex_t TaskIndex, uint8_t rel_index); diff --git a/src/src/DataStructs/ESPEasyControllerCache.h b/src/src/DataStructs/ESPEasyControllerCache.h index 63a9c8c5a..1ddff5a28 100644 --- a/src/src/DataStructs/ESPEasyControllerCache.h +++ b/src/src/DataStructs/ESPEasyControllerCache.h @@ -37,6 +37,8 @@ struct ControllerCache_struct { bool deleteAllCacheBlocks(); + void closeOpenFiles(); + void resetpeek(); bool peekDataAvailable() const; diff --git a/src/src/DataStructs/ESPeasyControllerCache.cpp b/src/src/DataStructs/ESPeasyControllerCache.cpp index 3f77336da..a8de1bef1 100644 --- a/src/src/DataStructs/ESPeasyControllerCache.cpp +++ b/src/src/DataStructs/ESPeasyControllerCache.cpp @@ -54,6 +54,12 @@ bool ControllerCache_struct::deleteOldestCacheBlock() { return false; } +void ControllerCache_struct::closeOpenFiles() { + if (_RTC_cache_handler != nullptr) { + _RTC_cache_handler->closeOpenFiles(); + } +} + bool ControllerCache_struct::deleteAllCacheBlocks() { if (_RTC_cache_handler != nullptr) { return _RTC_cache_handler->deleteAllCacheBlocks(); diff --git a/src/src/DataStructs/RTC_cache_handler_struct.cpp b/src/src/DataStructs/RTC_cache_handler_struct.cpp index 0b26b3fa5..3a9f5ee5f 100644 --- a/src/src/DataStructs/RTC_cache_handler_struct.cpp +++ b/src/src/DataStructs/RTC_cache_handler_struct.cpp @@ -75,8 +75,9 @@ bool RTC_cache_handler_struct::peekDataAvailable() const { if (_peekfilenr == RTC_cache.writeFileNr) { if (fw) { - if ((_peekreadpos + 1) < fw.position()) { return true; } + return ((_peekreadpos + 1) < fw.position()); } + return true; } return false; } @@ -361,6 +362,17 @@ bool RTC_cache_handler_struct::deleteOldestCacheBlock() { return false; } +void RTC_cache_handler_struct::closeOpenFiles() +{ + if (fr) { + fr.close(); + } + + if (fp) { + fp.close(); + } +} + bool RTC_cache_handler_struct::deleteAllCacheBlocks() { if (updateRTC_filenameCounters()) { diff --git a/src/src/DataStructs/RTC_cache_handler_struct.h b/src/src/DataStructs/RTC_cache_handler_struct.h index 56b30a393..cacb221e4 100644 --- a/src/src/DataStructs/RTC_cache_handler_struct.h +++ b/src/src/DataStructs/RTC_cache_handler_struct.h @@ -70,6 +70,9 @@ struct RTC_cache_handler_struct bool deleteAllCacheBlocks(); + // When trying to access cache files, like deleting them, these files must be closed first. + void closeOpenFiles(); + private: bool loadMetaData(); diff --git a/src/src/DataStructs/TimingStats.cpp b/src/src/DataStructs/TimingStats.cpp index ce0e0bf05..11e7c3184 100644 --- a/src/src/DataStructs/TimingStats.cpp +++ b/src/src/DataStructs/TimingStats.cpp @@ -208,9 +208,9 @@ const __FlashStringHelper* getMiscStatsName_F(TimingStatsElements stat) { case TimingStatsElements::SENSOR_SEND_TASK: return F("SensorSendTask()"); case TimingStatsElements::SEND_DATA_STATS: return F("sendData()"); case TimingStatsElements::COMPUTE_FORMULA_STATS: return F("Compute formula"); + case TimingStatsElements::COMPUTE_STATS: return F("Compute()"); case TimingStatsElements::PLUGIN_CALL_DEVICETIMER_IN: return F("PLUGIN_DEVICETIMER_IN"); case TimingStatsElements::SET_NEW_TIMER: return F("setNewTimerAt()"); - case TimingStatsElements::TIME_DIFF_COMPUTE: return F("timeDiff()"); case TimingStatsElements::MQTT_DELAY_QUEUE: return F("Delay queue MQTT"); case TimingStatsElements::TRY_CONNECT_HOST_TCP: return F("try_connect_host() (TCP)"); case TimingStatsElements::TRY_CONNECT_HOST_UDP: return F("try_connect_host() (UDP)"); diff --git a/src/src/DataStructs/TimingStats.h b/src/src/DataStructs/TimingStats.h index fcdc7ea7e..e92ac5779 100644 --- a/src/src/DataStructs/TimingStats.h +++ b/src/src/DataStructs/TimingStats.h @@ -23,21 +23,8 @@ // These TimingStatsElements must not be excluded when FEATURE_TIMING_STATS is not defined. // The Cxxx_DELAY_QUEUE defines are used in the macros to process the controller queues. enum class TimingStatsElements { - LOADFILE_STATS, - SAVEFILE_STATS, - LOOP_STATS, - PLUGIN_CALL_50PS, - PLUGIN_CALL_10PS, - PLUGIN_CALL_10PSU, - PLUGIN_CALL_1PS, - SENSOR_SEND_TASK, - CPLUGIN_CALL_10PS, - CPLUGIN_CALL_50PS, - SEND_DATA_STATS, - COMPUTE_FORMULA_STATS, - PLUGIN_CALL_DEVICETIMER_IN, - SET_NEW_TIMER, - TIME_DIFF_COMPUTE, + + // Controller queue MQTT_DELAY_QUEUE, C001_DELAY_QUEUE, C002_DELAY_QUEUE, @@ -57,6 +44,7 @@ enum class TimingStatsElements { C016_DELAY_QUEUE, C017_DELAY_QUEUE, C018_DELAY_QUEUE, + C018_AIR_TIME, C019_DELAY_QUEUE, C020_DELAY_QUEUE, C021_DELAY_QUEUE, @@ -64,40 +52,69 @@ enum class TimingStatsElements { C023_DELAY_QUEUE, C024_DELAY_QUEUE, C025_DELAY_QUEUE, - C018_AIR_TIME, - LOAD_TASK_SETTINGS, - SAVE_TASK_SETTINGS, - LOAD_CUSTOM_TASK_STATS, - LOAD_CONTROLLER_SETTINGS, - TRY_OPEN_FILE, - FS_GC_SUCCESS, - FS_GC_FAIL, + + + // Related to Task runs & sending data + rules + PLUGIN_CALL_50PS, + PLUGIN_CALL_10PS, + PLUGIN_CALL_10PSU, + PLUGIN_CALL_1PS, + CPLUGIN_CALL_10PS, + CPLUGIN_CALL_50PS, + SENSOR_SEND_TASK, + SEND_DATA_STATS, + COMPUTE_FORMULA_STATS, + COMPUTE_STATS, PARSE_SYSVAR, PARSE_SYSVAR_NOCHANGE, PARSE_TEMPLATE_PADDED, - RULES_PROCESSING, - RULES_PARSE_LINE, - RULES_PROCESS_MATCHED, - RULES_MATCH, - GRAT_ARP_STATS, - SAVE_TO_RTC, - BACKGROUND_TASKS, PROCESS_SYSTEM_EVENT_QUEUE, - HANDLE_SCHEDULER_IDLE, + RULES_MATCH, + RULES_PROCESSING, + RULES_PROCESS_MATCHED, + RULES_PARSE_LINE, + + // Related to file access + LOADFILE_STATS, + LOAD_TASK_SETTINGS, + LOAD_CUSTOM_TASK_STATS, + LOAD_CONTROLLER_SETTINGS, + SAVEFILE_STATS, + SAVE_TASK_SETTINGS, + TRY_OPEN_FILE, + FS_GC_SUCCESS, + FS_GC_FAIL, + + // Scheduler related + SAVE_TO_RTC, + PLUGIN_CALL_DEVICETIMER_IN, + SET_NEW_TIMER, HANDLE_SCHEDULER_TASK, + HANDLE_SCHEDULER_IDLE, + BACKGROUND_TASKS, + + // Web serving HANDLE_SERVING_WEBPAGE, HANDLE_SERVING_WEBPAGE_JSON, + + // Network related TRY_CONNECT_HOST_TCP, TRY_CONNECT_HOST_UDP, HOST_BY_NAME_STATS, + GRAT_ARP_STATS, WIFI_ISCONNECTED_STATS, WIFI_NOTCONNECTED_STATS, CONNECT_CLIENT_STATS, WIFI_SCAN_ASYNC, WIFI_SCAN_SYNC, + + // Time sync (also network related) NTP_SUCCESS, NTP_FAIL, - SYSTIME_UPDATED + SYSTIME_UPDATED, + + // Close to the lifetime stats shown on the timing stats page + LOOP_STATS }; #if FEATURE_TIMING_STATS diff --git a/src/src/ESPEasyCore/Controller.cpp b/src/src/ESPEasyCore/Controller.cpp index 13157bf53..e2d3ab5c4 100644 --- a/src/src/ESPEasyCore/Controller.cpp +++ b/src/src/ESPEasyCore/Controller.cpp @@ -648,7 +648,7 @@ void SensorSendTask(taskIndex_t TaskIndex) const uint8_t valueCount = getValueCountForTask(TaskIndex); // Store the previous value, in case %pvalue% is used in the formula String preValue[VARS_PER_TASK]; - if (Device[DeviceIndex].FormulaOption) { + if (Device[DeviceIndex].FormulaOption && Cache.hasFormula(TaskIndex)) { for (uint8_t varNr = 0; varNr < valueCount; varNr++) { const String formula = Cache.getTaskDeviceFormula(TaskIndex, varNr); @@ -672,14 +672,14 @@ void SensorSendTask(taskIndex_t TaskIndex) if (success) { - if (Device[DeviceIndex].FormulaOption) { - START_TIMER; - + if (Device[DeviceIndex].FormulaOption && Cache.hasFormula(TaskIndex)) { for (uint8_t varNr = 0; varNr < valueCount; varNr++) { String formula = Cache.getTaskDeviceFormula(TaskIndex, varNr); if (!formula.isEmpty()) { + START_TIMER; + // TD-er: Should we use the set nr of decimals here, or not round at all? // See: https://github.com/letscontrolit/ESPEasy/issues/3721#issuecomment-889649437 formula.replace(F("%pvalue%"), preValue[varNr]); @@ -689,9 +689,10 @@ void SensorSendTask(taskIndex_t TaskIndex) if (!isError(Calculate(parseTemplate(formula), result))) { UserVar[TempEvent.BaseVarIndex + varNr] = result; } + + STOP_TIMER(COMPUTE_FORMULA_STATS); } } - STOP_TIMER(COMPUTE_FORMULA_STATS); } sendData(&TempEvent); } diff --git a/src/src/Helpers/ESPEasyStatistics.cpp b/src/src/Helpers/ESPEasyStatistics.cpp index 612924149..91a0b082e 100644 --- a/src/src/Helpers/ESPEasyStatistics.cpp +++ b/src/src/Helpers/ESPEasyStatistics.cpp @@ -8,49 +8,6 @@ #include "../Globals/Protocol.h" #include "../Helpers/Convert.h" -/* - void logStatistics(uint8_t loglevel, bool clearStats) { - if (loglevelActiveFor(loglevel)) { - String log; - log.reserve(80); - for (auto& x: pluginStats) { - if (!x.second.isEmpty()) { - const int deviceIndex = x.first/256; - log = F("PluginStats P_"); - log += deviceIndex + 1; - log += '_'; - log += getPluginNameFromDeviceIndex(deviceIndex); - log += ' '; - log += getPluginFunctionName(x.first%256); - log += ' '; - log += getLogLine(x.second); - addLog(loglevel, log); - if (clearStats) x.second.reset(); - } - } - for (auto& x: miscStats) { - if (!x.second.isEmpty()) { - log = getMiscStatsName(x.first); - log += F(" stats: "); - log += getLogLine(x.second); - addLog(loglevel, log); - if (clearStats) x.second.reset(); - } - } - log = getMiscStatsName(TIME_DIFF_COMPUTE); - log += F(" stats: Count: "); - log += timediff_calls; - log += F(" - CPU cycles per call: "); - log += static_cast(timediff_cpu_cycles_total) / static_cast(timediff_calls); - addLog(loglevel, log); - if (clearStats) { - timediff_calls = 0; - timediff_cpu_cycles_total = 0; - } - } - } - */ - void stream_json_timing_stats(const TimingStats& stats, long timeSinceLastReset) { uint64_t minVal, maxVal; uint64_t count = stats.getMinMax(minVal, maxVal); diff --git a/src/src/Helpers/ESPEasy_Storage.cpp b/src/src/Helpers/ESPEasy_Storage.cpp index 2d16fc4d9..e88a568cb 100644 --- a/src/src/Helpers/ESPEasy_Storage.cpp +++ b/src/src/Helpers/ESPEasy_Storage.cpp @@ -49,6 +49,9 @@ #include "../Helpers/StringConverter.h" #include "../Helpers/StringParser.h" +#if FEATURE_RTC_CACHE_STORAGE +# include "../Globals/C016_ControllerCache.h" +#endif #ifdef ESP32 #include @@ -124,7 +127,7 @@ String appendToFile(const String& fname, const uint8_t *data, unsigned int size) SPIFFS_CHECK(f, fname.c_str()); SPIFFS_CHECK(f.write(data, size), fname.c_str()); f.close(); - return ""; + return EMPTY_STRING; } bool fileExists(const __FlashStringHelper * fname) @@ -148,8 +151,12 @@ bool fileExists(const String& fname) { res = SD.exists(patched_fname); } #endif - // Only keep track of existing files. Not the non-existing files from the cache controller - if (res || patched_fname.indexOf(F("cache_")) == -1) { + // Only keep track of existing files or non-existing filenames that may be requested several times. + // Not the non-existing files from the cache controller + #if FEATURE_RTC_CACHE_STORAGE + if (res || !isCacheFile(patched_fname)) + #endif + { Cache.fileExistsMap[patched_fname] = res; } if (Cache.fileCacheClearMoment == 0) { @@ -213,6 +220,11 @@ bool tryRenameFile(const String& fname_old, const String& fname_new) { bool tryDeleteFile(const String& fname) { if (fname.length() > 0) { + #if FEATURE_RTC_CACHE_STORAGE + if (isCacheFile(fname)) { + ControllerCache.closeOpenFiles(); + } + #endif if (fileMatchesTaskSettingsType(fname)) { clearAllCaches(); } else { @@ -1736,7 +1748,7 @@ String createCacheFilename(unsigned int count) { // Match string with an integer between '_' and ".bin" int getCacheFileCountFromFilename(const String& fname) { - if (fname.indexOf(F("cache_")) == -1) return -1; + if (!isCacheFile(fname)) return -1; int startpos = fname.indexOf('_'); if (startpos < 0) { return -1; } @@ -1753,6 +1765,10 @@ int getCacheFileCountFromFilename(const String& fname) { return -1; } +bool isCacheFile(const String& fname) { + return fname.indexOf(F("cache_")) != -1; +} + // Look into the filesystem to see if there are any cache files present on the filesystem // Return true if any found. bool getCacheFileCounters(uint16_t& lowest, uint16_t& highest, size_t& filesizeHighest) { diff --git a/src/src/Helpers/ESPEasy_Storage.h b/src/src/Helpers/ESPEasy_Storage.h index 81b7aca43..9ab80afcb 100644 --- a/src/src/Helpers/ESPEasy_Storage.h +++ b/src/src/Helpers/ESPEasy_Storage.h @@ -300,6 +300,8 @@ bool SpiffsFull(); \*********************************************************************************************/ String createCacheFilename(unsigned int count); +bool isCacheFile(const String& fname); + // Match string with an integer between '_' and ".bin" int getCacheFileCountFromFilename(const String& fname); diff --git a/src/src/Helpers/Rules_calculate.cpp b/src/src/Helpers/Rules_calculate.cpp index 8e06e5934..c7342d571 100644 --- a/src/src/Helpers/Rules_calculate.cpp +++ b/src/src/Helpers/Rules_calculate.cpp @@ -2,6 +2,7 @@ #include +#include "../DataStructs/TimingStats.h" #include "../ESPEasyCore/ESPEasy_Log.h" #include "../Globals/RamTracker.h" #include "../Helpers/ESPEasy_math.h" @@ -649,6 +650,7 @@ int CalculateParam(const String& TmpStr) { CalculateReturnCode Calculate(const String& input, double & result) { + START_TIMER; CalculateReturnCode returnCode = RulesCalculate.doCalculate( RulesCalculate_t::preProces(input).c_str(), &result); @@ -690,5 +692,6 @@ CalculateReturnCode Calculate(const String& input, addLogMove(LOG_LEVEL_ERROR, log); } } + STOP_TIMER(COMPUTE_STATS); return returnCode; }