[Log] Fix half printed loglines + memory management + other cleanups

- Events in EventQueue will be stored in 2nd heap/PSRAM when available
- LogEntries that need heap alloc will be stored in 2nd heap/PSRAM when available
- LogConsumer will also be marked as active read when checks for new log lines
- Log expiration will remove less important logs earlier when running low on memory
- Add formatting functions padToMinimumLength and prefixToMinimumLength (+ added comments where to use them later)
- Fix JSON outputting correct JSON when key-value is formatted JSON string
- Fix sending no more than available bytes to serial port for printing logs (which may block writing, thus speed optimization)
- Update CPU-load when it is queried (minimum interval 1000 msec)
- Make presentation of info more consistent when to use pre-formatted text (monospaced)
- Exclude code which will never be used on the rootpage

src/src/DataStructs/EventQueue.cpp

@@ -10,7 +10,7 @@ void EventQueueStruct::add(const String& event, bool deduplicate)
   if (!deduplicate || !isDuplicate(event)) {
 #if defined(USE_SECOND_HEAP) || defined(ESP32)
     String tmp;
-    reserve_special(tmp, event.length());
+    reserve_special(tmp, event.length(), 0);
     tmp = event;
 
 #ifdef USE_SECOND_HEAP
@@ -29,7 +29,7 @@ void EventQueueStruct::add(const String& event, bool deduplicate)
 void EventQueueStruct::add(const __FlashStringHelper *event, bool deduplicate)
 {
   String str;
-  move_special(str, String(event));
+  move_special(str, String(event), 0);
   add(str, deduplicate);
 }
 
@@ -51,7 +51,7 @@ void EventQueueStruct::addMove(String&& event, bool deduplicate)
   if (!deduplicate || !isDuplicate(event)) {
     #if defined(USE_SECOND_HEAP) || defined(ESP32)
     String tmp;
-    move_special(tmp, std::move(event));
+    move_special(tmp, std::move(event), 0);
 
     #ifdef USE_SECOND_HEAP
     // Do not add to the list while on 2nd heap

src/src/DataStructs/LogBuffer.cpp

@@ -55,8 +55,9 @@ bool LogBuffer::getNext(LogDestination logDestination, uint32_t& timestamp, Stri
 bool LogBuffer::hasMessages(LogDestination logDestination)
 {
   if (logDestination >= NR_LOG_TO_DESTINATIONS) { return false; }
+  lastReadTimeStamp[logDestination] = millis(); // Reset if we aren't going to fetch a next message
 
-  clearExpiredEntries(); // Cleanup the old stuff first
+  clearExpiredEntries();                        // Cleanup the old stuff first
 
   auto pos = cache_iterator_pos[logDestination];
 
@@ -67,7 +68,6 @@ bool LogBuffer::hasMessages(LogDestination logDestination)
     }
   }
 
-  lastReadTimeStamp[logDestination] = millis(); // Reset if we aren't going to fetch a next message
   return false;
 }
 
@@ -92,11 +92,13 @@ void LogBuffer::clearExpiredEntries() {
   if (clearExpiredEntriesRunning) { return; }
   clearExpiredEntriesRunning = true;
 
+  const auto freeMem = FreeMem();
+
   for (auto it = LogEntries.begin(); it != LogEntries.end();)
   {
-    it->updateSubscribers();
+    it->setSubscribers();
 
-    if (it->isExpired()) {
+    if (it->hasExpired(freeMem)) {
       auto next = LogEntries.erase(it);
 
       for (size_t i = 0; i < NR_LOG_TO_DESTINATIONS; ++i) {

src/src/DataStructs/LogBuffer.h

@@ -30,7 +30,7 @@
 #ifdef ESP32
   # define LOG_BUFFER_ACTIVE_READ_TIMEOUT 30000
 #else
-  # define LOG_BUFFER_ACTIVE_READ_TIMEOUT 5000
+  # define LOG_BUFFER_ACTIVE_READ_TIMEOUT LOG_BUFFER_EXPIRE
 #endif // ifdef ESP32
 
 typedef std::list<LogEntry_t> LogEntry_queue;

src/src/DataStructs/LogEntry.cpp

@@ -25,7 +25,7 @@ LogEntry_t::LogEntry_t(const uint8_t logLevel,
   _subscriberPendingRead(0)
 {
   if (_strLength && loglevelActiveFor(logLevel)) {
-    _message = special_calloc(1, _strLength + 1);
+    _message = special_calloc(1, _strLength + 1, false);
 
     if (_message) {
       memcpy_P(_message, message, _strLength);
@@ -44,7 +44,7 @@ LogEntry_t::LogEntry_t(const uint8_t logLevel,
 {
   if (_strLength && loglevelActiveFor(logLevel)) {
 
-    _message = special_calloc(1, _strLength + 1);
+    _message = special_calloc(1, _strLength + 1, false);
 
     if (_message) {
       memcpy_P(_message, message.begin(), _strLength);
@@ -66,7 +66,7 @@ LogEntry_t::LogEntry_t(const uint8_t logLevel,
   String str = std::move(message);
 
   if (_strLength && loglevelActiveFor(logLevel)) {
-    _message = special_calloc(1, _strLength + 1);
+    _message = special_calloc(1, _strLength + 1, false);
 
     if (_message) {
       memcpy_P(_message, str.begin(), _strLength);
@@ -128,23 +128,6 @@ void LogEntry_t::setSubscribers()
   }
 }
 
-void LogEntry_t::updateSubscribers()
-{
-  if (isValid()) {
-    for (uint32_t i = 0; _subscriberPendingRead && i < NR_LOG_TO_DESTINATIONS; ++i) {
-      if (bitRead(_subscriberPendingRead, i)) {
-        if (!loglevelActiveFor(static_cast<LogDestination>(i), _logLevel)) {
-          bitClear(_subscriberPendingRead, i);
-        }
-      }
-    }
-
-    if (_subscriberPendingRead == 0) {
-      clear();
-    }
-  }
-}
-
 void LogEntry_t::markReadBySubscriber(uint8_t subscriber)
 {
   if (subscriber < NR_LOG_TO_DESTINATIONS) {
@@ -160,7 +143,7 @@ bool LogEntry_t::validForSubscriber(uint8_t subscriber) const
 {
   return isValid() && (subscriber < NR_LOG_TO_DESTINATIONS) && bitRead(_subscriberPendingRead, subscriber);
 }
-
+/*
 size_t LogEntry_t::print(Print& out, size_t offset, size_t length) const
 {
   if (offset > _strLength) { return 0; }
@@ -181,7 +164,7 @@ size_t LogEntry_t::print(Print& out, size_t offset, size_t length) const
   }
   return pos - begin - offset;
 }
-
+*/
 String LogEntry_t::getMessage() const
 {
   if (_message == nullptr) { return EMPTY_STRING; }
@@ -192,11 +175,39 @@ String LogEntry_t::getMessage() const
   return String((const char *)_message);
 }
 
-bool LogEntry_t::isExpired() const
+bool LogEntry_t::hasExpired(size_t freeMem) const
 {
-  return
-    !isValid() ||
-    timePassedSince(_timestamp) >= LOG_BUFFER_EXPIRE;
+  auto age = timePassedSince(_timestamp);
+
+  if (!isValid()) {
+    return true; 
+  }
+
+  if (freeMem < LOGENTRY_FREEMEM_THRESHOLD) {
+    // We need to free up memory, so try to throw away log entries
+    // which are less important (debug logs) and/or take up more memory
+    switch (_logLevel)
+    {
+      case LOG_LEVEL_ERROR: break;
+      case LOG_LEVEL_INFO:
+      {
+        // Keep non-flash string logs for half the expire duration
+        if (!_isFlashString) {
+          age *= 2;
+        }
+        break;
+      }
+      default:
+      {
+        // Keep non-flash string logs for 1/4th of the expire duration
+        // flash string logs for half the expire duration
+        const auto ageFactor = _isFlashString ? 2 : 4;
+        age *= ageFactor;
+        break;
+      }
+    }
+  }
+  return age >= LOG_BUFFER_EXPIRE;
 }
 
 bool LogEntry_t::isValid() const

src/src/DataStructs/LogEntry.h

@@ -6,9 +6,12 @@
 
 
 #ifdef ESP32
-  # define LOG_BUFFER_EXPIRE         30000 // Time after which a buffered log item is considered expired.
-#else
-  # define LOG_BUFFER_EXPIRE         5000  // Time after which a buffered log item is considered expired.
+  # define LOGENTRY_FREEMEM_THRESHOLD   20000
+  # define LOG_BUFFER_EXPIRE            30000 // Time after which a buffered log item is considered expired.
+#endif // ifdef ESP32
+#ifdef ESP8266
+  # define LOGENTRY_FREEMEM_THRESHOLD   5000
+  # define LOG_BUFFER_EXPIRE            10000 // Time after which a buffered log item is considered expired.
 #endif // ifdef ESP32
 
 
@@ -45,20 +48,17 @@ struct LogEntry_t {
 
   void   setSubscribers();
 
-  // This may clear pending reads when a log subscriber is no longer actively reading
-  void   updateSubscribers();
-
   void   markReadBySubscriber(uint8_t subscriber);
 
   bool   validForSubscriber(uint8_t subscriber) const;
-
+/*
   size_t print(Print& out,
                size_t offset = 0,
                size_t length = std::numeric_limits<size_t>::max()) const;
-
+*/
   String   getMessage() const;
 
-  bool     isExpired() const;
+  bool     hasExpired(size_t freeMem) const;
 
   bool     isValid() const;
 

src/src/DataStructs/NodeStruct.cpp

@@ -84,8 +84,8 @@ bool NodeStruct::validate(const IPAddress& remoteIP) {
 }
 
 bool NodeStruct::operator<(const NodeStruct &other) const {
-  const bool thisExpired = isExpired();
-  if (thisExpired != other.isExpired()) {
+  const bool thisExpired = hasExpired();
+  if (thisExpired != other.hasExpired()) {
     return !thisExpired;
   }
 
@@ -106,7 +106,7 @@ bool NodeStruct::operator<(const NodeStruct &other) const {
   int score_other = other.getLoad();
 
   if (distance != other.distance) {
-    if (!isExpired() && !other.isExpired()) {
+    if (!hasExpired() && !other.hasExpired()) {
       // Distance is not the same, so take distance into account.
       return distance < other.distance;
 /*
@@ -207,7 +207,7 @@ uint32_t NodeStruct::getAge() const {
   return timePassedSince(lastUpdated);
 }
 
-bool  NodeStruct::isExpired() const {
+bool  NodeStruct::hasExpired() const {
   return getAge() > NODE_STRUCT_AGE_TIMEOUT;
 }
 

src/src/DataStructs/NodeStruct.h

@@ -48,7 +48,7 @@ struct __attribute__((__packed__)) NodeStruct
 
   uint32_t getAge() const;
 
-  bool          isExpired() const;
+  bool          hasExpired() const;
 
   float         getLoad() const;
 

src/src/DataStructs/NodesHandler.cpp

@@ -473,7 +473,7 @@ void NodesHandler::updateThisNode() {
     const NodeStruct *preferred = getPreferredNode_notMatching(thisNode.sta_mac);
 
     if (preferred != nullptr) {
-      if (!preferred->isExpired()) {
+      if (!preferred->hasExpired()) {
         // Only take the distance of another node if it is running a build which does not send out traceroute
         // If it is a build sending traceroute, only consider having a distance if you know how to reach the gateway node
         // This does impose an issue when a gateway node is running an older version, as the next hops never will have a traceroute too.

src/src/ESPEasyCore/ESPEasy_Console_Port.cpp

@@ -253,10 +253,10 @@ bool EspEasy_Console_Port::process_serialWriteBuffer()
 
       return totalWritten > 0;
     }
-
+    
     const size_t chunkSize = availableForWrite > CONSOLE_MAX_WRITE_CHUNKSIZE
-? CONSOLE_MAX_WRITE_CHUNKSIZE
-: availableForWrite;
+      ? CONSOLE_MAX_WRITE_CHUNKSIZE
+      : availableForWrite;
 
     availableForWrite -= chunkSize;
 

src/src/Globals/Plugins.cpp

@@ -745,19 +745,19 @@ bool PluginCall(uint8_t Function, struct EventStruct *event, String& str)
               if (taskIndex < 9) { log += ' '; }
               log += taskIndex + 1;
 
-              while (log.length() < 30) { log += ' '; }
+              padToMinimumLength(log, 30);
               log += F("Free mem after: ");
               log += freemem_end;
 
-              while (log.length() < 53) { log += ' '; }
+              padToMinimumLength(log, 53);
               log += F("plugin: ");
               log += freemem_begin - freemem_end;
 
-              while (log.length() < 67) { log += ' '; }
+              padToMinimumLength(log, 67);
 
               log += Settings.TaskDeviceEnabled[taskIndex] ? F("[ena]") : F("[dis]");
 
-              while (log.length() < 73) { log += ' '; }
+              padToMinimumLength(log, 73);
               log += getPluginNameFromDeviceIndex(getDeviceIndex_from_TaskIndex(taskIndex));
 
               addLogMove(LOG_LEVEL_DEBUG, log);

src/src/Helpers/KeyValueWriter_JSON.cpp

@@ -197,11 +197,7 @@ void KeyValueWriter_JSON::writeValue(const ValueStruct& val, bool forceString)
     case ValueStruct::ValueType::String:
     case ValueStruct::ValueType::FlashString:
       if (forceString) {
-        String tmp(to_json_value(str));
-        if (!isWrappedWithQuotes(tmp)) {
-          tmp = wrap_String(tmp, '"');
-        }
-        pr.print(tmp);
+        pr.print(to_json_value(str, true));
         return;
       }
       break;

src/src/Helpers/LogStreamWriter.cpp

@@ -24,7 +24,8 @@ bool LogStreamWriter::process() { return false; }
 
 bool LogStreamWriter::hasMessages() const
 {
-  return Logging.hasMessages(_log_destination);
+  // _timestamp != 0 means there is still a logentry not finished.
+  return Logging.hasMessages(_log_destination) || _timestamp != 0;
 }
 
 size_t LogStreamWriter::write(Print& stream, size_t nrBytesToWrite)
@@ -117,7 +118,8 @@ size_t LogStreamWriter::write_part(String& str, Print& stream, size_t  nrBytesTo
 
   if (_readpos < str.length()) {
     const size_t bytesLeft = str.length() - _readpos;
-    bytesWritten = stream.write(&str[_readpos], bytesLeft);
+
+    bytesWritten = stream.write(&str[_readpos], std::min(bytesLeft, nrBytesToWrite));
     _readpos    += bytesWritten;
   }
 

src/src/Helpers/Memory.cpp

@@ -172,7 +172,12 @@ void* special_realloc(void *ptr, size_t size) {
   return res;
 }
 
-void* special_calloc(size_t num, size_t size) {
+void *special_calloc(size_t num, size_t size)
+{
+  return special_calloc(num, size, true);
+}
+
+void* special_calloc(size_t num, size_t size, bool preferHeap) {
   void *res = nullptr;
 
 #ifdef ESP32
@@ -182,7 +187,7 @@ void* special_calloc(size_t num, size_t size) {
   }
 #else // ifdef ESP32
 # ifdef USE_SECOND_HEAP
-  if (size > 64 || FreeMem() < 5000) {
+  if (!preferHeap || size > 64 || FreeMem() < 5000) {
 
     // Try allocating on ESP8266 2nd heap, only when sufficiently large data is needed
     HeapSelectIram ephemeral;
@@ -212,12 +217,17 @@ void* special_calloc(size_t num, size_t size) {
 
 #ifdef ESP8266
 bool String_reserve_special(String& str, size_t size) {
+  return String_reserve_special(str, size, 48);
+}
+
+bool String_reserve_special(String& str, size_t size, size_t strLengthThreshold) {
+
   if (str.length() >= size) {
     // Nothing needs to be done
     return true;
   }
   #ifdef USE_SECOND_HEAP
-  if (size >= 48 || FreeMem() < 5000) {
+  if (size >= strLengthThreshold || FreeMem() < 5000) {
     // Only try to store larger strings here as those tend to be kept for a longer period.
     HeapSelectIram ephemeral;
     // String does round up to nearest multiple of 16 bytes, so no need to round up to multiples of 32 bit here
@@ -256,8 +266,11 @@ class PSRAM_String : public String {
   }
 };
 
-
 bool String_reserve_special(String& str, size_t size) {
+  return String_reserve_special(str, size, 48);
+}
+
+bool String_reserve_special(String& str, size_t size, size_t strLengthThreshold) {
   if (size == 0) {
     return true;
   }

src/src/Helpers/Memory.h

@@ -56,10 +56,12 @@ uint32_t getMaxFreeBlock();
 void *special_malloc(uint32_t size);
 void *special_realloc(void *ptr, size_t size);
 void *special_calloc(size_t num, size_t size);
+void *special_calloc(size_t num, size_t size, bool preferHeap);
 
 
 // Perform a special memory allocate for the buffer pointer in the String object
 // Allocation is tried either on 2nd heap (ESP8266) or PSRAM (ESP32)
 // when possible.
 bool String_reserve_special(String& str, size_t size);
+bool String_reserve_special(String& str, size_t size, size_t strLengthThreshold);
 

src/src/Helpers/Misc.cpp

@@ -635,13 +635,11 @@ void logMemUsageAfter(const __FlashStringHelper *function, int value) {
         log += value;
       }
 
-      while (log.length() < 30) { log += ' ';
-      }
+      padToMinimumLength(log, 30);
       log += F("Free mem after: ");
       log += freemem_end;
 
-      while (log.length() < 55) { log += ' ';
-      }
+      padToMinimumLength(log, 55);
       log += F("diff: ");
       log += last_freemem - freemem_end;
       addLogMove(LOG_LEVEL_DEBUG, log);

src/src/Helpers/Scheduler.cpp

@@ -107,7 +107,7 @@ void ESPEasy_Scheduler::updateIdleTimeStats() {
   msecTimerHandler.updateIdleTimeStats();
 }
 
-float ESPEasy_Scheduler::getIdleTimePct() const {
+float ESPEasy_Scheduler::getIdleTimePct()  {
   return msecTimerHandler.getIdleTimePct();
 }
 

src/src/Helpers/Scheduler.h

@@ -316,7 +316,7 @@ void                 setPluginTaskTimer(unsigned long msecFromNow,
 
   void   updateIdleTimeStats();
 
-  float  getIdleTimePct() const;
+  float  getIdleTimePct() ;
 
   void   setEcoMode(bool enabled);
 

src/src/Helpers/SerialWriteBuffer.cpp

@@ -5,8 +5,8 @@
 #include "../Globals/Settings.h"
 
 
-String SerialWriteBuffer_t::colorize(const String& str) const {
 #if FEATURE_COLORIZE_CONSOLE_LOGS
+String SerialWriteBuffer_t::colorize(const String& str) const {
   if (!Settings.ColorizeSerialLog()) return str;
   const __FlashStringHelper *format = F("%s");
 
@@ -31,10 +31,8 @@ String SerialWriteBuffer_t::colorize(const String& str) const {
 
   }
   return strformat(format, str.c_str());
-#else // if FEATURE_COLORIZE_CONSOLE_LOGS
-  return str;
-#endif // if FEATURE_COLORIZE_CONSOLE_LOGS
 }
+#endif // if FEATURE_COLORIZE_CONSOLE_LOGS
 
 
 size_t SerialWriteBuffer_t::write_skipping(Print& stream)
@@ -49,7 +47,11 @@ void SerialWriteBuffer_t::prepare_prefix()
   _prefix = format_msec_duration(_timestamp);
 
   if (_loglevel == LOG_LEVEL_NONE) {
+#if FEATURE_COLORIZE_CONSOLE_LOGS
     _prefix += colorize(F(" : >  "));
+#else
+    _prefix += F(" : >  ");
+#endif
   } else {
       #ifndef LIMIT_BUILD_SIZE
     _prefix += strformat(F(" : (%d) "), FreeMem());
@@ -59,10 +61,12 @@ void SerialWriteBuffer_t::prepare_prefix()
     {
       String loglevelDisplayString = getLogLevelDisplayString(_loglevel);
 
-      while (loglevelDisplayString.length() < LOG_LEVEL_MAX_STRING_LENGTH) {
-        loglevelDisplayString += ' ';
-      }
+      padToMinimumLength(loglevelDisplayString, LOG_LEVEL_MAX_STRING_LENGTH);
+#if FEATURE_COLORIZE_CONSOLE_LOGS
       _prefix += colorize(loglevelDisplayString);
+#else
+      _prefix += loglevelDisplayString;
+#endif
     }
     _prefix += F(" | ");
   }

src/src/Helpers/SerialWriteBuffer.h

@@ -19,8 +19,9 @@ private:
   void prepare_prefix() override;
 
 
-
+#if FEATURE_COLORIZE_CONSOLE_LOGS
   String colorize(const String& str) const;
+#endif
 
 };
 

src/src/Helpers/StringConverter.cpp

@@ -85,8 +85,12 @@ bool equals(const String& str, const char& c) {
 }
 
 void move_special(String& dest, String&& source) {
+  move_special(dest, std::move(source), 64);
+}
+
+void move_special(String& dest, String&& source, size_t strLengthThreshold) {
   // Only try to store larger strings here as those tend to be kept for a longer period.
-  if ((source.length() >= 64) 
+  if ((source.length() >= strLengthThreshold) 
 #ifdef USE_SECOND_HEAP
       && mmu_is_dram(&(source[0]))
 #endif
@@ -106,8 +110,12 @@ void move_special(String& dest, String&& source) {
 }
 
 String move_special(String&& source) {
+  return move_special(std::move(source), 64);
+}
+
+String move_special(String&& source, size_t strLengthThreshold) {
   String dest;
-  move_special(dest, std::move(source));
+  move_special(dest, std::move(source), strLengthThreshold);
   return dest;
 }
 
@@ -116,6 +124,11 @@ bool reserve_special(String& str, size_t size) {
   return String_reserve_special(str, size);
 }
 
+bool reserve_special(String& str, size_t size, size_t strLengthThreshold)
+{
+  return String_reserve_special(str, size, strLengthThreshold);
+}
+
 void free_string(String& str) {
   // This is a call specifically tailored to what is done in:
   //  void String::move(String &rhs)
@@ -312,6 +325,22 @@ void replaceUnicodeByChar(String& line, char replChar) {
   }
 }
 
+void padToMinimumLength(String& line, uint32_t minimumLength, char padChar)
+{
+  if (line.length() >= minimumLength) return;
+  line.reserve(minimumLength);
+  while (line.length() < minimumLength) line += padChar;
+}
+
+void prefixToMinimumLength(String& line, uint32_t minimumLength, char prefixChar)
+{
+  if (line.length() >= minimumLength) return;
+ 
+  String prefix;
+  padToMinimumLength(prefix, minimumLength - line.length(), prefixChar);
+  line = prefix + line;
+}
+
 /*********************************************************************************************\
    Format a value to the set number of decimals
 \*********************************************************************************************/
@@ -474,21 +503,15 @@ String get_formatted_Plugin_number(pluginID_t pluginID)
 
 String formatIntLeadingZeroes(int value, int nrDigits)
 {
-  const String fmt = strformat(F("%%0%dd"), nrDigits);
-  return strformat(fmt, value);
-//  return formatIntLeadingZeroes(String(value), nrDigits);
+  String res(value);
+  prefixToMinimumLength(res, nrDigits, '0');
+  return res;
 }
 
 String formatIntLeadingZeroes(const String& value, int nrDigits)
 {
-  String res;
-  res.reserve(nrDigits);
-  int nrZeroes = nrDigits - value.length();
-  while (nrZeroes > 0) {
-    --nrZeroes;
-    res += '0';
-  }
-  res += value;
+  String res(value);
+  prefixToMinimumLength(res, nrDigits, '0');
   return res;
 }
 
@@ -548,19 +571,8 @@ String to_json_value(const String& value, bool wrapInQuotes) {
     // Empty string
     return F("\"\"");
   }
-  const size_t val_length = value.length();
-
-  if (val_length > 2) {
-    // Check for JSON objects or arrays
-    const char firstchar = value[0];
-    const char lastchar = value[val_length - 1];
-    if ((firstchar == '[' && lastchar == ']') ||
-        (firstchar == '{' && lastchar == '}')) 
-    {
-      return value;
-    }
-  }
-
+  
+  if (is_json_formatted(value)) return value;
 
   if (wrapInQuotes || mustConsiderAsJSONString(value)) {
     // Is not a numerical value, or BIN/HEX notation, thus wrap with quotes
@@ -571,7 +583,8 @@ String to_json_value(const String& value, bool wrapInQuotes) {
     const bool backslash_or_doubleQuote_found = 
       value.indexOf('\\') != -1 ||
       value.indexOf('"') != -1;
-    
+  
+    const size_t val_length = value.length();
     for (size_t i = 0; i < val_length; ++i) {
       const char c = value[i];
       // Special characters not allowed in JSON:
@@ -604,6 +617,22 @@ String to_json_value(const String& value, bool wrapInQuotes) {
 }
 
 
+bool is_json_formatted(const String& value)
+{
+  const size_t val_length = value.length();
+  if (val_length > 2) {
+    // Check for JSON objects or arrays
+    const char firstchar = value[0];
+    const char lastchar = value[val_length - 1];
+    if ((firstchar == '[' && lastchar == ']') ||
+        (firstchar == '{' && lastchar == '}')) 
+    {
+      return true;
+    }
+  }
+  return false;
+}
+
 /*********************************************************************************************\
    Strip wrapping chars (e.g. quotes)
 \*********************************************************************************************/

src/src/Helpers/StringConverter.h

@@ -59,8 +59,13 @@ bool equals(const String& str, const char& c);
 void move_special(String& dest, String&& source);
 String move_special(String&& source);
 
+void move_special(String& dest, String&& source, size_t strLengthThreshold);
+String move_special(String&& source, size_t strLengthThreshold);
+
+
 // Try to reserve on the heap with the most space available
 bool reserve_special(String& str, size_t size);
+bool reserve_special(String& str, size_t size, size_t strLengthThreshold);
 
 // Arduino String does not have a function to de-allocate its internal buffer
 // This is a special trick to de-allocate its internal buffer and thus free up memory.
@@ -114,6 +119,13 @@ size_t UTF8_charLength(uint8_t firstByte);
 
 void   replaceUnicodeByChar(String& line, char replChar);
 
+// Pad string with given char to minimum length
+void   padToMinimumLength(String& line, uint32_t minimumLength, char padChar = ' ');
+
+// Prefix string with given char to minimum length
+void   prefixToMinimumLength(String& line, uint32_t minimumLength, char prefixChar = ' ');
+
+
 /*********************************************************************************************\
    Format a value to the set number of decimals
 \*********************************************************************************************/
@@ -178,6 +190,8 @@ String wrapWithQuotesIfContainsParameterSeparatorChar(const String& text);
 String to_json_value(const String& value,
                      bool wrapInQuotes = false);
 
+bool is_json_formatted(const String& value);
+
 /*********************************************************************************************\
    Strip wrapping chars (e.g. quotes)
 \*********************************************************************************************/

src/src/Helpers/StringParser.cpp

@@ -522,9 +522,7 @@ String parseTemplate_padded(String& tmpString, uint8_t minimal_lineSize, bool us
   parse_string_commands(newString);
 
   // padding spaces
-  while (newString.length() < minimal_lineSize) {
-    newString += ' ';
-  }
+  padToMinimumLength(newString, minimal_lineSize);
 
   STOP_TIMER(PARSE_TEMPLATE_PADDED);
   #ifndef BUILD_NO_RAM_TRACKER
@@ -800,6 +798,7 @@ void transformValue(
 
               if (valueJustLength > 1)
               {
+                // TODO TD-er: Check if we can use prefixToMinimumLength here
                 if (isDigit(valueJust[1]))                          // Check Pn where n is between 0 and 9
                 {
                   int filler = valueJust[1] - value.length() - '0'; // char '0' = 48; char '9' = 58
@@ -814,6 +813,7 @@ void transformValue(
 
               if (valueJustLength > 1)
               {
+                // TODO TD-er: Check if we can use padToMinimumLength here
                 if (isDigit(valueJust[1]))                          // Check Sn where n is between 0 and 9
                 {
                   int filler = valueJust[1] - value.length() - '0'; // 48
@@ -888,6 +888,7 @@ void transformValue(
 
       if (rightJustify)
       {
+        // TODO TD-er: Check if we can use prefixToMinimumLength here
         int filler = lineSize - newString.length() - value.length() - tmpString.length();
 
         for (uint8_t f = 0; f < filler; f++) {

src/src/Helpers/StringProvider.cpp

@@ -600,13 +600,13 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
 #endif // if FEATURE_USE_IPV6
     case LabelType::IP_ADDRESS:
     {
-      KeyValueStruct kv(F("IP Address"), formatIP(ESPEasy::net::NetworkLocalIP()));
+      KeyValueStruct kv(F("IP Address"), formatIP(ESPEasy::net::NetworkLocalIP()), KeyValueStruct::Format::PreFormatted);
       KV_SETID(F("ip"));
       return kv;
     }
     case LabelType::IP_SUBNET:
     {
-      KeyValueStruct kv(F("IP Subnet"), formatIP(ESPEasy::net::NetworkSubnetMask()));
+      KeyValueStruct kv(F("IP Subnet"), formatIP(ESPEasy::net::NetworkSubnetMask()), KeyValueStruct::Format::PreFormatted);
       KV_SETID(F("subnet"));
       return kv;
     }
@@ -619,13 +619,13 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
     }
     case LabelType::GATEWAY:
     {
-      KeyValueStruct kv(F("Gateway"), formatIP(ESPEasy::net::NetworkGatewayIP()));
+      KeyValueStruct kv(F("Gateway"), formatIP(ESPEasy::net::NetworkGatewayIP()), KeyValueStruct::Format::PreFormatted);
       KV_SETID(F("gw"));
       return kv;
     }
     case LabelType::CLIENT_IP:
     {
-      return KeyValueStruct(F("Client IP"), formatIP(web_server.client().remoteIP(), true));
+      return KeyValueStruct(F("Client IP"), formatIP(web_server.client().remoteIP(), true), KeyValueStruct::Format::PreFormatted);
     }
     #if FEATURE_MDNS
     case LabelType::M_DNS:
@@ -679,38 +679,38 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
     case LabelType::DNS_1:
     {
       if (extendedValue) { break; }
-      KeyValueStruct kv(F("DNS 1"), formatIP(ESPEasy::net::NetworkDnsIP(0)));
+      KeyValueStruct kv(F("DNS 1"), formatIP(ESPEasy::net::NetworkDnsIP(0)), KeyValueStruct::Format::PreFormatted);
       KV_SETID(F("dns1"));
       return kv;
     }
     case LabelType::DNS_2:
     {
       if (extendedValue) { break; }
-      KeyValueStruct kv(F("DNS 2"), formatIP(ESPEasy::net::NetworkDnsIP(1)));
+      KeyValueStruct kv(F("DNS 2"), formatIP(ESPEasy::net::NetworkDnsIP(1)), KeyValueStruct::Format::PreFormatted);
       KV_SETID(F("dns2"));
       return kv;
     }
     case LabelType::ALLOWED_IP_RANGE:
     {
-      KeyValueStruct kv(F("Allowed IP Range"), ESPEasy::net::describeAllowedIPrange());
+      KeyValueStruct kv(F("Allowed IP Range"), ESPEasy::net::describeAllowedIPrange(), KeyValueStruct::Format::PreFormatted);
       KV_SETID(F("allowed_range"));
       return kv;
     }
     case LabelType::STA_MAC:
     {
-      return KeyValueStruct(F("STA MAC"), ESPEasy::net::WifiSTAmacAddress().toString());
+      return KeyValueStruct(F("STA MAC"), ESPEasy::net::WifiSTAmacAddress().toString(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::AP_MAC:
     {
-      return KeyValueStruct(F("AP MAC"), ESPEasy::net::WifiSoftAPmacAddress().toString());
+      return KeyValueStruct(F("AP MAC"), ESPEasy::net::WifiSoftAPmacAddress().toString(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::SSID:
     {
-      return KeyValueStruct(F("SSID"), WiFi.SSID());
+      return KeyValueStruct(F("SSID"), WiFi.SSID(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::BSSID:
     {
-      return KeyValueStruct(F("BSSID"), WiFi.BSSIDstr());
+      return KeyValueStruct(F("BSSID"), WiFi.BSSIDstr(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::CHANNEL:
     {
@@ -724,11 +724,11 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
     }
     case LabelType::CONNECTED:
     {
-      return KeyValueStruct(F("Connected"), format_msec_duration(ESPEasy::net::NetworkConnectDuration_ms()));
+      return KeyValueStruct(F("Connected"), format_msec_duration(ESPEasy::net::NetworkConnectDuration_ms()), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::CONNECTED_MSEC:
     {
-      return KeyValueStruct(F("Connected msec"), ESPEasy::net::NetworkConnectDuration_ms());
+      return KeyValueStruct(F("Connected msec"), ESPEasy::net::NetworkConnectDuration_ms(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::LAST_DISCONNECT_REASON:
     {
@@ -744,11 +744,11 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
     }
     case LabelType::WIFI_STORED_SSID1:
     {
-      return KeyValueStruct(F("Configured SSID1"), SecuritySettings.WifiSSID);
+      return KeyValueStruct(F("Configured SSID1"), SecuritySettings.WifiSSID, KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::WIFI_STORED_SSID2:
     {
-      return KeyValueStruct(F("Configured SSID2"), SecuritySettings.WifiSSID2);
+      return KeyValueStruct(F("Configured SSID2"), SecuritySettings.WifiSSID2, KeyValueStruct::Format::PreFormatted);
     }
 
 
@@ -815,7 +815,7 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
         descr += ' ';
         descr += F(BUILD_NOTES);
       }
-      return KeyValueStruct(F("Build"), descr);
+      return KeyValueStruct(F("Build"), descr, KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::BUILD_ORIGIN:
     {
@@ -826,20 +826,23 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
       String res(F(BUILD_GIT));
 
       if (res.isEmpty()) { res = get_git_head(); }
-      return KeyValueStruct(F("Git Build"), res);
+      return KeyValueStruct(F("Git Build"), res, KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::SYSTEM_LIBRARIES:
     {
-      return KeyValueStruct(F("System Libraries"), getSystemLibraryString());
+      return KeyValueStruct(F("System Libraries"), getSystemLibraryString(), KeyValueStruct::Format::PreFormatted);
     }
 #ifdef ESP32
     case LabelType::ESP_IDF_SDK_VERSION:
     {
-      return KeyValueStruct(F("ESP-IDF Version"), strformat(
-                              F("%d.%d.%d"),
-                              ESP_IDF_VERSION_MAJOR,
-                              ESP_IDF_VERSION_MINOR,
-                              ESP_IDF_VERSION_PATCH));
+      return KeyValueStruct(
+        F("ESP-IDF Version"), 
+        strformat(
+          F("%d.%d.%d"),
+          ESP_IDF_VERSION_MAJOR,
+          ESP_IDF_VERSION_MINOR,
+          ESP_IDF_VERSION_PATCH), 
+        KeyValueStruct::Format::PreFormatted);
     }
 #endif // ifdef ESP32
     case LabelType::PLUGIN_COUNT:
@@ -848,28 +851,28 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
     }
     case LabelType::PLUGIN_DESCRIPTION:
     {
-      return KeyValueStruct(F("Plugin Description"), getPluginDescriptionString());
+      return KeyValueStruct(F("Plugin Description"), getPluginDescriptionString(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::BUILD_TIME:
     {
-      return KeyValueStruct(F("Build Time"), String(get_build_date()) + ' ' + get_build_time());
+      return KeyValueStruct(F("Build Time"), String(get_build_date()) + ' ' + get_build_time(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::BINARY_FILENAME:
     {
-      return KeyValueStruct(F("Binary Filename"), get_binary_filename());
+      return KeyValueStruct(F("Binary Filename"), get_binary_filename(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::BUILD_PLATFORM:
     {
-      return KeyValueStruct(F("Build Platform"), get_build_platform());
+      return KeyValueStruct(F("Build Platform"), get_build_platform(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::GIT_HEAD:
     {
-      return KeyValueStruct(F("Git HEAD"), get_git_head());
+      return KeyValueStruct(F("Git HEAD"), get_git_head(), KeyValueStruct::Format::PreFormatted);
     }
     #ifdef CONFIGURATION_CODE
     case LabelType::CONFIGURATION_CODE_LBL:
     {
-      return KeyValueStruct(F("Configuration code"), getConfigurationCode());
+      return KeyValueStruct(F("Configuration code"), getConfigurationCode(), KeyValueStruct::Format::PreFormatted);
     }
     #endif // ifdef CONFIGURATION_CODE
 #if FEATURE_CLEAR_I2C_STUCK
@@ -907,7 +910,7 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
 
     case LabelType::ESP_CHIP_ID:
     {
-      return KeyValueStruct(F("ESP Chip ID"), formatToHex(getChipId(), 6));
+      return KeyValueStruct(F("ESP Chip ID"), formatToHex(getChipId(), 6), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::ESP_CHIP_FREQ:
     {
@@ -937,7 +940,7 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
 #endif // ifdef ESP32
     case LabelType::ESP_CHIP_MODEL:
     {
-      return KeyValueStruct(F("ESP Chip Model"), getChipModel());
+      return KeyValueStruct(F("ESP Chip Model"), getChipModel(), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::ESP_CHIP_REVISION:
     {
@@ -958,7 +961,7 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
       auto flashChipId = getFlashChipId();
 
       if (flashChipId == 0) { break; }
-      return KeyValueStruct(F("Flash Chip ID"), formatToHex(flashChipId, 6));
+      return KeyValueStruct(F("Flash Chip ID"), formatToHex(flashChipId, 6), KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::FLASH_CHIP_VENDOR:
     {
@@ -971,7 +974,7 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
         id += concat(F(" (PUYA"), puyaSupport() ? F(", supported") : F(HTML_SYMBOL_WARNING)) + ')';
       }
 
-      return KeyValueStruct(F("Flash Chip Vendor"), id);
+      return KeyValueStruct(F("Flash Chip Vendor"), id, KeyValueStruct::Format::PreFormatted);
     }
     case LabelType::FLASH_CHIP_MODEL:
     {
@@ -1025,7 +1028,7 @@ KeyValueStruct getKeyValue(LabelType::Enum label, bool extendedValue)
     }
     case LabelType::FLASH_IDE_MODE:
     {
-      KeyValueStruct kv(F("Flash IDE Mode"), getFlashChipMode());
+      KeyValueStruct kv(F("Flash IDE Mode"), getFlashChipMode(), KeyValueStruct::Format::PreFormatted);
       KV_SETID(F("mode"));
       return kv;
     }

src/src/Helpers/msecTimerHandlerStruct.cpp

@@ -99,15 +99,20 @@
 
   void msecTimerHandlerStruct::updateIdleTimeStats() {
     const long duration = timePassedSince(last_log_start_time) * 10;
-    if (duration == 0) return;
-    recordRunning();
+    if (duration <= 1000) {
+      // No need to recompute it over a small interval
+      // Also makes sure duration != 0 as it is used in division
+      return;
+    }
+    recordRunning(); // Make sure currently active 'idle' time is included
     last_log_start_time  = millis();
     idle_time_pct = static_cast<float>(total_idle_time_usec);
     idle_time_pct /= static_cast<float>(duration);
     total_idle_time_usec = 0;
   }
 
-  float msecTimerHandlerStruct::getIdleTimePct() const {
+  float msecTimerHandlerStruct::getIdleTimePct()  {
+    updateIdleTimeStats();
     return idle_time_pct;
   }
 

src/src/Helpers/msecTimerHandlerStruct.h

@@ -32,7 +32,7 @@ struct msecTimerHandlerStruct {
 
   void   updateIdleTimeStats();
 
-  float  getIdleTimePct() const;
+  float  getIdleTimePct() ;
 
 private:
 

src/src/PluginStructs/P023_data_struct.cpp

@@ -319,6 +319,7 @@ void P023_data_struct::setCurrentText(const String& string, int X, int Y) {
     if (currentLines[X].length() >= static_cast<size_t>(Y)) {
       currentLines[X] = currentLines[X].substring(0, Y + 1) + string;
     } else {
+      // TODO TD-er: Check if we can use prefixToMinimumLength here
       for (size_t i = currentLines[X].length(); i < static_cast<size_t>(Y); ++i) {
         currentLines[X] += ' ';
       }

src/src/PluginStructs/P036_data_struct.cpp

@@ -2183,6 +2183,7 @@ void P036_data_struct::P036_DisplayPage(struct EventStruct *event)
               }
 
               if (addSpaces > 0) {
+                // TODO TD-er: Check if we can use prefixToMinimumLength here
                 currentLines[i].reserve(currentLines[i].length() + addSpaces);
 
                 while (addSpaces > 0) {
@@ -2288,10 +2289,13 @@ String P036_data_struct::P36_parseTemplate(String& tmpString, uint8_t lineIdx) {
   }
   # endif // if P036_ENABLE_TICKER
 
+  // FIXME TD-er: It looks like the "Align left" and "Align right" are used here as the exact opposite
+  // Might be functionally correct, but the enum names are then confusing to say the least.
   switch (iTextAlignment) {
     case TEXT_ALIGN_LEFT:
 
       // add leading spaces from tmpString to the result
+      // TODO TD-er: Check if we can use prefixToMinimumLength here
       for (uint16_t l = 0; l < tmpString.length(); ++l) {
         if (tmpString[l] != ' ') {
           break;
@@ -2302,6 +2306,7 @@ String P036_data_struct::P36_parseTemplate(String& tmpString, uint8_t lineIdx) {
     case TEXT_ALIGN_RIGHT:
 
       // add trailing spaces from tmpString to the result
+      // TODO TD-er: Check if we can use padToMinimumLength here
       for (int16_t l = tmpString.length() - 1; l >= 0; --l) {
         if (tmpString[l] != ' ') {
           break;

src/src/PluginStructs/P157_data_struct.cpp

@@ -360,9 +360,7 @@ void P157_data_struct::fillBufferWithString(const String& textToShow,
     buf = buf.substring(0, buf.length() - 1);
   }
 
-  while (buf.length() < bufLen) {
-    buf += ' ';
-  }
+  padToMinimumLength(buf, bufLen);
   memcpy(showbuffer, buf.c_str(), bufLen);
 
   # ifdef P157_DEBUG
@@ -518,6 +516,8 @@ void P157_data_struct::setTextToScroll(const String& text) {
       const int8_t colonCompensation = P157_is7SegColonDisplay(displayModel) ? (displays - 1) : 0;
       _textToScroll.reserve(text.length() + bufLen + (scrollFull ? bufLen : 0));
 
+      // TODO TD-er: Check if we can use padToMinimumLength and prefixToMinimumLength here
+
       for (uint8_t i = 0; scrollFull && i < (bufLen - colonCompensation); ++i) { // Scroll text in from the right, so start with all spaces
         _textToScroll += ' ';
       }
@@ -628,9 +628,7 @@ void P157_data_struct::printBuffer() {
         buf = buf.substring(0, buf.length() - 1);
       }
 
-      while (buf.length() < bufLen) { // TODO Account for offset?
-        buf += ' ';
-      }
+      padToMinimumLength(buf, bufLen); // TODO Account for offset?
 
       # ifdef P157_DEBUG
 

src/src/WebServer/KeyValueWriter_WebForm.cpp

@@ -68,7 +68,7 @@ void KeyValueWriter_WebForm::write(const KeyValueStruct& kv)
   const size_t nrValues   = kv._values.size();
   const bool   format_pre =
     !plain_text &&
-    !summary_value_only &&
+//    !summary_value_only &&
     (nrValues > 1 || kv._format == KeyValueStruct::Format::PreFormatted);
 
   const bool format_note =

src/src/WebServer/RootPage.cpp

@@ -73,9 +73,9 @@ void handle_root() {
     return;
   }
 */
-  if (!MAIN_PAGE_SHOW_BASIC_INFO_NOT_LOGGED_IN) {
+  #if !MAIN_PAGE_SHOW_BASIC_INFO_NOT_LOGGED_IN
     if (!isLoggedIn()) { return; }
-  }
+  #endif
 
   const bool loggedIn = isLoggedIn(false);
 
@@ -228,13 +228,13 @@ void handle_root() {
     html_table_header(F("Node List"));
     html_table_header(F("Name"));
 
-    if (MAIN_PAGE_SHOW_NODE_LIST_BUILD) {
+    #if MAIN_PAGE_SHOW_NODE_LIST_BUILD
       html_table_header(getLabel(LabelType::BUILD_DESC));
-    }
+    #endif
 
-    if (MAIN_PAGE_SHOW_NODE_LIST_TYPE) {
+    #if MAIN_PAGE_SHOW_NODE_LIST_TYPE
       html_table_header(F("Type"));
-    }
+    #endif
     html_table_header(F("IP"), 160); // Should fit "255.255.255.255"
     html_table_header(F("Load (%)"));
     html_table_header(F("Age (s)"));
@@ -271,17 +271,17 @@ void handle_root() {
         }
         html_TD();
 
-        if (MAIN_PAGE_SHOW_NODE_LIST_BUILD) {
+        #if MAIN_PAGE_SHOW_NODE_LIST_BUILD
           if (it->second.build) {
             addHtml(formatSystemBuildNr(it->second.build));
           }
           html_TD();
-        }
+        #endif
 
-        if (MAIN_PAGE_SHOW_NODE_LIST_TYPE) {
+        #if MAIN_PAGE_SHOW_NODE_LIST_TYPE
           addHtml(it->second.getNodeTypeDisplayString());
           html_TD();
-        }
+        #endif
 
         if (it->second.ip[0] != 0
 #  if FEATURE_USE_IPV6