automatically updated release notes for mega-20181007

Move stack to user memory space and close file handles + Add WROVER dev kit debug support

dist/Release_notes.txt

@@ -1,3 +1,36 @@
+-------------------------------------------------
+Changes in release mega-20181007 (since mega-20181006)
+-------------------------------------------------
+
+Release date: Sun Oct  7 04:00:09 CEST 2018
+
+Max (2):
+      Force slash to be converted in htmlEscape
+      Fix XSS-Vulnerabilites on wifi configuration pages
+
+Plebs (3):
+      Add three new TOGGLE commands
+      fixed float to int conversion
+      added "mcpgpiotoggle" command
+
+TD-er (3):
+      [ESP32] Allow Wrover dev kit to be used for PIO debug
+      [Stack] Move Arduino 'cont' stack into user RAM
+      [SPIFFS] Make sure to close open files on SPIFFS
+
+
+-------------------------------------------------
+Changes in release mega-20181006 (since mega-20181004)
+-------------------------------------------------
+
+Release date: Sat Oct  6 04:00:13 CEST 2018
+
+TD-er (3):
+      [Stack] Increase stack to 5k and reduce stack allocations in rules
+      [WiFi] Add delay to connection attempts in ControllerSettings
+      [Build flags] Remove NO_EXTRA_4K_HEAP
+
+
 -------------------------------------------------
 Changes in release mega-20181004 (since mega-20181003)
 -------------------------------------------------

platformio.ini

@@ -107,6 +107,8 @@ monitor_speed             = 115200
 ; NO_EXTRA_4K_HEAP - this forces the default NONOS-SDK user's heap location
 ;     Default currently overlaps cont stack (Arduino) with sys stack (System)
 ;     to save up-to 4 kB of heap. (starting core_2.4.2)
+; ESP8266_DISABLE_EXTRA4K  - Calls disable_extra4k_at_link_time() from setup
+;                            to force the linker keep user's stack in user ram.
 ; CONT_STACKSIZE to set the 'cont' (Arduino) stack size. Default = 4096
 
 [common]
@@ -131,15 +133,15 @@ monitor_speed             = 115200
 
 [normal]
 platform                  = ${common.platform}
-build_flags               = -DNO_EXTRA_4K_HEAP
+build_flags               = -DCONT_STACKSIZE=5120
 
 [testing]
 platform                  = ${core_2_4_1.platform}
-build_flags               = -DNO_EXTRA_4K_HEAP -DPLUGIN_BUILD_TESTING
+build_flags               = -DCONT_STACKSIZE=5120 -DPLUGIN_BUILD_TESTING
 
 [dev]
 platform                  = ${core_2_4_1.platform}
-build_flags               = -DNO_EXTRA_4K_HEAP -DPLUGIN_BUILD_DEV
+build_flags               = -DCONT_STACKSIZE=5120 -DPLUGIN_BUILD_DEV
 
 [ir]
 lib_ignore                = ESP32_ping, ESP32WebServer
@@ -202,6 +204,25 @@ build_flags               = ${common.build_flags} -Wl,-Tesp8266.flash.4m1m.ld
 ; Will probably not work, not tested and guaranteed to take a few hours time of some     ;
 ; still trying to build the version without reading this warning.                        ;
 ; *********************************************************************
+[env:esp-wrover-kit_test_1M8_partition]
+platform                  = ${core_esp32.platform}
+board                     = esp-wrover-kit
+build_unflags             = ${core_esp32.build_unflags}
+build_flags               = ${core_esp32.build_flags}  -DPLUGIN_SET_TEST_ESP32
+lib_deps                  = ${core_esp32.lib_deps}
+lib_ignore                = ${core_esp32.lib_ignore}
+lib_ldf_mode              = ${common.lib_ldf_mode}
+lib_archive               = ${common.lib_archive}
+framework                 = ${common.framework}
+upload_speed              = ${common.upload_speed}
+upload_protocol           = ftdi
+debug_tool                = ftdi
+monitor_speed             = ${common.monitor_speed}
+board_build.partitions    = esp32_partition_app1810k_spiffs316k.csv
+board_upload.maximum_size = 1900544
+debug_extra_cmds          = break Misc.ino:3011
+
+
 [env:esp32dev]
 platform                  = ${core_esp32.platform}
 board                     = esp32dev

src/Command.ino

@@ -131,7 +131,8 @@ String doExecuteCommand(const char * cmd, struct EventStruct *event, const char*
 	  COMMAND_CASE("taskclear"              , Command_Task_Clear);                 // Tasks.h
 	  COMMAND_CASE("taskclearall"           , Command_Task_ClearAll);              // Tasks.h
 	  COMMAND_CASE("taskrun"                , Command_Task_Run);                   // Tasks.h
-	  COMMAND_CASE("taskvalueset"           , Command_Task_ValueSet);              // Tasks.h
+    COMMAND_CASE("taskvalueset"           , Command_Task_ValueSet);              // Tasks.h
+    COMMAND_CASE("taskvaluetoggle"        , Command_Task_ValueToggle);           // Tasks.h
 	  COMMAND_CASE("taskvaluesetandrun"     , Command_Task_ValueSetAndRun);        // Tasks.h
 	  COMMAND_CASE("timerpause"             , Command_Timer_Pause);                // Timers.h
 	  COMMAND_CASE("timerresume"            , Command_Timer_Resume);               // Timers.h

src/Commands/Tasks.h

@@ -30,6 +30,15 @@ String Command_Task_ValueSet(struct EventStruct *event, const char* Line)
 	return return_command_success();
 }
 
+String Command_Task_ValueToggle(struct EventStruct *event, const char* Line)
+{
+	const int result = round(UserVar[(VARS_PER_TASK * (event->Par1 - 1)) + event->Par2 - 1]);
+	if (result == 0 || result == 1) {
+	  UserVar[(VARS_PER_TASK * (event->Par1 - 1)) + event->Par2 - 1] = !result;
+	}
+	return return_command_success();
+}
+
 String Command_Task_ValueSetAndRun(struct EventStruct *event, const char* Line)
 {
 	char TmpStr1[INPUT_COMMAND_SIZE];

src/ESPEasy-Globals.h

@@ -191,6 +191,8 @@
 
 #define MAX_FLASHWRITES_PER_DAY           100 // per 24 hour window
 #define INPUT_COMMAND_SIZE                240 // Affects maximum command length in rules and other commands
+// FIXME TD-er: INPUT_COMMAND_SIZE is also used in commands where simply a check for valid parameter is needed
+// and some may need less memory. (which is stack allocated)
 
 #define NODE_TYPE_ID_ESP_EASY_STD           1
 #define NODE_TYPE_ID_ESP_EASYM_STD         17
@@ -955,6 +957,7 @@ struct ControllerSettingsStruct
     if (!WiFiConnected(10)) {
       return false; // Not connected, so no use in wasting time to connect to a host.
     }
+    delay(1); // Make sure the Watchdog will not trigger a reset.
     if (quick && ipSet()) return true;
     if (UseDNS) {
       if (!updateIPcache()) {
@@ -1493,6 +1496,7 @@ unsigned long connectionFailures = 0;
 unsigned long wdcounter = 0;
 unsigned long timerAPoff = 0;
 unsigned long timerAwakeFromDeepSleep = 0;
+unsigned long last_system_event_run = 0;
 
 #if FEATURE_ADC_VCC
 float vcc = -1.0;

src/ESPEasy.ino

@@ -106,6 +106,9 @@ int firstEnabledBlynkController() {
 \*********************************************************************************************/
 void setup()
 {
+#ifdef ESP8266_DISABLE_EXTRA4K
+  disable_extra4k_at_link_time();
+#endif
   WiFi.persistent(false); // Do not use SDK storage of SSID/WPA parameters
   WiFi.setAutoReconnect(false);
   setWifiMode(WIFI_OFF);

src/ESPEasyStorage.ino

@@ -113,7 +113,7 @@ void fileSystemCheck()
     {
       ResetFactory();
     }
-    f.close();
+    if (f) f.close();
   }
   else
   {
@@ -518,13 +518,15 @@ String InitFile(const char* fname, int datasize)
   FLASH_GUARD();
 
   fs::File f = SPIFFS.open(fname, "w");
-  SPIFFS_CHECK(f, fname);
+  if (f) {
+    SPIFFS_CHECK(f, fname);
 
-  for (int x = 0; x < datasize ; x++)
-  {
-    SPIFFS_CHECK(f.write(0), fname);
+    for (int x = 0; x < datasize ; x++)
+    {
+      SPIFFS_CHECK(f.write(0), fname);
+    }
+    f.close();
   }
-  f.close();
 
   //OK
   return String();
@@ -556,19 +558,27 @@ String SaveToFile(char* fname, int index, byte* memAddress, int datasize)
   FLASH_GUARD();
 
   fs::File f = SPIFFS.open(fname, "r+");
-  SPIFFS_CHECK(f, fname);
+  if (f) {
+    SPIFFS_CHECK(f, fname);
 
-  SPIFFS_CHECK(f.seek(index, fs::SeekSet), fname);
-  byte *pointerToByteToSave = memAddress;
-  for (int x = 0; x < datasize ; x++)
-  {
-    SPIFFS_CHECK(f.write(*pointerToByteToSave), fname);
-    pointerToByteToSave++;
+    SPIFFS_CHECK(f.seek(index, fs::SeekSet), fname);
+    byte *pointerToByteToSave = memAddress;
+    for (int x = 0; x < datasize ; x++)
+    {
+      SPIFFS_CHECK(f.write(*pointerToByteToSave), fname);
+      pointerToByteToSave++;
+    }
+    f.close();
+    String log = F("FILE : Saved ");
+    log=log+fname;
+    addLog(LOG_LEVEL_INFO, log);
+  } else {
+    String log = F("SaveToFile: ");
+    log += fname;
+    log += F(" ERROR, Cannot save to file");
+    addLog(LOG_LEVEL_ERROR, log);
+    return log;
   }
-  f.close();
-  String log = F("FILE : Saved ");
-  log=log+fname;
-  addLog(LOG_LEVEL_INFO, log);
 
   //OK
   return String();
@@ -591,14 +601,22 @@ String ClearInFile(char* fname, int index, int datasize)
   FLASH_GUARD();
 
   fs::File f = SPIFFS.open(fname, "r+");
-  SPIFFS_CHECK(f, fname);
+  if (f) {
+    SPIFFS_CHECK(f, fname);
 
-  SPIFFS_CHECK(f.seek(index, fs::SeekSet), fname);
-  for (int x = 0; x < datasize ; x++)
-  {
-    SPIFFS_CHECK(f.write(0), fname);
+    SPIFFS_CHECK(f.seek(index, fs::SeekSet), fname);
+    for (int x = 0; x < datasize ; x++)
+    {
+      SPIFFS_CHECK(f.write(0), fname);
+    }
+    f.close();
+  } else {
+    String log = F("ClearInFile: ");
+    log += fname;
+    log += F(" ERROR, Cannot save to file");
+    addLog(LOG_LEVEL_ERROR, log);
+    return log;
   }
-  f.close();
 
   //OK
   return String();

src/ESPEasyWifi.ino

@@ -678,6 +678,7 @@ void WifiScan()
 
 String formatScanResult(int i, const String& separator) {
   String result = WiFi.SSID(i);
+  htmlEscape(result);
   #ifndef ESP32
   if (WiFi.isHidden(i)) {
     result += F("#Hidden#");

src/Misc.ino

@@ -445,23 +445,19 @@ void delayBackground(unsigned long delay)
 void parseCommandString(struct EventStruct *event, const String& string)
 {
   checkRAM(F("parseCommandString"));
-  char command[INPUT_COMMAND_SIZE];
-  command[0] = 0;
   char TmpStr1[INPUT_COMMAND_SIZE];
   TmpStr1[0] = 0;
-
-  string.toCharArray(command, INPUT_COMMAND_SIZE);
   event->Par1 = 0;
   event->Par2 = 0;
   event->Par3 = 0;
   event->Par4 = 0;
   event->Par5 = 0;
 
-  if (GetArgv(command, TmpStr1, 2)) event->Par1 = CalculateParam(TmpStr1);
-  if (GetArgv(command, TmpStr1, 3)) event->Par2 = CalculateParam(TmpStr1);
-  if (GetArgv(command, TmpStr1, 4)) event->Par3 = CalculateParam(TmpStr1);
-  if (GetArgv(command, TmpStr1, 5)) event->Par4 = CalculateParam(TmpStr1);
-  if (GetArgv(command, TmpStr1, 6)) event->Par5 = CalculateParam(TmpStr1);
+  if (GetArgv(string.c_str(), TmpStr1, 2)) { event->Par1 = CalculateParam(TmpStr1); }
+  if (GetArgv(string.c_str(), TmpStr1, 3)) { event->Par2 = CalculateParam(TmpStr1); }
+  if (GetArgv(string.c_str(), TmpStr1, 4)) { event->Par3 = CalculateParam(TmpStr1); }
+  if (GetArgv(string.c_str(), TmpStr1, 5)) { event->Par4 = CalculateParam(TmpStr1); }
+  if (GetArgv(string.c_str(), TmpStr1, 6)) { event->Par5 = CalculateParam(TmpStr1); }
 }
 
 /********************************************************************************************\
@@ -572,15 +568,18 @@ boolean GetArgv(const char *string, char *argv, unsigned int argc) {
 
 boolean GetArgv(const char *string, char *argv, unsigned int argv_size, unsigned int argc)
 {
+  size_t string_len = strlen(string);
   unsigned int string_pos = 0, argv_pos = 0, argc_pos = 0;
   char c, d;
   boolean parenthesis = false;
   char matching_parenthesis = '"';
 
-  while (string_pos < strlen(string))
+  while (string_pos < string_len)
   {
     c = string[string_pos];
-    d = string[string_pos + 1];
+    d = 0;
+    if ((string_pos + 1) < string_len)
+      d = string[string_pos + 1];
 
     if       (!parenthesis && c == ' ' && d == ' ') {}
     else if  (!parenthesis && c == ' ' && d == ',') {}
@@ -2208,7 +2207,7 @@ void rulesProcessing(String& event)
 /********************************************************************************************\
   Rules processing
   \*********************************************************************************************/
-String rulesProcessingFile(String fileName, String& event)
+String rulesProcessingFile(const String& fileName, String& event)
 {
   checkRAM(F("rulesProcessingFile"));
   if (Settings.SerialLogLevel == LOG_LEVEL_DEBUG_DEV){
@@ -2271,6 +2270,7 @@ String rulesProcessingFile(String fileName, String& event)
       }
     }
   }
+  if (f) f.close();
 
   nestingLevel--;
   checkRAM(F("rulesProcessingFile2"));
@@ -2504,19 +2504,16 @@ void processMatchedRule(
       {
         String tmpString = event.substring(equalsPos + 1);
 
-        char command[INPUT_COMMAND_SIZE];
-        command[0] = 0;
         char tmpParam[INPUT_COMMAND_SIZE];
         tmpParam[0] = 0;
-        tmpString.toCharArray(command, INPUT_COMMAND_SIZE);
 
-        if (GetArgv(command,tmpParam,1)) {
+        if (GetArgv(tmpString.c_str(),tmpParam,1)) {
            action.replace(F("%eventvalue%"), tmpParam); // for compatibility issues
            action.replace(F("%eventvalue1%"), tmpParam); // substitute %eventvalue1% in actions with the actual value from the event
         }
-        if (GetArgv(command,tmpParam,2)) action.replace(F("%eventvalue2%"), tmpParam); // substitute %eventvalue2% in actions with the actual value from the event
-        if (GetArgv(command,tmpParam,3)) action.replace(F("%eventvalue3%"), tmpParam); // substitute %eventvalue3% in actions with the actual value from the event
-        if (GetArgv(command,tmpParam,4)) action.replace(F("%eventvalue4%"), tmpParam); // substitute %eventvalue4% in actions with the actual value from the event
+        if (GetArgv(tmpString.c_str(),tmpParam,2)) action.replace(F("%eventvalue2%"), tmpParam); // substitute %eventvalue2% in actions with the actual value from the event
+        if (GetArgv(tmpString.c_str(),tmpParam,3)) action.replace(F("%eventvalue3%"), tmpParam); // substitute %eventvalue3% in actions with the actual value from the event
+        if (GetArgv(tmpString.c_str(),tmpParam,4)) action.replace(F("%eventvalue4%"), tmpParam); // substitute %eventvalue4% in actions with the actual value from the event
       }
     }
 

src/Scheduler.ino

@@ -41,19 +41,26 @@ unsigned long getMixedId(unsigned long timerType, unsigned long id) {
 \*********************************************************************************************/
 void handle_schedule() {
   unsigned long timer;
-  const unsigned long mixed_id = msecTimerHandler.getNextId(timer);
+  unsigned long mixed_id = 0;
+  if (timePassedSince(last_system_event_run) < 500) {
+    // Make sure system event queue will be looked at every now and then.
+    mixed_id = msecTimerHandler.getNextId(timer);
+  }
   if (mixed_id == 0) {
     // No id ready to run right now.
     // Events are not that important to run immediately.
     // Make sure normal scheduled jobs run at higher priority.
     backgroundtasks();
     process_system_event_queue();
+    last_system_event_run = millis();
     return;
   }
   const unsigned long timerType = (mixed_id >> TIMER_ID_SHIFT);
   const unsigned long mask = (1 << TIMER_ID_SHIFT) -1;
   const unsigned long id = mixed_id & mask;
 
+  yield(); // See: https://github.com/letscontrolit/ESPEasy/issues/1818#issuecomment-425351328
+
   switch (timerType) {
     case CONST_INTERVAL_TIMER:
       process_interval_timer(id, timer);

src/StringConverter.ino

@@ -373,8 +373,31 @@ void htmlEscape(String & html)
   html.replace("'",  F("'"));
   html.replace("<",  F("&lt;"));
   html.replace(">",  F("&gt;"));
+  html.replace("/", F("&#047;"));
 }
 
+void htmlStrongEscape(String & html)
+{
+  String escaped;
+  for (unsigned i = 0; i < html.length(); ++i)
+  {
+    if ((html[i] >= 'a' && html[i] <= 'z') || (html[i] >= 'A' && html[i] <= 'Z') || (html[i] >= '0' && html[i] <= '9'))
+    {
+      escaped += html[i];
+    }
+    else
+    {
+      char s [4];
+      sprintf(s, "%03d", static_cast<int>(html[i]));
+      escaped += "&#";
+      escaped += s;
+      escaped += ";";
+    }
+  }
+  html = escaped;
+}
+
+
 /********************************************************************************************\
   replace other system variables like %sysname%, %systime%, %ip%
   \*********************************************************************************************/

src/WebServer.ino

@@ -3665,6 +3665,7 @@ void handle_control() {
   }
   else if (command.equalsIgnoreCase(F("taskrun")) ||
            command.equalsIgnoreCase(F("taskvalueset")) ||
+           command.equalsIgnoreCase(F("taskvaluetoggle")) ||
            command.equalsIgnoreCase(F("rules"))) {
     addLog(LOG_LEVEL_INFO,String(F("HTTP : ")) + webrequest);
     ExecuteCommand(VALUE_SOURCE_HTTP,webrequest.c_str());
@@ -4206,6 +4207,7 @@ void handle_download()
 
   WebServer.sendHeader(F("Content-Disposition"), str);
   WebServer.streamFile(dataFile, F("application/octet-stream"));
+  dataFile.close();
 }
 
 
@@ -4629,7 +4631,10 @@ void handle_filelist() {
     TXBuffer += F("</a>");
     fs::File f = dir.openFile("r");
     html_TD();
-    TXBuffer += f.size();
+    if (f) {
+      TXBuffer += f.size();
+      f.close();
+    }
     if (count >= endIdx)
     {
       break;
@@ -4987,7 +4992,11 @@ void handle_setup() {
       {
         html_TR_TD(); TXBuffer += F("<label class='container2'>");
         TXBuffer += F("<input type='radio' name='ssid' value='");
-        TXBuffer += WiFi.SSID(i);
+        {
+          String escapeBuffer = WiFi.SSID(i);
+          htmlStrongEscape(escapeBuffer);
+          TXBuffer += escapeBuffer;
+        }
         TXBuffer += F("'");
         if (WiFi.SSID(i) == ssid)
           TXBuffer += F(" checked ");
@@ -5126,7 +5135,7 @@ void handle_rules() {
         log += F(" Create new file: ");
         log += fileName;
         fs::File f = SPIFFS.open(fileName, "w");
-        f.close();
+        if (f) f.close();
       }
     }
     addLog(LOG_LEVEL_INFO, log);

src/_P001_Switch.ino

@@ -257,6 +257,22 @@ boolean Plugin_001(byte function, struct EventStruct *event, String& string)
           }
         }
 
+        if (command == F("gpiotoggle"))
+        {
+          success = true;
+          byte mode;
+          uint16_t currentState;
+
+          getPinState(PLUGIN_ID_001, event->Par1, &mode, &currentState);
+          if (mode == PIN_MODE_OUTPUT || mode == PIN_MODE_UNDEFINED) { //toggle only output pins
+            digitalWrite(event->Par1, !currentState);
+            setPinState(PLUGIN_ID_001, event->Par1, PIN_MODE_OUTPUT, !currentState);
+            log = String(F("SW   : Toggle GPIO ")) + String(event->Par1) + String(F(" Set to ")) + String(!currentState);
+            addLog(LOG_LEVEL_INFO, log);
+            SendStatus(event->Source, getPinStateJSON(SEARCH_PIN_STATE, PLUGIN_ID_001, event->Par1, log, 0));
+          }
+        }
+
         if (command == F("pwm"))
         {
           success = true;

src/_P009_MCP.ino

@@ -124,6 +124,22 @@ boolean Plugin_009(byte function, struct EventStruct *event, String& string)
           SendStatus(event->Source, getPinStateJSON(SEARCH_PIN_STATE, PLUGIN_ID_009, event->Par1, log, 0));
         }
 
+        if (command == F("mcpgpiotoggle"))
+        {
+          success = true;
+          byte mode;
+          uint16_t currentState;
+
+          getPinState(PLUGIN_ID_009, event->Par1, &mode, &currentState);
+          if (mode == PIN_MODE_OUTPUT || mode == PIN_MODE_UNDEFINED) {
+            setPinState(PLUGIN_ID_009, event->Par1, PIN_MODE_OUTPUT, !currentState);
+            Plugin_009_Write(event->Par1, !currentState);
+            log = String(F("MCP  : Toggle GPIO ")) + String(event->Par1) + String(F(" Set to ")) + String(!currentState);
+            addLog(LOG_LEVEL_INFO, log);
+            SendStatus(event->Source, getPinStateJSON(SEARCH_PIN_STATE, PLUGIN_ID_009, event->Par1, log, 0));
+          }
+        }
+
         if (command == F("mcppulse"))
         {
           success = true;

src/_P019_PCF8574.ino

@@ -128,6 +128,22 @@ boolean Plugin_019(byte function, struct EventStruct *event, String& string)
           SendStatus(event->Source, getPinStateJSON(SEARCH_PIN_STATE, PLUGIN_ID_019, event->Par1, log, 0));
         }
 
+        if (command == F("pcfgpiotoggle"))
+        {
+          success = true;
+          byte mode;
+          uint16_t currentState;
+
+          getPinState(PLUGIN_ID_019, event->Par1, &mode, &currentState);
+          if (mode == PIN_MODE_OUTPUT || mode == PIN_MODE_UNDEFINED) {
+            setPinState(PLUGIN_ID_019, event->Par1, PIN_MODE_OUTPUT, !currentState);
+            Plugin_019_Write(event->Par1, !currentState);
+            log = String(F("PCF  : Toggle GPIO ")) + String(event->Par1) + String(F(" Set to ")) + String(!currentState);
+            addLog(LOG_LEVEL_INFO, log);
+            SendStatus(event->Source, getPinStateJSON(SEARCH_PIN_STATE, PLUGIN_ID_019, event->Par1, log, 0));
+          }
+        }
+
         if (command == F("pcfpulse"))
         {
           success = true;