[Factory Reset] Fix keeping settings (#4263)

Fixes: #4263
Fixes: #2746
Fixes: #2226

docs/source/Plugin/P000_commands.repl

@@ -64,6 +64,32 @@
     This data is persistent as long as the node is powered and thus survives a reboot or crash.
 
     ``ClearRTCram``"
+    "
+    ClearSdkWifi","
+    :red:`Internal`","
+	  ESP8266 only (added: 2022/11/11)
+    Clear the 12k SDK WiFi partition at the end of the flash memory.
+    This data is used by the ESP8266 Arduino SDK to keep some persistent WiFi settings.
+    When WiFi is performing badly, it might be useful to clear this section. 
+    Especially when switching SDK build revisions, like when switching between regular builds and "beta" or "alt.WiFi" builds.
+    Please reboot after clearing this partition.
+    See also ``ClearWifiRFcal`` command.
+
+    ``ClearSdkWifi``"
+    "
+    ClearWifiRFcal","
+    :red:`Internal`","
+    ESP8266 only (added: 2022/11/11)
+    Clear the 4k RFcal partition near the end of the flash memory.
+    This data is used by the ESP8266 Arduino SDK to keep store WiFi RF calibration data.
+    When WiFi is performing badly, it might be useful to clear this section. 
+    Especially when switching SDK build revisions, like when switching between regular builds and "beta" or "alt.WiFi" builds.
+    Another reason to clear the RF calibration is when initial RF calibration was done with a different power supply, or when WiFi performs badly after placing the ESP in a different enclosure which may de-tune the antenna.
+    Please reboot after clearing this partition.
+    See also ``ClearSdkWifi`` command.
+
+    ``ClearWifiRFcal``"
+
     "
     Config","
     :red:`Internal`","

src/src/Commands/InternalCommands.cpp

@@ -275,6 +275,10 @@ bool executeInternalCommand(command_case_data & data)
       COMMAND_CASE_R( "clearaccessblock", Command_AccessInfo_Clear,   0); // Network Command
       COMMAND_CASE_R(    "clearpassword", Command_Settings_Password_Clear,     1); // Settings.h
       COMMAND_CASE_R(      "clearrtcram", Command_RTC_Clear,          0); // RTC.h
+      #ifdef ESP8266
+      COMMAND_CASE_R(     "clearsdkwifi", Command_System_Erase_SDK_WiFiconfig,  0); // System.h
+      COMMAND_CASE_R(  "clearwifirfcall", Command_System_Erase_RFcal,  0); // System.h
+      #endif
       COMMAND_CASE_R(           "config", Command_Task_RemoteConfig, -1); // Tasks.h
       COMMAND_CASE_R("controllerdisable", Command_Controller_Disable, 1); // Controller.h
       COMMAND_CASE_R( "controllerenable", Command_Controller_Enable,  1); // Controller.h

src/src/Commands/System.cpp

@@ -8,6 +8,7 @@
 #include "../Globals/Settings.h"
 
 #include "../Helpers/DeepSleep.h"
+#include "../Helpers/ESPEasy_Storage.h"
 #include "../Helpers/Misc.h"
 #include "../Helpers/Scheduler.h"
 
@@ -36,3 +37,23 @@ const __FlashStringHelper * Command_System_Reboot(struct EventStruct *event, con
 	return return_command_success();
 }
 
+#ifdef ESP8266
+// Erase the RF calibration partition (4k)
+const __FlashStringHelper * Command_System_Erase_RFcal(struct EventStruct *event, const char* Line)
+{
+	if (clearRFcalPartition()) {
+		return F("Cleared RFcal partition. Please reboot!");
+	}
+	return return_command_failed();
+}
+
+// Erase the SDK WiFI partition (12k)
+const __FlashStringHelper * Command_System_Erase_SDK_WiFiconfig(struct EventStruct *event, const char* Line)
+{
+	if (clearWiFiSDKpartition()) {
+		return F("Cleared SDK WiFi partition. Please reboot!");
+	}
+	return return_command_failed();
+}
+
+#endif

src/src/Commands/System.h

@@ -7,4 +7,11 @@ const __FlashStringHelper * Command_System_NoSleep(struct EventStruct *event, co
 const __FlashStringHelper * Command_System_deepSleep(struct EventStruct *event, const char* Line);
 const __FlashStringHelper * Command_System_Reboot(struct EventStruct *event, const char* Line);
 
+#ifdef ESP8266
+// Erase the RF calibration partition (4k)
+const __FlashStringHelper * Command_System_Erase_RFcal(struct EventStruct *event, const char* Line);
+// Erase the SDK WiFI partition (12k)
+const __FlashStringHelper * Command_System_Erase_SDK_WiFiconfig(struct EventStruct *event, const char* Line);
+#endif
+
 #endif // COMMAND_SYSTEM_H

src/src/DataStructs/ExtendedControllerCredentialsStruct.cpp

@@ -39,6 +39,12 @@ bool ExtendedControllerCredentialsStruct::computeChecksum(uint8_t checksum[16])
   return true;
 }
 
+void ExtendedControllerCredentialsStruct::clear() {
+  for (size_t i = 0; i < CONTROLLER_MAX * 2; ++i) {
+    _strings[i] = String();
+  }
+}
+
 String ExtendedControllerCredentialsStruct::load()
 {
   const String res =

src/src/DataStructs/ExtendedControllerCredentialsStruct.h

@@ -18,6 +18,8 @@ struct ExtendedControllerCredentialsStruct
   // @retval true when checksum matches
   bool computeChecksum(uint8_t checksum[16]) const;
 
+  void clear();
+
   String load();
   String save() const;
 

src/src/DataStructs/SecurityStruct.cpp

@@ -33,6 +33,10 @@ void SecurityStruct::validate() {
   ZERO_TERMINATE(Password);
 }
 
+void SecurityStruct::forceSave() {
+  memset(md5, 0, 16);
+}
+
 void SecurityStruct::clearWiFiCredentials() {
   ZERO_FILL(WifiSSID);
   ZERO_FILL(WifiKey);

src/src/DataStructs/SecurityStruct.h

@@ -19,6 +19,9 @@ struct SecurityStruct
 
   void validate();
 
+  // Clear the checksum to make sure file will be saved
+  void forceSave();
+
   void clearWiFiCredentials();
 
   void clearWiFiCredentials(WiFiCredentialsSlot slot);

src/src/Helpers/ESPEasy_FactoryDefault.cpp

@@ -10,6 +10,7 @@
 #include "../DataStructs/FactoryDefaultPref.h"
 #include "../DataStructs/GpioFactorySettingsStruct.h"
 
+#include "../ESPEasyCore/ESPEasy_backgroundtasks.h"
 #include "../ESPEasyCore/ESPEasyWifi.h"
 #include "../ESPEasyCore/Serial.h"
 
@@ -57,6 +58,7 @@ void ResetFactory()
   serialPrint(F("RESET: Resetting factory defaults... using "));
   serialPrint(getDeviceModelString(ResetFactoryDefaultPreference.getDeviceModel()));
   serialPrintln(F(" settings"));
+  process_serialWriteBuffer();
   delay(1000);
 
   if (readFromRTC())
@@ -84,10 +86,11 @@ void ResetFactory()
   saveToRTC();
 
   // always format on factory reset, in case of corrupt FS
-//  ESPEASY_FS.end();
+  ESPEASY_FS.end();
   serialPrintln(F("RESET: formatting..."));
   FS_format();
   serialPrintln(F("RESET: formatting done..."));
+  process_serialWriteBuffer();
 
   if (!ESPEASY_FS.begin())
   {
@@ -273,8 +276,8 @@ void ResetFactory()
     }
   }
 #endif // if DEFAULT_CONTROLLER
-
-  SaveSettings();
+  const bool forFactoryReset = true;
+  SaveSettings(forFactoryReset);
   #ifndef BUILD_NO_RAM_TRACKER
   checkRAM(F("ResetFactory2"));
   #endif

src/src/Helpers/ESPEasy_Storage.cpp

@@ -449,6 +449,31 @@ int getPartionCount(uint8_t pType, uint8_t pSubType) {
 
 #endif
 
+ #ifdef ESP8266
+bool clearPartition(ESP8266_partition_type ptype) {
+  uint32_t address;
+  int32_t size;
+  int32_t sector = getPartitionInfo(ESP8266_partition_type::rf_cal, address, size);
+  while (size > 0) {
+    if (!ESP.flashEraseSector(sector)) return false;
+    ++sector;
+    size -= SPI_FLASH_SEC_SIZE;
+
+  }
+  return true;
+}
+
+bool clearRFcalPartition() {
+  return clearPartition(ESP8266_partition_type::rf_cal);
+}
+
+bool clearWiFiSDKpartition() {
+  return clearPartition(ESP8266_partition_type::wifi);
+}
+
+#endif
+
+
 /********************************************************************************************\
    Garbage collection
  \*********************************************************************************************/
@@ -509,7 +534,7 @@ bool computeChecksum(
 /********************************************************************************************\
    Save settings to file system
  \*********************************************************************************************/
-String SaveSettings()
+String SaveSettings(bool forFactoryReset)
 {
   #ifndef BUILD_NO_RAM_TRACKER
   checkRAM(F("SaveSettings"));
@@ -549,23 +574,27 @@ String SaveSettings()
 
   //  }
 
-  err = SaveSecuritySettings();
+  err = SaveSecuritySettings(forFactoryReset);
 
   return err;
 }
 
-String SaveSecuritySettings() {
+String SaveSecuritySettings(bool forFactoryReset) {
   String     err;
 
   SecuritySettings.validate();
   memcpy(SecuritySettings.ProgmemMd5, CRCValues.runTimeMD5, 16);
 
+  if (forFactoryReset) {
+    SecuritySettings.forceSave();
+  }
+
   if (!COMPUTE_STRUCT_CHECKSUM_UPDATE(SecurityStruct, SecuritySettings)) {
     // Settings have changed, save to file.
     err = SaveToFile(SettingsType::getSettingsFileName(SettingsType::Enum::SecuritySettings_Type).c_str(), 0, reinterpret_cast<const uint8_t *>(&SecuritySettings), sizeof(SecuritySettings));
 
     // Security settings are saved, may be update of WiFi settings or hostname.
-    if (!NetworkConnected()) {
+    if (!forFactoryReset && !NetworkConnected()) {
       if (SecuritySettings.hasWiFiCredentials() && active_network_medium == NetworkMedium_t::WIFI) {
         WiFiEventData.wifiConnectAttemptNeeded = true;
         WiFi_AP_Candidates.force_reload(); // Force reload of the credentials and found APs from the last scan
@@ -575,12 +604,15 @@ String SaveSecuritySettings() {
     }
   } else {
     addLog(LOG_LEVEL_INFO, F("Skip saving SecuritySettings, not changed"));
-
   }
 
   // FIXME TD-er: How to check if these have changed?
+  if (forFactoryReset) {
+    ExtendedControllerCredentials.clear();
+  }
   ExtendedControllerCredentials.save();
-  afterloadSettings();
+  if (!forFactoryReset)
+    afterloadSettings();
   return err;
 }
 

src/src/Helpers/ESPEasy_Storage.h

@@ -56,6 +56,17 @@ int  getPartionCount(uint8_t pType, uint8_t pSubType = 0xFF);
 
 #endif
 
+/********************************************************************************************\
+   Low level clear RFcal and SDK WiFi parameters.
+ \*********************************************************************************************/
+ #ifdef ESP8266
+bool clearRFcalPartition();
+
+bool clearWiFiSDKpartition();
+
+#endif
+
+
 /********************************************************************************************\
    Garbage collection
  \*********************************************************************************************/
@@ -89,9 +100,9 @@ bool computeChecksum(
 /********************************************************************************************\
    Save settings to file system
  \*********************************************************************************************/
-String SaveSettings();
+String SaveSettings(bool forFactoryReset = false);
 
-String SaveSecuritySettings();
+String SaveSecuritySettings(bool forFactoryReset = false);
 
 void afterloadSettings();
 

src/src/Helpers/FS_Helper.h

@@ -38,15 +38,19 @@ using namespace fs;
 
 
 #if defined(ESP8266)
-  extern "C" {
-    #include <spi_flash.h>
-  }
   #ifdef CORE_POST_2_6_0
+  extern "C" {
+    #include <flash_hal.h>
+  }
     extern "C" uint32_t _FS_start;
     extern "C" uint32_t _FS_end;
     extern "C" uint32_t _FS_page;
     extern "C" uint32_t _FS_block;
+    extern "C" uint32_t _EEPROM_start;
   #else
+  extern "C" {
+    #include <spi_flash.h>
+  }
     extern "C" uint32_t _SPIFFS_start;
     extern "C" uint32_t _SPIFFS_end;
     extern "C" uint32_t _SPIFFS_page;

src/src/Helpers/Hardware.cpp

@@ -14,6 +14,7 @@
 
 #include "../Helpers/ESPEasy_FactoryDefault.h"
 #include "../Helpers/ESPEasy_Storage.h"
+#include "../Helpers/FS_Helper.h"
 #include "../Helpers/Misc.h"
 #include "../Helpers/PortStatus.h"
 #include "../Helpers/StringConverter.h"
@@ -599,6 +600,48 @@ int espeasy_analogRead(int pin, bool readAsTouch) {
 /********************************************************************************************\
    Hardware information
  \*********************************************************************************************/
+#ifdef ESP8266
+int32_t getPartitionInfo(ESP8266_partition_type ptype, uint32_t& address, int32_t& size)
+{
+  address = 0;
+  size = -1;
+  const uint32_t addr_offset = 0x40200000;
+  const uint32_t realSize = getFlashRealSizeInBytes();
+  switch(ptype) {
+    case ESP8266_partition_type::sketch:
+      address = 0;
+      size = getSketchSize();
+      break;
+    case ESP8266_partition_type::ota:
+      address = getSketchSize();
+      size = getFreeSketchSpace();
+      break;
+    case ESP8266_partition_type::fs:
+      address = ((uint32_t)&_FS_start - addr_offset);
+      size = ((uint32_t)((uint32_t)&_FS_end - (uint32_t)&_FS_start));
+      break;
+    case ESP8266_partition_type::eeprom:
+      address = ((uint32_t)&_EEPROM_start - addr_offset);
+      size = realSize - address - 16384;
+      break;
+    case ESP8266_partition_type::rf_cal:
+      address = realSize - 16384;
+      size = 4096;
+      break;
+    case ESP8266_partition_type::wifi:
+      address = realSize - 12288;
+      size = 12288;
+      break;
+  }
+  if (size > 0)
+    return address / SPI_FLASH_SEC_SIZE;
+  return -1;
+}
+
+
+#endif
+
+
 uint32_t getFlashChipId() {
   // Cache since size does not change
   static uint32_t flashChipId = 0;

src/src/Helpers/Hardware.h

@@ -98,6 +98,24 @@ extern esp_adc_cal_characteristics_t adc_chars[ADC_ATTEN_MAX];
 /********************************************************************************************\
    Hardware information
  \*********************************************************************************************/
+#ifdef ESP8266
+enum class ESP8266_partition_type {
+  sketch,
+  ota,
+  fs,
+  eeprom,
+  rf_cal,
+  wifi
+};
+
+// Get info on the partition type
+// @retval The flash sector. (negative on unknown ptype)
+int32_t getPartitionInfo(ESP8266_partition_type ptype, uint32_t& address, int32_t& size);
+
+
+#endif
+
+
 uint32_t                   getFlashChipId();
 
 uint32_t                   getFlashRealSizeInBytes();

src/src/Helpers/PeriodicalActions.cpp

@@ -475,6 +475,7 @@ void prepareShutdown(ESPEasy_Scheduler::IntendedRebootReason_e reason)
   saveUserVarToRTC();
   setWifiMode(WIFI_OFF);
   ESPEASY_FS.end();
+  process_serialWriteBuffer();
   delay(100); // give the node time to flush all before reboot or sleep
   node_time.now();
   Scheduler.markIntendedReboot(reason);

src/src/WebServer/ESPEasy_WebServer.cpp

@@ -1043,6 +1043,27 @@ void getStorageTableSVG(SettingsType::Enum settingsType) {
 
 #endif // ifndef BUILD_MINIMAL_OTA
 
+
+void drawPartitionChartSVG(
+                          float yOffset, 
+                          uint32_t realSize, 
+                          uint32_t partitionAddress, 
+                          uint32_t partitionSize,
+                          unsigned int partitionColor,
+                          const String& label,
+                          const String& name)
+{
+  createSvgHorRectPath(0xcdcdcd,       0,                yOffset, realSize,      SVG_BAR_HEIGHT - 2, realSize, SVG_BAR_WIDTH);
+  createSvgHorRectPath(partitionColor, partitionAddress, yOffset, partitionSize, SVG_BAR_HEIGHT - 2, realSize, SVG_BAR_WIDTH);
+  float textXoffset = SVG_BAR_WIDTH + 2;
+  float textYoffset = yOffset + 0.9f * SVG_BAR_HEIGHT;
+  createSvgTextElement(formatHumanReadable(partitionSize, 1024),          textXoffset, textYoffset);
+  textXoffset = SVG_BAR_WIDTH + 60;
+  createSvgTextElement(label, textXoffset, textYoffset);
+  textXoffset = SVG_BAR_WIDTH + 130;
+  createSvgTextElement(name, textXoffset, textYoffset);
+}
+
 #ifdef ESP32
 
 # include <esp_partition.h>
@@ -1065,15 +1086,14 @@ void getPartitionTableSVG(uint8_t pType, unsigned int partitionColor) {
   if (_mypartiterator) {
     do {
       _mypart = esp_partition_get(_mypartiterator);
-      createSvgHorRectPath(0xcdcdcd,       0,                yOffset, realSize,      SVG_BAR_HEIGHT - 2, realSize, SVG_BAR_WIDTH);
-      createSvgHorRectPath(partitionColor, _mypart->address, yOffset, _mypart->size, SVG_BAR_HEIGHT - 2, realSize, SVG_BAR_WIDTH);
-      float textXoffset = SVG_BAR_WIDTH + 2;
-      float textYoffset = yOffset + 0.9f * SVG_BAR_HEIGHT;
-      createSvgTextElement(formatHumanReadable(_mypart->size, 1024),          textXoffset, textYoffset);
-      textXoffset = SVG_BAR_WIDTH + 60;
-      createSvgTextElement(_mypart->label,                                    textXoffset, textYoffset);
-      textXoffset = SVG_BAR_WIDTH + 130;
-      createSvgTextElement(getPartitionType(_mypart->type, _mypart->subtype), textXoffset, textYoffset);
+      drawPartitionChartSVG(
+        yOffset, 
+        realSize, 
+        _mypart->address, 
+        _mypart->size, 
+        partitionColor, 
+        _mypart->label,
+        getPartitionType(_mypart->type, _mypart->subtype));
       yOffset += SVG_BAR_HEIGHT;
     } while ((_mypartiterator = esp_partition_next(_mypartiterator)) != nullptr);
   }
@@ -1083,6 +1103,82 @@ void getPartitionTableSVG(uint8_t pType, unsigned int partitionColor) {
 
 #endif // ifdef ESP32
 
+#ifdef ESP8266
+void getPartitionTableSVG() {
+  // sketch / OTA / FS / EEPROM / RFcal / wifi
+  const int nrPartitions = 6;
+  const int shiftY = 2;
+  write_SVG_image_header(SVG_BAR_WIDTH + 250, nrPartitions * SVG_BAR_HEIGHT + shiftY);
+  float yOffset = shiftY;
+
+  for (int i = 0; i < nrPartitions; ++i) {
+    const ESP8266_partition_type ptype = static_cast<ESP8266_partition_type>(i);
+    uint32_t partitionAddress = 0;
+    int32_t partitionSize = 0;
+    const int32_t partitionSector = getPartitionInfo(ptype, partitionAddress, partitionSize);
+
+    const __FlashStringHelper * label = F("");
+    String descr;
+    unsigned int partitionColor = 0xab56e6;
+    switch (ptype) {
+      case ESP8266_partition_type::sketch:
+        label = F("sketch");
+        partitionColor = 0xab56e6;
+        break;
+      case ESP8266_partition_type::ota:
+        label = F("ota");
+        partitionColor = 0x5856e6;
+        break;
+      case ESP8266_partition_type::fs:
+        label = F("fs");
+        partitionColor = 0xff7f00;
+        #ifdef USE_LITTLEFS
+        descr = F("LittleFS");
+        #else
+        descr = F("SPIFFS");
+        #endif
+        break;
+      case ESP8266_partition_type::eeprom:
+        label = F("eeprom");
+        descr = concat(F("sector:"), partitionSector);
+        partitionColor = 0x7fff00;
+        break;
+      case ESP8266_partition_type::rf_cal:
+        label = F("RFcal");
+        partitionColor = 0xff007f;
+        break;
+      case ESP8266_partition_type::wifi:
+        label = F("WiFi");
+        partitionColor = 0xff00ff;
+        break;
+
+    }
+
+    drawPartitionChartSVG(
+        yOffset, 
+        getFlashRealSizeInBytes(), 
+        partitionAddress, 
+        partitionSize, 
+        partitionColor, 
+        label,
+        descr);
+    yOffset += SVG_BAR_HEIGHT;
+
+/*
+    String debuglog = concat(F("partition: "), (i+1));
+    debuglog += concat(F(" FS_st: "), formatToHex((uint32_t)&_FS_start));
+    debuglog += concat(F(" FS_end: "), formatToHex((uint32_t)&_FS_end));
+    debuglog += concat(F(" EEPROM: "), formatToHex((uint32_t)&_EEPROM_start));
+    debuglog += concat(F(" addr: "), formatToHex(partitionAddress, 8));
+    debuglog += concat(F(" part.size: "), partitionSize);
+    debuglog += concat(F(" label: "), label);
+    addLog(LOG_LEVEL_INFO, debuglog);
+*/
+  }
+  addHtml(F("</svg>\n"));
+}
+#endif
+
 bool webArg2ip(const __FlashStringHelper * arg, uint8_t *IP) {
   return str2ip(webArg(arg), IP);
 }

src/src/WebServer/ESPEasy_WebServer.h

@@ -190,6 +190,9 @@ void getPartitionTableSVG(uint8_t         pType,
                           unsigned int partitionColor);
 
 #endif // ifdef ESP32
+#ifdef ESP8266
+void getPartitionTableSVG();
+#endif
 
 bool webArg2ip(const __FlashStringHelper * arg,
                uint8_t         *IP);

src/src/WebServer/SysInfoPage.cpp

@@ -751,6 +751,14 @@ void handle_sysinfo_Storage() {
   //   TXBuffer += getPartitionTable(ESP_PARTITION_TYPE_APP , F(" - "), F("<BR>"));
   getPartitionTableSVG(ESP_PARTITION_TYPE_APP, 0xab56e6);
   # endif // ifdef ESP32
+
+  #ifdef ESP8266
+  addTableSeparator(F("Partitions"), 2, 3);
+
+  addRowLabel(F("Partition Table"));
+
+  getPartitionTableSVG();
+  #endif
 }
 #endif