From 9da650d0386b6b106b612996559780644cd5d1d6 Mon Sep 17 00:00:00 2001 From: TD-er Date: Mon, 9 May 2022 13:11:49 +0200 Subject: [PATCH 01/14] [Provisioning] Add command to download firmware OTA --- platformio_core_defs.ini | 2 +- src/src/Commands/InternalCommands.cpp | 1 + src/src/Commands/Provisioning.cpp | 11 +++ src/src/Commands/Provisioning.h | 2 + src/src/DataTypes/ESPEasyFileType.cpp | 3 + src/src/DataTypes/ESPEasyFileType.h | 2 +- src/src/Helpers/Networking.cpp | 123 +++++++++++++++++++++++--- src/src/Helpers/Networking.h | 2 + 8 files changed, 132 insertions(+), 14 deletions(-) diff --git a/platformio_core_defs.ini b/platformio_core_defs.ini index 948010d82..c7bf7da0a 100644 --- a/platformio_core_defs.ini +++ b/platformio_core_defs.ini @@ -167,7 +167,7 @@ build_flags = -DESP32_STAGE ; IDF 4.4 = platform-espressif32 3.4.x = espressif/arduino-esp32 tag 2.0.3 ; Just for those who lost track of the extremely confusing numbering schema. [core_esp32_IDF4_4__2_0_3] -platform = https://github.com/tasmota/platform-espressif32/releases/download/v2.0.3rc1/platform-espressif32-2.0.3new.zip +platform = https://github.com/tasmota/platform-espressif32/releases/download/v.2.0.3/platform-espressif32-v.2.0.3.zip platform_packages = build_flags = -DESP32_STAGE diff --git a/src/src/Commands/InternalCommands.cpp b/src/src/Commands/InternalCommands.cpp index 42df34a8d..0bce7667c 100644 --- a/src/src/Commands/InternalCommands.cpp +++ b/src/src/Commands/InternalCommands.cpp @@ -391,6 +391,7 @@ bool executeInternalCommand(command_case_data & data) COMMAND_CASE_A( "provisionnotification", Command_Provisioning_Notification, 0); // Provisioning.h COMMAND_CASE_A( "provisionprovision", Command_Provisioning_Provision, 0); // Provisioning.h COMMAND_CASE_A( "provisionrules", Command_Provisioning_Rules, 1); // Provisioning.h + COMMAND_CASE_A( "provisionfirmware", Command_Provisioning_Firmware, 1); // Provisioning.h #endif COMMAND_CASE_A( "pulse", Command_GPIO_Pulse, 3); // GPIO.h #ifdef USES_MQTT diff --git a/src/src/Commands/Provisioning.cpp b/src/src/Commands/Provisioning.cpp index 5c4ac7791..018e16e00 100644 --- a/src/src/Commands/Provisioning.cpp +++ b/src/src/Commands/Provisioning.cpp @@ -39,4 +39,15 @@ String Command_Provisioning_Rules(struct EventStruct *event, const char *Line) return downloadFileType(FileType::RULES_TXT, event->Par1 - 1); } +String Command_Provisioning_Firmware(struct EventStruct *event, const char *Line) +{ + const String url = parseStringToEndKeepCase(Line, 2); + String error; + if (downloadFirmware(url, error)) { + // TODO TD-er: send events + } + return error; +} + + #endif // ifdef USE_CUSTOM_PROVISIONING diff --git a/src/src/Commands/Provisioning.h b/src/src/Commands/Provisioning.h index 16b448acb..2911ed0dd 100644 --- a/src/src/Commands/Provisioning.h +++ b/src/src/Commands/Provisioning.h @@ -18,6 +18,8 @@ String Command_Provisioning_Provision(struct EventStruct *event, String Command_Provisioning_Rules(struct EventStruct *event, const char *Line); +String Command_Provisioning_Firmware(struct EventStruct *event, + const char *Line); #endif // ifdef USE_CUSTOM_PROVISIONING diff --git a/src/src/DataTypes/ESPEasyFileType.cpp b/src/src/DataTypes/ESPEasyFileType.cpp index 0166b4ab9..0bbb73104 100644 --- a/src/src/DataTypes/ESPEasyFileType.cpp +++ b/src/src/DataTypes/ESPEasyFileType.cpp @@ -31,6 +31,9 @@ const __FlashStringHelper * getFileName(FileType::Enum filetype) { case FileType::RULES_TXT: // Use getRulesFileName break; + case FileType::FIRMWARE: + // File name may differ each time. + break; case FileType::MAX_FILETYPE: break; diff --git a/src/src/DataTypes/ESPEasyFileType.h b/src/src/DataTypes/ESPEasyFileType.h index 56bbad8b9..38503b71d 100644 --- a/src/src/DataTypes/ESPEasyFileType.h +++ b/src/src/DataTypes/ESPEasyFileType.h @@ -10,7 +10,7 @@ struct FileType { RULES_TXT, NOTIFICATION_DAT, PROVISIONING_DAT, - + FIRMWARE, MAX_FILETYPE }; diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index f1845ec52..c47932553 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -61,6 +61,7 @@ void etharp_gratuitous_r(struct netif *netif) { # endif // ifdef ESP8266 # ifdef ESP32 # include +# include # endif // ifdef ESP32 #endif @@ -1190,7 +1191,7 @@ bool downloadFile(const String& url, String file_save) { return downloadFile(url, file_save, EMPTY_STRING, EMPTY_STRING, error); } -bool downloadFile(const String& url, String file_save, const String& user, const String& pass, String& error) { +bool start_downloadFile(WiFiClient& client, HTTPClient& http, const String& url, String& file_save, const String& user, const String& pass, String& error) { String host, file; uint16_t port; String uri = splitURL(url, host, port, file); @@ -1219,16 +1220,6 @@ bool downloadFile(const String& url, String file_save, const String& user, const return false; } - if (fileExists(file_save)) { - error = F("File exists: "); - error += file_save; - addLog(LOG_LEVEL_ERROR, error); - return false; - } - unsigned long timeout = millis() + 2000; - WiFiClient client; - HTTPClient http; - http.begin(client, host, port, uri); { if ((user.length() > 0) && (pass.length() > 0)) { @@ -1255,6 +1246,24 @@ bool downloadFile(const String& url, String file_save, const String& user, const http.end(); return false; } + return true; +} + +bool downloadFile(const String& url, String file_save, const String& user, const String& pass, String& error) { + WiFiClient client; + HTTPClient http; + + if (!start_downloadFile(client, http, url, file_save, user, pass, error)) { + return false; + } + + if (fileExists(file_save)) { + http.end(); + error = F("File exists: "); + error += file_save; + addLog(LOG_LEVEL_ERROR, error); + return false; + } long len = http.getSize(); fs::File f = tryOpenFile(file_save, "w"); @@ -1263,6 +1272,7 @@ bool downloadFile(const String& url, String file_save, const String& user, const const size_t downloadBuffSize = 256; uint8_t buff[downloadBuffSize]; size_t bytesWritten = 0; + unsigned long timeout = millis() + 2000; // get tcp stream WiFiClient *stream = &client; @@ -1306,14 +1316,103 @@ bool downloadFile(const String& url, String file_save, const String& user, const String log = F("downloadFile: "); log += file_save; log += F(" Success"); - addLog(LOG_LEVEL_INFO, log); + addLogMove(LOG_LEVEL_INFO, log); } return true; } + http.end(); error = F("Failed to open file for writing: "); error += file_save; addLog(LOG_LEVEL_ERROR, error); return false; } +bool downloadFirmware(const String& url, String& error) +{ + String file_save; + String user; + String pass; + WiFiClient client; + HTTPClient http; + client.setTimeout(2000); + + if (!start_downloadFile(client, http, url, file_save, user, pass, error)) { + return false; + } + + size_t len = http.getSize(); + + if (Update.begin(len, U_FLASH, Settings.Pin_status_led, Settings.Pin_status_led_Inversed ? LOW : HIGH)) { + const size_t downloadBuffSize = 256; + uint8_t buff[downloadBuffSize]; + size_t bytesWritten = 0; + unsigned long timeout = millis() + 2000; + + // get tcp stream + WiFiClient *stream = &client; + while (http.connected() && (len > 0 || len == -1)) { + // read up to downloadBuffSize at a time. + const size_t c = stream->readBytes(buff, std::min(static_cast(len), downloadBuffSize)); + + if (c > 0) { + timeout = millis() + 2000; + if (Update.write(buff, c) != c) { + error = F("Error saving firmware update: "); + error += file_save; + error += ' '; + error += bytesWritten; + error += F(" Bytes written"); + addLog(LOG_LEVEL_ERROR, error); + Update.end(); + http.end(); + return false; + } + bytesWritten += c; + if (len > 0) { len -= c; } + } + + if (timeOutReached(timeout)) { + error = F("Timeout: "); + error += file_save; + addLog(LOG_LEVEL_ERROR, error); + delay(0); + Update.end(); + http.end(); + return false; + } + if (!UseRTOSMultitasking) { + // On ESP32 the schedule is executed on the 2nd core. + Scheduler.handle_schedule(); + } + backgroundtasks(); + } + http.end(); + if (loglevelActiveFor(LOG_LEVEL_INFO)) { + String log = F("downloadFile: "); + log += file_save; + log += F(" Success"); + addLogMove(LOG_LEVEL_INFO, log); + } + if (Update.end()) { + if (Settings.UseRules) { + String event = F("ProvisionFirmware#success="); + event += file_save; + eventQueue.addMove(std::move(event)); + } + } + return true; + } + http.end(); + Update.end(); + error = F("Failed update firmware: "); + error += file_save; + addLog(LOG_LEVEL_ERROR, error); + if (Settings.UseRules) { + String event = F("ProvisionFirmware#failed="); + event += file_save; + eventQueue.addMove(std::move(event)); + } + return false; +} + #endif diff --git a/src/src/Helpers/Networking.h b/src/src/Helpers/Networking.h index 5e4a677a5..dd5c53cbe 100644 --- a/src/src/Helpers/Networking.h +++ b/src/src/Helpers/Networking.h @@ -163,6 +163,8 @@ bool downloadFile(const String& url, String file_save); bool downloadFile(const String& url, String file_save, const String& user, const String& pass, String& error); +bool downloadFirmware(const String& url, String& error); + #endif From 6bcfb4275bda8753caae5a6f416ac4f018c6a671 Mon Sep 17 00:00:00 2001 From: TD-er Date: Mon, 16 May 2022 21:34:15 +0200 Subject: [PATCH 02/14] [Provisioning] Fix missing includes --- src/src/Commands/Provisioning.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/src/Commands/Provisioning.cpp b/src/src/Commands/Provisioning.cpp index 018e16e00..4ee832fdf 100644 --- a/src/src/Commands/Provisioning.cpp +++ b/src/src/Commands/Provisioning.cpp @@ -9,7 +9,8 @@ # include "../DataTypes/ESPEasyFileType.h" # include "../DataStructs/ESPEasy_EventStruct.h" # include "../Helpers/ESPEasy_Storage.h" - +# include "../Helpers/Networking.h" +# include "../Helpers/StringConverter.h" String Command_Provisioning_Config(struct EventStruct *event, const char *Line) { From c4d430e7647c6beda547445e6fedefd7eb747fe5 Mon Sep 17 00:00:00 2001 From: TD-er Date: Tue, 17 May 2022 18:42:50 +0200 Subject: [PATCH 03/14] [Build] Fix missing line in .ini file due to merge issues --- src/src/Helpers/Networking.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index c47932553..aab3a1ce5 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -5,10 +5,12 @@ #include "../DataStructs/TimingStats.h" #include "../DataTypes/EventValueSource.h" #include "../ESPEasyCore/ESPEasy_Log.h" +#include "../ESPEasyCore/ESPEasy_backgroundtasks.h" #include "../ESPEasyCore/ESPEasyNetwork.h" #include "../ESPEasyCore/ESPEasyWifi.h" #include "../Globals/ESPEasyWiFiEvent.h" #include "../Globals/ESPEasy_Scheduler.h" +#include "../Globals/EventQueue.h" #include "../Globals/NetworkState.h" #include "../Globals/Nodes.h" #include "../Globals/Settings.h" From 3bc446d5cf7b699a6243801d14e7f01c83ac9ac6 Mon Sep 17 00:00:00 2001 From: TD-er Date: Tue, 17 May 2022 18:55:23 +0200 Subject: [PATCH 04/14] [PlatformIO] Update to PIO 6.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 84a73a241..4c39156ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ MarkupSafe==2.0.1 marshmallow==3.14.0 packaging==21.0 pathtools==0.1.2 -platformio<6 +platformio>=6.0.1 port-for==0.6.1 pycparser==2.20 pyelftools==0.27 From 86fb8afa709d16f976c5cdd2872470bc8853da47 Mon Sep 17 00:00:00 2001 From: TD-er Date: Tue, 17 May 2022 21:15:11 +0200 Subject: [PATCH 05/14] [PlatformIO] Clean requirements.txt files to only the essentials All other dependencies will be defined by the packages and installed by pip. --- docs/requirements.txt | 34 +++------------------------------- requirements.txt | 40 ++-------------------------------------- 2 files changed, 5 insertions(+), 69 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 7564f24cc..90474f4ec 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,32 +1,4 @@ -Jinja2==3.0.2 -MarkupSafe==2.0.1 -Pygments==2.10.0 -Sphinx==4.2.0 -alabaster==0.7.12 -babel==2.9.1 -certifi==2021.10.8 -charset-normalizer==2.0.7 -colorama==0.4.4 -commonmark==0.9.1 -docutils==0.17.1 -idna==3.3 -imagesize==1.2.0 -livereload==2.6.3 -packaging==21.0 -pyparsing==2.4.7 -pytz==2021.3 -recommonmark==0.7.1 -requests==2.26.0 -six==1.16.0 -snowballstemmer==2.1.0 +Sphinx==4.5.0 sphinx-autobuild==2021.3.14 -sphinx-bootstrap-theme==0.8.0 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -sphinxcontrib-websupport==1.2.4 -tornado==6.1 -urllib3==1.26.7 \ No newline at end of file +sphinx-bootstrap-theme==0.8.1 +recommonmark==0.7.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4c39156ae..b76df45a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,39 +1,3 @@ -alabaster==0.7.12 -argh==0.26.2 -Babel==2.9.1 -bottle==0.12.19 -cached-property==1.5.2 -certifi==2021.10.8 -cffi==1.15.0 -chardet==4.0.0 -click==8.0.3 -colorama==0.4.4 -commonmark==0.9.1 -docutils==0.17.1 -esptool==3.2 -idna==3.3 -imagesize==1.2.0 -Jinja2==3.0.2 -livereload==2.6.3 -MarkupSafe==2.0.1 -marshmallow==3.14.0 -packaging==21.0 -pathtools==0.1.2 +esptool==4.0 platformio>=6.0.1 -port-for==0.6.1 -pycparser==2.20 -pyelftools==0.27 -pygit2==1.7.0 -Pygments==2.10.0 -pyparsing==2.4.7 -pyserial==3.5 -pytz==2021.3 -PyYAML==6.0 -recommonmark==0.7.1 -requests==2.26.0 -six==1.16.0 -snowballstemmer==2.1.0 -tabulate==0.8.9 -tornado==6.1 -urllib3==1.26.7 -watchdog==2.1.6 +pygit2==1.9.1 \ No newline at end of file From d2039aedfb13739980bb335d6d9781f43712a1c7 Mon Sep 17 00:00:00 2001 From: TD-er Date: Tue, 17 May 2022 21:32:48 +0200 Subject: [PATCH 06/14] [Provisioning] Fix missing include --- src/ESPEasy-Globals.h | 3 ++- src/src/Helpers/Networking.cpp | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ESPEasy-Globals.h b/src/ESPEasy-Globals.h index e728cdc21..354a9bcbe 100644 --- a/src/ESPEasy-Globals.h +++ b/src/ESPEasy-Globals.h @@ -97,7 +97,8 @@ extern float vcc; extern bool shouldReboot; extern bool firstLoop; - +// This is read from the settings at boot. +// Even if this setting is changed, you need to reboot to activate the changes. extern boolean UseRTOSMultitasking; #endif /* ESPEASY_GLOBALS_H_ */ diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index aab3a1ce5..4a7fc2b65 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -22,6 +22,8 @@ #include "../Helpers/StringConverter.h" #include "../Helpers/StringProvider.h" +#include "../../ESPEasy-Globals.h" + #include // Generic Networking routines From 03c592f2b2121352fed41a45066719e2017b0708 Mon Sep 17 00:00:00 2001 From: TD-er Date: Sat, 11 Jun 2022 15:53:27 +0200 Subject: [PATCH 07/14] [Firmware Download] Fix downloading with no content length set --- src/src/Helpers/Networking.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index 4a7fc2b65..3660a9886 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -1284,7 +1284,11 @@ bool downloadFile(const String& url, String file_save, const String& user, const // read all data from server while (http.connected() && (len > 0 || len == -1)) { // read up to downloadBuffSize at a time. - const size_t c = stream->readBytes(buff, std::min(static_cast(len), downloadBuffSize)); + size_t bytes_to_read = downloadBuffSize; + if (len > 0 && len < bytes_to_read) { + bytes_to_read = len; + } + const size_t c = stream->readBytes(buff, bytes_to_read); if (c > 0) { timeout = millis() + 2000; @@ -1344,7 +1348,7 @@ bool downloadFirmware(const String& url, String& error) return false; } - size_t len = http.getSize(); + int len = http.getSize(); if (Update.begin(len, U_FLASH, Settings.Pin_status_led, Settings.Pin_status_led_Inversed ? LOW : HIGH)) { const size_t downloadBuffSize = 256; @@ -1356,7 +1360,11 @@ bool downloadFirmware(const String& url, String& error) WiFiClient *stream = &client; while (http.connected() && (len > 0 || len == -1)) { // read up to downloadBuffSize at a time. - const size_t c = stream->readBytes(buff, std::min(static_cast(len), downloadBuffSize)); + size_t bytes_to_read = downloadBuffSize; + if (len > 0 && len < bytes_to_read) { + bytes_to_read = len; + } + const size_t c = stream->readBytes(buff, bytes_to_read); if (c > 0) { timeout = millis() + 2000; From 0777b0679b46d2fd6a0cc6c25131b4f38c27c313 Mon Sep 17 00:00:00 2001 From: TD-er Date: Sat, 11 Jun 2022 16:48:02 +0200 Subject: [PATCH 08/14] [Download] Fix warning signed/unsigned --- src/src/Helpers/Networking.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index 3660a9886..39598ee9a 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -1285,7 +1285,7 @@ bool downloadFile(const String& url, String file_save, const String& user, const while (http.connected() && (len > 0 || len == -1)) { // read up to downloadBuffSize at a time. size_t bytes_to_read = downloadBuffSize; - if (len > 0 && len < bytes_to_read) { + if (len > 0 && len < static_cast(bytes_to_read)) { bytes_to_read = len; } const size_t c = stream->readBytes(buff, bytes_to_read); @@ -1361,7 +1361,7 @@ bool downloadFirmware(const String& url, String& error) while (http.connected() && (len > 0 || len == -1)) { // read up to downloadBuffSize at a time. size_t bytes_to_read = downloadBuffSize; - if (len > 0 && len < bytes_to_read) { + if (len > 0 && len < static_cast(bytes_to_read)) { bytes_to_read = len; } const size_t c = stream->readBytes(buff, bytes_to_read); From c25a3286e799f809fce0ff2fcdf673253820e163 Mon Sep 17 00:00:00 2001 From: TD-er Date: Sat, 23 Jul 2022 15:28:04 +0200 Subject: [PATCH 09/14] [Download] Update using new HTTP Digest Auth implementation --- src/src/DataTypes/ESPEasyFileType.cpp | 1 + src/src/Helpers/Hardware.cpp | 1 + src/src/Helpers/Networking.cpp | 497 +++++++++++++++++---- src/src/Helpers/Networking.h | 42 +- src/src/Helpers/StringGenerator_System.cpp | 1 + src/src/Helpers/_CPlugin_Helper.cpp | 275 +----------- src/src/Helpers/_CPlugin_Helper.h | 18 +- src/src/WebServer/SettingsArchive.cpp | 1 + 8 files changed, 467 insertions(+), 369 deletions(-) diff --git a/src/src/DataTypes/ESPEasyFileType.cpp b/src/src/DataTypes/ESPEasyFileType.cpp index a690b29cd..d659ae319 100644 --- a/src/src/DataTypes/ESPEasyFileType.cpp +++ b/src/src/DataTypes/ESPEasyFileType.cpp @@ -71,6 +71,7 @@ bool getDownloadFiletypeChecked(FileType::Enum filetype, unsigned int filenr) { case FileType::PROVISIONING_DAT: isChecked = ResetFactoryDefaultPreference.fetchProvisioningDat(); break; break; + case FileType::FIRMWARE: // FIXME TD-er: Must decide what to do with firmware description/protection on provisioning settings case FileType::MAX_FILETYPE: break; } diff --git a/src/src/Helpers/Hardware.cpp b/src/src/Helpers/Hardware.cpp index 5a6367080..4491ee86c 100644 --- a/src/src/Helpers/Hardware.cpp +++ b/src/src/Helpers/Hardware.cpp @@ -1085,6 +1085,7 @@ void readBootCause() { case TG1WDT_CPU_RESET: lastBootCause = BOOT_CAUSE_EXT_WD; break; case SUPER_WDT_RESET: lastBootCause = BOOT_CAUSE_EXT_WD; break; case GLITCH_RTC_RESET: lastBootCause = BOOT_CAUSE_POWER_UNSTABLE; break; // FIXME TD-er: Does this need a different reason? + case EFUSE_RESET: break; // FIXME TD-er: No idea what may cause this reset reason. # endif // ifdef ESP32S2 } } diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index c1d9897d1..485850806 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -2,6 +2,7 @@ #include "../../ESPEasy_common.h" #include "../Commands/InternalCommands.h" +#include "../CustomBuild/CompiletimeDefines.h" #include "../DataStructs/TimingStats.h" #include "../DataTypes/EventValueSource.h" #include "../ESPEasyCore/ESPEasy_Log.h" @@ -67,7 +68,7 @@ void etharp_gratuitous_r(struct netif *netif) { # include # include # endif // ifdef ESP32 -#endif +#endif // ifdef USE_DOWNLOAD #include @@ -81,6 +82,7 @@ void sendSyslog(uint8_t logLevel, const String& message) IPAddress broadcastIP(Settings.Syslog_IP[0], Settings.Syslog_IP[1], Settings.Syslog_IP[2], Settings.Syslog_IP[3]); FeedSW_watchdog(); + if (portUDP.beginPacket(broadcastIP, Settings.SyslogPort) == 0) { // problem resolving the hostname or port return; @@ -112,7 +114,7 @@ void sendSyslog(uint8_t logLevel, const String& message) header += hostname; header += F(" EspEasy: "); #ifdef ESP8266 - portUDP.write(header.c_str(), header.length()); + portUDP.write(header.c_str(), header.length()); #endif // ifdef ESP8266 #ifdef ESP32 portUDP.write(reinterpret_cast(header.c_str()), header.length()); @@ -120,6 +122,7 @@ void sendSyslog(uint8_t logLevel, const String& message) } const size_t messageLength = message.length(); + for (size_t i = 0; i < messageLength; ++i) { #ifdef ESP8266 portUDP.write(message[i]); @@ -134,7 +137,6 @@ void sendSyslog(uint8_t logLevel, const String& message) } } - #if FEATURE_ESPEASY_P2P /*********************************************************************************************\ @@ -145,6 +147,7 @@ void SendUDPCommand(uint8_t destUnit, const char *data, uint8_t dataLength) if (!NetworkConnected(10)) { return; } + if (destUnit != 0) { sendUDP(destUnit, (const uint8_t *)data, dataLength); @@ -175,14 +178,14 @@ void sendUDP(uint8_t unit, const uint8_t *data, uint8_t size) return; } -#ifndef BUILD_NO_DEBUG +# ifndef BUILD_NO_DEBUG if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) { String log = F("UDP : Send UDP message to "); log += unit; addLogMove(LOG_LEVEL_DEBUG_MORE, log); } -#endif // ifndef BUILD_NO_DEBUG +# endif // ifndef BUILD_NO_DEBUG statusLED(true); FeedSW_watchdog(); @@ -295,9 +298,9 @@ void checkUDP() break; } uint8_t unit = packetBuffer[12]; -#ifndef BUILD_NO_DEBUG +# ifndef BUILD_NO_DEBUG MAC_address mac; - uint8_t ip[4]; + uint8_t ip[4]; for (uint8_t x = 0; x < 6; x++) { mac.mac[x] = packetBuffer[x + 2]; @@ -306,12 +309,13 @@ void checkUDP() for (uint8_t x = 0; x < 4; x++) { ip[x] = packetBuffer[x + 8]; } -#endif // ifndef BUILD_NO_DEBUG +# endif // ifndef BUILD_NO_DEBUG { - #ifdef USE_SECOND_HEAP + # ifdef USE_SECOND_HEAP HeapSelectIram ephemeral; + // TD-er: Disabled for now as it is suspect for crashes. - #endif + # endif // ifdef USE_SECOND_HEAP Nodes[unit].age = 0; // Create a new element when not present } @@ -328,11 +332,11 @@ void checkUDP() it->second.build = makeWord(packetBuffer[14], packetBuffer[13]); char tmpNodeName[26] = { 0 }; memcpy(&tmpNodeName[0], reinterpret_cast(&packetBuffer[15]), 25); - tmpNodeName[25] = 0; + tmpNodeName[25] = 0; { - #ifdef USE_SECOND_HEAP + # ifdef USE_SECOND_HEAP HeapSelectIram ephemeral; - #endif + # endif // ifdef USE_SECOND_HEAP it->second.nodeName = tmpNodeName; it->second.nodeName.trim(); @@ -346,7 +350,7 @@ void checkUDP() } } -#ifndef BUILD_NO_DEBUG +# ifndef BUILD_NO_DEBUG if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) { String log; @@ -358,7 +362,7 @@ void checkUDP() log += unit; addLogMove(LOG_LEVEL_DEBUG_MORE, log); } -#endif // ifndef BUILD_NO_DEBUG +# endif // ifndef BUILD_NO_DEBUG break; } @@ -487,9 +491,9 @@ void sendSysInfoUDP(uint8_t repeats) // 1 uint8_t node type id // send my info to the world... -#ifndef BUILD_NO_DEBUG +# ifndef BUILD_NO_DEBUG addLog(LOG_LEVEL_DEBUG_MORE, F("UDP : Send Sysinfo message")); -#endif // ifndef BUILD_NO_DEBUG +# endif // ifndef BUILD_NO_DEBUG for (uint8_t counter = 0; counter < repeats; counter++) { @@ -499,6 +503,7 @@ void sendSysInfoUDP(uint8_t repeats) { const MAC_address macread = NetworkMacAddress(); + for (uint8_t x = 0; x < 6; x++) { data[x + 2] = macread.mac[x]; } @@ -506,6 +511,7 @@ void sendSysInfoUDP(uint8_t repeats) { const IPAddress ip = NetworkLocalIP(); + for (uint8_t x = 0; x < 4; x++) { data[x + 8] = ip[x]; } @@ -532,13 +538,15 @@ void sendSysInfoUDP(uint8_t repeats) } { - #ifdef USE_SECOND_HEAP + # ifdef USE_SECOND_HEAP + // HeapSelectIram ephemeral; // TD-er: disabled for now as it is suspect for crashes. - #endif + # endif // ifdef USE_SECOND_HEAP Nodes[Settings.Unit].age = 0; // Create new node when not already present. } + // store my own info also in the list NodesMap::iterator it = Nodes.find(Settings.Unit); @@ -579,42 +587,42 @@ void SSDP_schema(WiFiClient& client) { (uint16_t)chipId & 0xff); client.print(F( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/xml\r\n" - "Connection: close\r\n" - "Access-Control-Allow-Origin: *\r\n" - "\r\n" - "" - "" - "" - "1" - "0" - "" - "http://")); + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/xml\r\n" + "Connection: close\r\n" + "Access-Control-Allow-Origin: *\r\n" + "\r\n" + "" + "" + "" + "1" + "0" + "" + "http://")); client.print(formatIP(ip)); client.print(F(":80/" - "" - "urn:schemas-upnp-org:device:BinaryLight:1" - "")); + "" + "urn:schemas-upnp-org:device:BinaryLight:1" + "")); client.print(Settings.Name); client.print(F("" - "/" - "")); + "/" + "")); client.print(String(ESP.getChipId())); client.print(F("" - "ESP Easy" - "")); + "ESP Easy" + "")); client.print(getValue(LabelType::GIT_BUILD)); client.print(F("" - "http://www.letscontrolit.com" - "http://www.letscontrolit.com" - "http://www.letscontrolit.com" - "uuid:")); + "http://www.letscontrolit.com" + "http://www.letscontrolit.com" + "http://www.letscontrolit.com" + "uuid:")); client.print(String(uuid)); client.print(F("" - "\r\n" - "\r\n")); + "\r\n" + "\r\n")); } /********************************************************************************************\ @@ -649,12 +657,14 @@ bool SSDP_begin() { if (_server != nullptr) { _server->unref(); + // FIXME TD-er: Shouldn't this also call delete _server ? - _server = nullptr; + _server = nullptr; } _server = new (std::nothrow) UdpContext; + if (_server == nullptr) { return false; } @@ -1091,6 +1101,7 @@ bool hostReachable(const String& hostname) { if (resolveHostByName(hostname.c_str(), remote_addr)) { return hostReachable(remote_addr); } + if (loglevelActiveFor(LOG_LEVEL_ERROR)) { String log = F("Hostname cannot be resolved: "); @@ -1187,8 +1198,307 @@ String splitURL(const String& fullURL, String& host, uint16_t& port, String& fil return fullURL.substring(endhost); } +String get_user_agent_string() { + static unsigned int agent_size = 20; + String userAgent; + + userAgent.reserve(agent_size); + userAgent += F("ESP Easy/"); + userAgent += BUILD; + userAgent += '/'; + userAgent += get_build_date(); + userAgent += ' '; + userAgent += get_build_time(); + agent_size = userAgent.length(); + return userAgent; +} + +bool splitHeaders(int& strpos, const String& multiHeaders, String& name, String& value) { + if (strpos < 0) { + return false; + } + int colonPos = multiHeaders.indexOf(':', strpos); + + if (colonPos < 0) { + return false; + } + name = multiHeaders.substring(strpos, colonPos); + int valueEndPos = multiHeaders.indexOf('\n', colonPos + 1); + + if (valueEndPos < 0) { + value = multiHeaders.substring(colonPos + 1); + strpos = -1; + } else { + value = multiHeaders.substring(colonPos + 1, valueEndPos); + strpos = valueEndPos + 1; + } + value.replace('\r', ' '); + value.trim(); + return true; +} + +String extractParam(const String& authReq, const String& param, const char delimit) { + int _begin = authReq.indexOf(param); + + if (_begin == -1) { return EMPTY_STRING; } + return authReq.substring(_begin + param.length(), authReq.indexOf(delimit, _begin + param.length())); +} + +String getCNonce(const int len) { + static const char alphanum[] = "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + String s; + + for (int i = 0; i < len; ++i) { + s += alphanum[rand() % (sizeof(alphanum) - 1)]; + } + + return s; +} + +String getDigestAuth(const String& authReq, + const String& username, + const String& password, + const String& method, + const String& uri, + unsigned int counter) { + // extracting required parameters for RFC 2069 simpler Digest + const String realm = extractParam(authReq, F("realm=\""), '"'); + const String nonce = extractParam(authReq, F("nonce=\""), '"'); + const String cNonce = getCNonce(8); + + char nc[9]; + + snprintf(nc, sizeof(nc), "%08x", counter); + + // parameters for the RFC 2617 newer Digest + MD5Builder md5; + + md5.begin(); + md5.add(username + ':' + realm + ':' + password); // md5 of the user:realm:user + md5.calculate(); + const String h1 = md5.toString(); + + md5.begin(); + md5.add(method + ':' + uri); + md5.calculate(); + const String h2 = md5.toString(); + + md5.begin(); + md5.add(h1 + ':' + nonce + ':' + String(nc) + ':' + cNonce + F(":auth:") + h2); + md5.calculate(); + const String response = md5.toString(); + + const String authorization = + String(F("Digest username=\"")) + username + + F("\", realm=\"") + realm + + F("\", nonce=\"") + nonce + + F("\", uri=\"") + uri + + F("\", algorithm=\"MD5\", qop=auth, nc=") + String(nc) + + F(", cnonce=\"") + cNonce + + F("\", response=\"") + response + + '"'; + + // Serial.println(authorization); + + return authorization; +} + +void log_http_result(const HTTPClient& http, + const String & logIdentifier, + const String & HttpMethod, + int httpCode, + const String & response) +{ + uint8_t loglevel = LOG_LEVEL_ERROR; + bool success = false; + + // HTTP codes: + // 1xx Informational response + // 2xx Success + if ((httpCode >= 100) && (httpCode < 300)) { + loglevel = LOG_LEVEL_INFO; + success = true; + } + + if (loglevelActiveFor(loglevel)) { + String log = F("HTTP : "); + log += logIdentifier; + log += ' '; + log += HttpMethod; + log += F("... "); + + if (!success) { + log += F("failed "); + } + log += F("HTTP code: "); + log += String(httpCode); + + if (!success) { + log += ' '; + log += http.errorToString(httpCode); + } + + if (response.length() > 0) { + log += ' '; + log += response.substring(0, 100); // Returned string may be huge, so only log the first part. + } + addLogMove(loglevel, log); + } +} + +int http_authenticate(const String& logIdentifier, + WiFiClient & client, + HTTPClient & http, + uint16_t timeout, + const String& user, + const String& pass, + const String& host, + uint16_t port, + const String& uri, + const String& HttpMethod, + const String& header, + const String& postStr) +{ + int httpCode = 0; + + http.setAuthorization(user.c_str(), pass.c_str()); + http.setTimeout(timeout); + http.setUserAgent(get_user_agent_string()); + + #ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS + + // See: https://github.com/espressif/arduino-esp32/pull/6676 + client.setTimeout((timeout + 500) / 1000); // in seconds!!!! + #else // ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS + client.setTimeout(timeout); // in msec as it should be! + #endif // ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS + + // Add request header as fall back. + // When adding another "accept" header, it may be interpreted as: + // "if you have XXX, send it; or failing that, just give me what you've got." + http.addHeader(F("Accept"), F("*/*;q=0.1")); + + delay(0); +#if defined(CORE_POST_2_6_0) || defined(ESP32) + http.begin(client, host, port, uri, false); // HTTP +#else // if defined(CORE_POST_2_6_0) || defined(ESP32) + http.begin(client, host, port, uri); +#endif // if defined(CORE_POST_2_6_0) || defined(ESP32) + + const char *keys[] = { "WWW-Authenticate" }; + http.collectHeaders(keys, 1); + + { + int headerpos = 0; + String name, value; + + while (splitHeaders(headerpos, header, name, value)) { + http.addHeader(name, value); + } + } + + // start connection and send HTTP header (and body) + if (HttpMethod.equals(F("HEAD")) || HttpMethod.equals(F("GET"))) { + httpCode = http.sendRequest(HttpMethod.c_str()); + } else { + httpCode = http.sendRequest(HttpMethod.c_str(), postStr); + } + + // Check to see if we need to try digest auth + if (httpCode == 401) { + const String authReq = http.header(String(F("WWW-Authenticate")).c_str()); + + if (authReq.indexOf(F("Digest")) != -1) { + // Use Digest authorization + if (loglevelActiveFor(LOG_LEVEL_INFO)) { + addLogMove(LOG_LEVEL_INFO, String(F("HTTP : Start Digest Authorization for ")) + host); + } + + http.setAuthorization(""); // Clear Basic authorization + const String authorization = getDigestAuth(authReq, user, pass, "GET", uri, 1); + + http.end(); +#if defined(CORE_POST_2_6_0) || defined(ESP32) + http.begin(client, host, port, uri, false); // HTTP, not HTTPS +#else // if defined(CORE_POST_2_6_0) || defined(ESP32) + http.begin(client, host, port, uri); +#endif // if defined(CORE_POST_2_6_0) || defined(ESP32) + + http.addHeader(F("Authorization"), authorization); + + // start connection and send HTTP header (and body) + if (HttpMethod.equals(F("HEAD")) || HttpMethod.equals(F("GET"))) { + httpCode = http.sendRequest(HttpMethod.c_str()); + } else { + httpCode = http.sendRequest(HttpMethod.c_str(), postStr); + } + } + } + + if (Settings.UseRules) { + // Generate event with the HTTP return code + // e.g. http#hostname=401 + String event = F("http#"); + event += host; + event += '='; + event += httpCode; + eventQueue.addMove(std::move(event)); + } + log_http_result(http, logIdentifier, HttpMethod, httpCode, EMPTY_STRING); + return httpCode; +} + +String send_via_http(const String& logIdentifier, + WiFiClient & client, + uint16_t timeout, + const String& user, + const String& pass, + const String& host, + uint16_t port, + const String& uri, + const String& HttpMethod, + const String& header, + const String& postStr, + int & httpCode, + bool must_check_reply) { + HTTPClient http; + + httpCode = http_authenticate( + logIdentifier, + client, + http, + timeout, + user, + pass, + host, + port, + uri, + HttpMethod, + header, + postStr); + + String response; + + if ((httpCode > 0) && must_check_reply) { + response = http.getString(); + + if (!response.isEmpty()) { + log_http_result(http, logIdentifier, HttpMethod, httpCode, response); + } + } + http.end(); + return response; +} + #ifdef USE_DOWNLOAD +// FIXME TD-er: Must set the timeout somewhere +# ifndef DOWNLOAD_FILE_TIMEOUT + # define DOWNLOAD_FILE_TIMEOUT 2000 +# endif // ifndef DOWNLOAD_FILE_TIMEOUT + // Download a file from a given URL and save to a local file named "file_save" // If the URL ends with a /, the file part will be assumed the same as file_save. // If file_save is empty, the file part from the URL will be used as local file name. @@ -1199,7 +1509,13 @@ bool downloadFile(const String& url, String file_save) { return downloadFile(url, file_save, EMPTY_STRING, EMPTY_STRING, error); } -bool start_downloadFile(WiFiClient& client, HTTPClient& http, const String& url, String& file_save, const String& user, const String& pass, String& error) { +bool start_downloadFile(WiFiClient & client, + HTTPClient & http, + const String& url, + String & file_save, + const String& user, + const String& pass, + String & error) { String host, file; uint16_t port; String uri = splitURL(url, host, port, file); @@ -1210,8 +1526,9 @@ bool start_downloadFile(WiFiClient& client, HTTPClient& http, const String& url, // file = file_save; uri += file_save; } +# ifndef BUILD_NO_DEBUG - if (loglevelActiveFor(LOG_LEVEL_INFO)) { + if (loglevelActiveFor(LOG_LEVEL_DEBUG)) { String log = F("downloadFile: URL: "); log += url; log += F(" decoded: "); @@ -1219,8 +1536,9 @@ bool start_downloadFile(WiFiClient& client, HTTPClient& http, const String& url, log += ':'; log += port; log += uri; - addLogMove(LOG_LEVEL_ERROR, log); + addLogMove(LOG_LEVEL_DEBUG, log); } +# endif // ifndef BUILD_NO_DEBUG if (file_save.isEmpty()) { error = F("Empty filename"); @@ -1228,17 +1546,20 @@ bool start_downloadFile(WiFiClient& client, HTTPClient& http, const String& url, return false; } - http.begin(client, host, port, uri); - { - if ((user.length() > 0) && (pass.length() > 0)) { - http.setAuthorization(user.c_str(), pass.c_str()); - } - - /* - http.setAuthorization(user, pass); - */ - } - int httpCode = http.GET(); + int httpCode = http_authenticate( + F("DownloadFile"), + client, + http, + DOWNLOAD_FILE_TIMEOUT, + user, + pass, + host, + port, + uri, + F("GET"), + EMPTY_STRING, // header + EMPTY_STRING // postStr + ); if (httpCode != HTTP_CODE_OK) { error = F("HTTP code: "); @@ -1254,29 +1575,28 @@ bool start_downloadFile(WiFiClient& client, HTTPClient& http, const String& url, } bool downloadFile(const String& url, String file_save, const String& user, const String& pass, String& error) { - WiFiClient client; - HTTPClient http; + WiFiClient client; + HTTPClient http; if (!start_downloadFile(client, http, url, file_save, user, pass, error)) { return false; } if (fileExists(file_save)) { - http.end(); - error = F("File exists: "); + error = F("File exists: "); error += file_save; addLog(LOG_LEVEL_ERROR, error); return false; } - long len = http.getSize(); - fs::File f = tryOpenFile(file_save, "w"); + long len = http.getSize(); + fs::File f = tryOpenFile(file_save, "w"); if (f) { const size_t downloadBuffSize = 256; uint8_t buff[downloadBuffSize]; - size_t bytesWritten = 0; - unsigned long timeout = millis() + 2000; + size_t bytesWritten = 0; + unsigned long timeout = millis() + DOWNLOAD_FILE_TIMEOUT; // get tcp stream WiFiClient *stream = &client; @@ -1285,13 +1605,14 @@ bool downloadFile(const String& url, String file_save, const String& user, const while (http.connected() && (len > 0 || len == -1)) { // read up to downloadBuffSize at a time. size_t bytes_to_read = downloadBuffSize; - if (len > 0 && len < static_cast(bytes_to_read)) { + + if ((len > 0) && (len < static_cast(bytes_to_read))) { bytes_to_read = len; } const size_t c = stream->readBytes(buff, bytes_to_read); if (c > 0) { - timeout = millis() + 2000; + timeout = millis() + DOWNLOAD_FILE_TIMEOUT; if (f.write(buff, c) != c) { error = F("Error saving file: "); @@ -1309,7 +1630,7 @@ bool downloadFile(const String& url, String file_save, const String& user, const } if (timeOutReached(timeout)) { - error = F("Timeout: "); + error = F("Timeout: "); error += file_save; addLog(LOG_LEVEL_ERROR, error); delay(0); @@ -1320,6 +1641,7 @@ bool downloadFile(const String& url, String file_save, const String& user, const } f.close(); http.end(); + if (loglevelActiveFor(LOG_LEVEL_INFO)) { String log = F("downloadFile: "); log += file_save; @@ -1329,7 +1651,7 @@ bool downloadFile(const String& url, String file_save, const String& user, const return true; } http.end(); - error = F("Failed to open file for writing: "); + error = F("Failed to open file for writing: "); error += file_save; addLog(LOG_LEVEL_ERROR, error); return false; @@ -1337,12 +1659,11 @@ bool downloadFile(const String& url, String file_save, const String& user, const bool downloadFirmware(const String& url, String& error) { - String file_save; - String user; - String pass; + String file_save; + String user; + String pass; WiFiClient client; HTTPClient http; - client.setTimeout(2000); if (!start_downloadFile(client, http, url, file_save, user, pass, error)) { return false; @@ -1353,21 +1674,24 @@ bool downloadFirmware(const String& url, String& error) if (Update.begin(len, U_FLASH, Settings.Pin_status_led, Settings.Pin_status_led_Inversed ? LOW : HIGH)) { const size_t downloadBuffSize = 256; uint8_t buff[downloadBuffSize]; - size_t bytesWritten = 0; - unsigned long timeout = millis() + 2000; + size_t bytesWritten = 0; + unsigned long timeout = millis() + DOWNLOAD_FILE_TIMEOUT; // get tcp stream WiFiClient *stream = &client; + while (http.connected() && (len > 0 || len == -1)) { // read up to downloadBuffSize at a time. size_t bytes_to_read = downloadBuffSize; - if (len > 0 && len < static_cast(bytes_to_read)) { + + if ((len > 0) && (len < static_cast(bytes_to_read))) { bytes_to_read = len; } const size_t c = stream->readBytes(buff, bytes_to_read); if (c > 0) { - timeout = millis() + 2000; + timeout = millis() + DOWNLOAD_FILE_TIMEOUT; + if (Update.write(buff, c) != c) { error = F("Error saving firmware update: "); error += file_save; @@ -1380,11 +1704,12 @@ bool downloadFirmware(const String& url, String& error) return false; } bytesWritten += c; + if (len > 0) { len -= c; } } if (timeOutReached(timeout)) { - error = F("Timeout: "); + error = F("Timeout: "); error += file_save; addLog(LOG_LEVEL_ERROR, error); delay(0); @@ -1392,6 +1717,7 @@ bool downloadFirmware(const String& url, String& error) http.end(); return false; } + if (!UseRTOSMultitasking) { // On ESP32 the schedule is executed on the 2nd core. Scheduler.handle_schedule(); @@ -1399,32 +1725,35 @@ bool downloadFirmware(const String& url, String& error) backgroundtasks(); } http.end(); + if (loglevelActiveFor(LOG_LEVEL_INFO)) { String log = F("downloadFile: "); log += file_save; log += F(" Success"); addLogMove(LOG_LEVEL_INFO, log); } + if (Update.end()) { if (Settings.UseRules) { String event = F("ProvisionFirmware#success="); event += file_save; - eventQueue.addMove(std::move(event)); + eventQueue.addMove(std::move(event)); } } return true; } http.end(); Update.end(); - error = F("Failed update firmware: "); + error = F("Failed update firmware: "); error += file_save; addLog(LOG_LEVEL_ERROR, error); + if (Settings.UseRules) { String event = F("ProvisionFirmware#failed="); event += file_save; - eventQueue.addMove(std::move(event)); + eventQueue.addMove(std::move(event)); } return false; } -#endif +#endif // ifdef USE_DOWNLOAD diff --git a/src/src/Helpers/Networking.h b/src/src/Helpers/Networking.h index 775dbbc13..d4353ade7 100644 --- a/src/src/Helpers/Networking.h +++ b/src/src/Helpers/Networking.h @@ -3,10 +3,18 @@ #include "../../ESPEasy_common.h" - +#include #include #include +#ifdef ESP8266 +# include +#endif // ifdef ESP8266 +#ifdef ESP32 +# include +#endif // ifdef ESP32 + + /*********************************************************************************************\ Syslog client \*********************************************************************************************/ @@ -156,6 +164,38 @@ bool splitHostPortString(const String& hostPortString, String& host, uint16_t& p // Return value is everything after the hostname:port section (including /) String splitURL(const String& fullURL, String& host, uint16_t& port, String& file); + +// Initiate the HTTP connection. +// Also try to authenticate using either Basic auth or Digest. +// @retval HTTP return code. +int http_authenticate(const String& logIdentifier, + WiFiClient & client, + HTTPClient & http, + uint16_t timeout, + const String& user, + const String& pass, + const String& host, + uint16_t port, + const String& uri, + const String& HttpMethod, + const String& header, + const String& postStr); + + +String send_via_http(const String& logIdentifier, + WiFiClient & client, + uint16_t timeout, + const String& user, + const String& pass, + const String& host, + uint16_t port, + const String& uri, + const String& HttpMethod, + const String& header, + const String& postStr, + int & httpCode, + bool must_check_reply); + #ifdef USE_DOWNLOAD // Download a file from a given URL and save to a local file named "file_save" diff --git a/src/src/Helpers/StringGenerator_System.cpp b/src/src/Helpers/StringGenerator_System.cpp index 44316ff9c..80dc95e06 100644 --- a/src/src/Helpers/StringGenerator_System.cpp +++ b/src/src/Helpers/StringGenerator_System.cpp @@ -82,6 +82,7 @@ const __FlashStringHelper * getResetReasonString_f(uint8_t icore, bool& isDEEPSL case TG1WDT_CPU_RESET: return F("Time Group1 reset CPU"); // 17 case SUPER_WDT_RESET: return F("Super watchdog reset digital core and rtc module"); // 18 case GLITCH_RTC_RESET: return F("Glitch reset digital core and rtc module"); // 19 + case EFUSE_RESET: return F("EFUSE_RESET"); // FIXME TD-er: No idea what may cause this case NO_MEAN: break; // Undefined, "No Meaning" } diff --git a/src/src/Helpers/_CPlugin_Helper.cpp b/src/src/Helpers/_CPlugin_Helper.cpp index 2ac97821e..b69c97853 100644 --- a/src/src/Helpers/_CPlugin_Helper.cpp +++ b/src/src/Helpers/_CPlugin_Helper.cpp @@ -33,13 +33,6 @@ #include -#ifdef ESP8266 -# include -#endif // ifdef ESP8266 -#ifdef ESP32 -# include -#endif // ifdef ESP32 - bool safeReadStringUntil(Stream & input, String & str, char terminator, @@ -90,22 +83,6 @@ bool safeReadStringUntil(Stream & input, return false; } -String get_user_agent_string() { - static unsigned int agent_size = 20; - String userAgent; - - userAgent.reserve(agent_size); - userAgent += F("ESP Easy/"); - userAgent += BUILD; - userAgent += '/'; - userAgent += get_build_date(); - userAgent += ' '; - userAgent += get_build_time(); - agent_size = userAgent.length(); - return userAgent; -} - - #ifndef BUILD_NO_DEBUG void log_connecting_to(const __FlashStringHelper *prefix, int controller_number, ControllerSettingsStruct& ControllerSettings) { if (loglevelActiveFor(LOG_LEVEL_DEBUG)) { @@ -152,11 +129,12 @@ bool try_connect_host(int controller_number, WiFiUDP& client, ControllerSettings if (!NetworkConnected()) { return false; } #ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS + // See: https://github.com/espressif/arduino-esp32/pull/6676 client.setTimeout((ControllerSettings.ClientTimeout + 500) / 1000); // in seconds!!!! - #else - client.setTimeout(ControllerSettings.ClientTimeout); // in msec as it should be! - #endif + #else // ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS + client.setTimeout(ControllerSettings.ClientTimeout); // in msec as it should be! + #endif // ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS delay(0); #ifndef BUILD_NO_DEBUG log_connecting_to(F("UDP : "), controller_number, ControllerSettings); @@ -184,11 +162,12 @@ bool try_connect_host(int controller_number, // Use WiFiClient class to create TCP connections delay(0); #ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS + // See: https://github.com/espressif/arduino-esp32/pull/6676 client.setTimeout((ControllerSettings.ClientTimeout + 500) / 1000); // in seconds!!!! - #else - client.setTimeout(ControllerSettings.ClientTimeout); // in msec as it should be! - #endif + #else // ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS + client.setTimeout(ControllerSettings.ClientTimeout); // in msec as it should be! + #endif // ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS #ifndef BUILD_NO_DEBUG log_connecting_to(loglabel, controller_number, ControllerSettings); @@ -209,7 +188,6 @@ bool client_available(WiFiClient& client) { return (client.available() != 0) || (client.connected() != 0); } - String send_via_http(int controller_number, const ControllerSettingsStruct& ControllerSettings, controllerIndex_t controller_idx, @@ -246,246 +224,9 @@ String send_via_http(int controller_number, return result; } -bool splitHeaders(int& strpos, const String& multiHeaders, String& name, String& value) { - if (strpos < 0) { - return false; - } - int colonPos = multiHeaders.indexOf(':', strpos); - if (colonPos < 0) { - return false; - } - name = multiHeaders.substring(strpos, colonPos); - int valueEndPos = multiHeaders.indexOf('\n', colonPos + 1); - if (valueEndPos < 0) { - value = multiHeaders.substring(colonPos + 1); - strpos = -1; - } else { - value = multiHeaders.substring(colonPos + 1, valueEndPos); - strpos = valueEndPos + 1; - } - value.replace('\r', ' '); - value.trim(); - return true; -} -String extractParam(const String& authReq, const String& param, const char delimit) { - int _begin = authReq.indexOf(param); - - if (_begin == -1) { return EMPTY_STRING; } - return authReq.substring(_begin + param.length(), authReq.indexOf(delimit, _begin + param.length())); -} - -String getCNonce(const int len) { - static const char alphanum[] = "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; - String s; - - for (int i = 0; i < len; ++i) { - s += alphanum[rand() % (sizeof(alphanum) - 1)]; - } - - return s; -} - -String getDigestAuth(const String& authReq, - const String& username, - const String& password, - const String& method, - const String& uri, - unsigned int counter) { - // extracting required parameters for RFC 2069 simpler Digest - const String realm = extractParam(authReq, F("realm=\""), '"'); - const String nonce = extractParam(authReq, F("nonce=\""), '"'); - const String cNonce = getCNonce(8); - - char nc[9]; - - snprintf(nc, sizeof(nc), "%08x", counter); - - // parameters for the RFC 2617 newer Digest - MD5Builder md5; - - md5.begin(); - md5.add(username + ':' + realm + ':' + password); // md5 of the user:realm:user - md5.calculate(); - const String h1 = md5.toString(); - - md5.begin(); - md5.add(method + ':' + uri); - md5.calculate(); - const String h2 = md5.toString(); - - md5.begin(); - md5.add(h1 + ':' + nonce + ':' + String(nc) + ':' + cNonce + F(":auth:") + h2); - md5.calculate(); - const String response = md5.toString(); - - const String authorization = - String(F("Digest username=\"")) + username + - F("\", realm=\"") + realm + - F("\", nonce=\"") + nonce + - F("\", uri=\"") + uri + - F("\", algorithm=\"MD5\", qop=auth, nc=") + String(nc) + - F(", cnonce=\"") + cNonce + - F("\", response=\"") + response + - '"'; - - // Serial.println(authorization); - - return authorization; -} - -void log_http_result(const HTTPClient& http, - const String & logIdentifier, - const String & HttpMethod, - int httpCode, - const String & response) -{ - uint8_t loglevel = LOG_LEVEL_ERROR; - bool success = false; - - // HTTP codes: - // 1xx Informational response - // 2xx Success - if ((httpCode >= 100) && (httpCode < 300)) { - loglevel = LOG_LEVEL_INFO; - success = true; - } - - if (loglevelActiveFor(loglevel)) { - String log = F("HTTP : "); - log += logIdentifier; - log += ' '; - log += HttpMethod; - log += F("... "); - - if (!success) { - log += F("failed "); - } - log += F("HTTP code: "); - log += String(httpCode); - - if (!success) { - log += ' '; - log += http.errorToString(httpCode); - } - - if (response.length() > 0) { - log += ' '; - log += response.substring(0, 100); // Returned string may be huge, so only log the first part. - } - addLogMove(loglevel, log); - } -} - -String send_via_http(const String& logIdentifier, - WiFiClient & client, - uint16_t timeout, - const String& user, - const String& pass, - const String& host, - uint16_t port, - const String& uri, - const String& HttpMethod, - const String& header, - const String& postStr, - int & httpCode, - bool must_check_reply) { - HTTPClient http; - http.setAuthorization(user.c_str(), pass.c_str()); - http.setTimeout(timeout); - http.setUserAgent(get_user_agent_string()); - - #ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS - // See: https://github.com/espressif/arduino-esp32/pull/6676 - client.setTimeout((timeout + 500) / 1000); // in seconds!!!! - #else - client.setTimeout(timeout); // in msec as it should be! - #endif - - // Add request header as fall back. - // When adding another "accept" header, it may be interpreted as: - // "if you have XXX, send it; or failing that, just give me what you've got." - http.addHeader(F("Accept"), F("*/*;q=0.1")); - - delay(0); -#if defined(CORE_POST_2_6_0) || defined(ESP32) - http.begin(client, host, port, uri, false); // HTTP -#else // if defined(CORE_POST_2_6_0) || defined(ESP32) - http.begin(client, host, port, uri); -#endif // if defined(CORE_POST_2_6_0) || defined(ESP32) - - const char *keys[] = { "WWW-Authenticate" }; - http.collectHeaders(keys, 1); - - { - int headerpos = 0; - String name, value; - - while (splitHeaders(headerpos, header, name, value)) { - http.addHeader(name, value); - } - } - - // start connection and send HTTP header (and body) - if (HttpMethod.equals(F("HEAD")) || HttpMethod.equals(F("GET"))) { - httpCode = http.sendRequest(HttpMethod.c_str()); - } else { - httpCode = http.sendRequest(HttpMethod.c_str(), postStr); - } - - String response; - - // httpCode will be negative on error - if (httpCode > 0) { - const String authReq = http.header(String(F("WWW-Authenticate")).c_str()); - - if ((httpCode == 401) && (authReq.indexOf(F("Digest")) != -1)) { - // Use Digest authorization - if (loglevelActiveFor(LOG_LEVEL_INFO)) { - addLogMove(LOG_LEVEL_INFO, String(F("HTTP : Start Digest Authorization for ")) + host); - } - - http.setAuthorization(""); // Clear Basic authorization - const String authorization = getDigestAuth(authReq, user, pass, "GET", uri, 1); - - http.end(); -#if defined(CORE_POST_2_6_0) || defined(ESP32) - http.begin(client, host, port, uri, false); // HTTP -#else // if defined(CORE_POST_2_6_0) || defined(ESP32) - http.begin(client, host, port, uri); -#endif // if defined(CORE_POST_2_6_0) || defined(ESP32) - - http.addHeader(F("Authorization"), authorization); - - // start connection and send HTTP header (and body) - if (HttpMethod.equals(F("HEAD")) || HttpMethod.equals(F("GET"))) { - httpCode = http.sendRequest(HttpMethod.c_str()); - } else { - httpCode = http.sendRequest(HttpMethod.c_str(), postStr); - } - } - - if (httpCode > 0 && must_check_reply) { - response = http.getString(); - } - } - log_http_result(http, logIdentifier, HttpMethod, httpCode, response); - http.end(); - if (Settings.UseRules) { - // Generate event with the HTTP return code - // e.g. http#hostname=401 - String event = F("http#"); - event += host; - event += '='; - event += httpCode; - eventQueue.addMove(std::move(event)); - } - return response; -} String getControllerUser(controllerIndex_t controller_idx, const ControllerSettingsStruct& ControllerSettings) { diff --git a/src/src/Helpers/_CPlugin_Helper.h b/src/src/Helpers/_CPlugin_Helper.h index b7b950682..35004463e 100644 --- a/src/src/Helpers/_CPlugin_Helper.h +++ b/src/src/Helpers/_CPlugin_Helper.h @@ -1,10 +1,6 @@ #ifndef CPLUGIN_HELPER_H #define CPLUGIN_HELPER_H -#include -#include -#include - #include "../../ESPEasy_common.h" #include "../../_Plugin_Helper.h" @@ -19,6 +15,7 @@ #include "../Helpers/_CPlugin_init.h" #include "../Helpers/Misc.h" #include "../Helpers/Network.h" +#include "../Helpers/Networking.h" #include "../Helpers/Numerical.h" #include "../Helpers/StringConverter.h" #include "../Helpers/_CPlugin_Helper_webform.h" @@ -55,19 +52,6 @@ bool try_connect_host(int controller_number, WiFiClient& client, ControllerSetti bool client_available(WiFiClient& client); -String send_via_http(const String& logIdentifier, - WiFiClient & client, - uint16_t timeout, - const String& user, - const String& pass, - const String& host, - uint16_t port, - const String& uri, - const String& HttpMethod, - const String& header, - const String& postStr, - int & httpCode, - bool must_check_reply); String send_via_http(int controller_number, const ControllerSettingsStruct& ControllerSettings, diff --git a/src/src/WebServer/SettingsArchive.cpp b/src/src/WebServer/SettingsArchive.cpp index a0f325d71..c9b30b0bf 100644 --- a/src/src/WebServer/SettingsArchive.cpp +++ b/src/src/WebServer/SettingsArchive.cpp @@ -222,6 +222,7 @@ void storeDownloadFiletypeCheckbox(FileType::Enum filetype, unsigned int filenr) case FileType::NOTIFICATION_DAT: ResetFactoryDefaultPreference.fetchNotificationDat(isChecked); break; case FileType::RULES_TXT: { ResetFactoryDefaultPreference.fetchRulesTXT(filenr, isChecked); break; } case FileType::PROVISIONING_DAT: { ResetFactoryDefaultPreference.fetchProvisioningDat(isChecked); break; } + case FileType::FIRMWARE: // FIXME TD-er: Still have to decide what to do with protecting firmware downloads case FileType::MAX_FILETYPE: break; From 7b9b17bc73b41645ad93b8792ae5d66036ea156c Mon Sep 17 00:00:00 2001 From: TD-er Date: Sat, 23 Jul 2022 15:37:36 +0200 Subject: [PATCH 10/14] [Download] Fix missing include --- src/src/Helpers/Networking.cpp | 3 +++ src/src/Helpers/_CPlugin_Helper.cpp | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index 485850806..b23a1d56c 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -26,6 +26,9 @@ #include "../../ESPEasy-Globals.h" #include +#include +#include + // Generic Networking routines diff --git a/src/src/Helpers/_CPlugin_Helper.cpp b/src/src/Helpers/_CPlugin_Helper.cpp index b69c97853..3fbdfc9aa 100644 --- a/src/src/Helpers/_CPlugin_Helper.cpp +++ b/src/src/Helpers/_CPlugin_Helper.cpp @@ -29,8 +29,6 @@ #include #include -#include -#include bool safeReadStringUntil(Stream & input, From 9c3150ed7301ba090f2727e95c3b32745c1797dd Mon Sep 17 00:00:00 2001 From: DMoosh <106955770+duskmushroom@users.noreply.github.com> Date: Sat, 23 Jul 2022 16:48:23 +0300 Subject: [PATCH 11/14] Update README.md Grammar/spelling --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1254540e5..481960c3a 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ To see what plugins are included in which testing set, you can find that on the ## Documentation & more info -Our new, in-depth documentation can be found at [ESPEasy.readthedocs.io](https://espeasy.readthedocs.io/en/latest/). Automatically built, so always up-to-date according to the contributed contents. The old Wiki documention can be found at [letscontrolit.com/wiki](https://www.letscontrolit.com/wiki/index.php?title=ESPEasy). +Our new, in-depth documentation can be found at [ESPEasy.readthedocs.io](https://espeasy.readthedocs.io/en/latest/). Automatically built, so always up-to-date according to the contributed contents. The old Wiki documentation can be found at [letscontrolit.com/wiki](https://www.letscontrolit.com/wiki/index.php?title=ESPEasy). Additional details and discussion are on the "Experimental" section of the forum: https://www.letscontrolit.com/forum/viewforum.php?f=18 From 40b0b2a0f07b553cad5ed823730163e37d81fb19 Mon Sep 17 00:00:00 2001 From: TD-er Date: Sat, 23 Jul 2022 18:05:27 +0200 Subject: [PATCH 12/14] [SendToHTTP] Allow to use http:// formatted URL --- src/src/Commands/HTTP.cpp | 80 ++++++++++++++++------------------ src/src/Helpers/Networking.cpp | 44 +++++++++++++++---- src/src/Helpers/Networking.h | 13 +++++- 3 files changed, 85 insertions(+), 52 deletions(-) diff --git a/src/src/Commands/HTTP.cpp b/src/src/Commands/HTTP.cpp index de728b4a1..d62001e77 100644 --- a/src/src/Commands/HTTP.cpp +++ b/src/src/Commands/HTTP.cpp @@ -21,22 +21,44 @@ const __FlashStringHelper* Command_HTTP_SendToHTTP(struct EventStruct *event, const char *Line) { if (NetworkConnected()) { - String user, pass; - String host = parseStringKeepCase(Line, 2); - const int pos_at = host.indexOf('@'); + String user, pass, host, file, path; + uint16_t port; - if (pos_at != -1) { - user = host.substring(0, pos_at); - host = host.substring(pos_at + 1); - const int pos_colon = user.indexOf(':'); + const String arg1 = parseStringKeepCase(Line, 2); - if (pos_colon != -1) { - pass = user.substring(pos_colon + 1); - user = user.substring(0, pos_colon); + if (arg1.indexOf('/') != -1) { + // Full url given + path = splitURL(arg1, user, pass, host, port, file); + } else { + // Command arguments are split into: host, port, url + if (!splitUserPass_HostPortString( + arg1, + user, + pass, + host, + port)) + { + return return_command_failed(); } - } - const int port = parseCommandArgumentInt(Line, 2); + const int port_arg = event->Par2; + + if ((port_arg > 0) && (port_arg < 65536)) { + port = port_arg; + } else { + if (loglevelActiveFor(LOG_LEVEL_ERROR)) { + String log = F("SendToHTTP: Invalid port argument: "); + log += port_arg; + log += F(" will use: "); + log += port; + addLogMove(LOG_LEVEL_ERROR, log); + } + } + + // FIXME TD-er: This is not using the tolerant settings option. + // String path = tolerantParseStringKeepCase(Line, 4); + path = parseStringToEndKeepCase(Line, 4); + } #ifndef BUILD_NO_DEBUG if (loglevelActiveFor(LOG_LEVEL_DEBUG)) { @@ -44,29 +66,16 @@ const __FlashStringHelper* Command_HTTP_SendToHTTP(struct EventStruct *event, co log += host; log += F(" port: "); log += port; - addLogMove(LOG_LEVEL_DEBUG, log); - } -#endif // ifndef BUILD_NO_DEBUG - - if ((port < 0) || (port > 65535)) { return return_command_failed(); } - - // FIXME TD-er: This is not using the tolerant settings option. - // String path = tolerantParseStringKeepCase(Line, 4); - const String path = parseStringToEndKeepCase(Line, 4); -#ifndef BUILD_NO_DEBUG - - if (loglevelActiveFor(LOG_LEVEL_DEBUG)) { - String log = F("SendToHTTP: Path: "); + log += F(" path: "); log += path; addLogMove(LOG_LEVEL_DEBUG, log); } #endif // ifndef BUILD_NO_DEBUG - int httpCode = -1; - WiFiClient client; - const String res = send_via_http( - F("Command_HTTP_SendToHTTP"), + WiFiClient client; + send_via_http( + F("SendToHTTP"), client, CONTROLLER_CLIENTTIMEOUT_MAX, user, @@ -80,22 +89,9 @@ const __FlashStringHelper* Command_HTTP_SendToHTTP(struct EventStruct *event, co httpCode, Settings.SendToHttp_ack()); - if (loglevelActiveFor(LOG_LEVEL_INFO)) { - String logstr; - logstr += F("SendToHTTP: "); - logstr += httpCode; - - if (!res.isEmpty()) { - logstr += F(" Received reply: "); - logstr += res; - } - addLog(LOG_LEVEL_INFO, logstr); - } - if ((httpCode >= 100) && (httpCode < 300)) { return return_command_success(); } - addLog(LOG_LEVEL_ERROR, String(F("SendToHTTP: HTTP code: ")) + httpCode); } else { addLog(LOG_LEVEL_ERROR, F("SendToHTTP Not connected to network")); } diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index b23a1d56c..16b347323 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -1180,9 +1180,26 @@ bool splitHostPortString(const String& hostPortString, String& host, uint16_t& p return true; } +bool splitUserPass_HostPortString(const String& hostPortString, String& user, String& pass, String& host, uint16_t& port) +{ + const int pos_at = hostPortString.indexOf('@'); + + if (pos_at != -1) { + user = hostPortString.substring(0, pos_at); + const int pos_colon = user.indexOf(':'); + + if (pos_colon != -1) { + pass = user.substring(pos_colon + 1); + user = user.substring(0, pos_colon); + } + return splitHostPortString(hostPortString.substring(pos_at + 1), host, port); + } + return splitHostPortString(hostPortString, host, port); +} + // Split a full URL like "http://hostname:port/path/file.htm" // Return value is everything after the hostname:port section (including /) -String splitURL(const String& fullURL, String& host, uint16_t& port, String& file) { +String splitURL(const String& fullURL, String& user, String& pass, String& host, uint16_t& port, String& file) { int starthost = fullURL.indexOf(F("//")); if (starthost == -1) { @@ -1192,7 +1209,7 @@ String splitURL(const String& fullURL, String& host, uint16_t& port, String& fil } int endhost = fullURL.indexOf('/', starthost); - splitHostPortString(fullURL.substring(starthost, endhost), host, port); + splitUserPass_HostPortString(fullURL.substring(starthost, endhost), user, pass, host, port); int startfile = fullURL.lastIndexOf('/'); if (startfile >= 0) { @@ -1310,6 +1327,7 @@ String getDigestAuth(const String& authReq, void log_http_result(const HTTPClient& http, const String & logIdentifier, + const String & host, const String & HttpMethod, int httpCode, const String & response) @@ -1329,6 +1347,8 @@ void log_http_result(const HTTPClient& http, String log = F("HTTP : "); log += logIdentifier; log += ' '; + log += host; + log += ' '; log += HttpMethod; log += F("... "); @@ -1344,7 +1364,7 @@ void log_http_result(const HTTPClient& http, } if (response.length() > 0) { - log += ' '; + log += F(" Received reply: "); log += response.substring(0, 100); // Returned string may be huge, so only log the first part. } addLogMove(loglevel, log); @@ -1370,6 +1390,10 @@ int http_authenticate(const String& logIdentifier, http.setTimeout(timeout); http.setUserAgent(get_user_agent_string()); + // FIXME TD-er: Must make this configurable + http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); + http.setRedirectLimit(2); + #ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS // See: https://github.com/espressif/arduino-esp32/pull/6676 @@ -1449,7 +1473,7 @@ int http_authenticate(const String& logIdentifier, event += httpCode; eventQueue.addMove(std::move(event)); } - log_http_result(http, logIdentifier, HttpMethod, httpCode, EMPTY_STRING); + log_http_result(http, logIdentifier, host, HttpMethod, httpCode, EMPTY_STRING); return httpCode; } @@ -1488,7 +1512,7 @@ String send_via_http(const String& logIdentifier, response = http.getString(); if (!response.isEmpty()) { - log_http_result(http, logIdentifier, HttpMethod, httpCode, response); + log_http_result(http, logIdentifier, host, HttpMethod, httpCode, response); } } http.end(); @@ -1512,16 +1536,18 @@ bool downloadFile(const String& url, String file_save) { return downloadFile(url, file_save, EMPTY_STRING, EMPTY_STRING, error); } +// User and Pass may be updated if they occur in the hostname part. +// Thus have to be copied instead of const reference. bool start_downloadFile(WiFiClient & client, HTTPClient & http, const String& url, String & file_save, - const String& user, - const String& pass, + String user, + String pass, String & error) { String host, file; uint16_t port; - String uri = splitURL(url, host, port, file); + String uri = splitURL(url, user, pass, host, port, file); if (file_save.isEmpty()) { file_save = file; @@ -1549,7 +1575,7 @@ bool start_downloadFile(WiFiClient & client, return false; } - int httpCode = http_authenticate( + const int httpCode = http_authenticate( F("DownloadFile"), client, http, diff --git a/src/src/Helpers/Networking.h b/src/src/Helpers/Networking.h index d4353ade7..096b24734 100644 --- a/src/src/Helpers/Networking.h +++ b/src/src/Helpers/Networking.h @@ -158,11 +158,22 @@ bool beginWiFiUDP_randomPort(WiFiUDP& udp); void sendGratuitousARP(); + bool splitHostPortString(const String& hostPortString, String& host, uint16_t& port); +// Split the username and password from a string like this: +// username:password@hostname:portnr +// @param hostPortString The string to parse +// @param user The found username (if any) +// @param pass The found password (if any) +// @param hostname The hostname stripped from any of the other possible parameters +// @param port The found portname (defaults to 80 when not specified) +// @retval Whether supplied hostPortString was valid. +bool splitUserPass_HostPortString(const String& hostPortString, String& user, String& pass, String& host, uint16_t& port); + // Split a full URL like "http://hostname:port/path/file.htm" // Return value is everything after the hostname:port section (including /) -String splitURL(const String& fullURL, String& host, uint16_t& port, String& file); +String splitURL(const String& fullURL, String& user, String& pass, String& host, uint16_t& port, String& file); // Initiate the HTTP connection. From b43802ec4d600c909999a20e75c631c86c11b062 Mon Sep 17 00:00:00 2001 From: TD-er Date: Sat, 23 Jul 2022 21:46:22 +0200 Subject: [PATCH 13/14] [HTTP] Document changes to handling HTTP calls + add follow redirects --- docs/source/Plugin/P000_events.repl | 17 +++++++++++++++++ docs/source/Rules/Rules.rst | 8 ++++++++ docs/source/Tools/Tools.rst | 1 + src/src/DataStructs/SettingsStruct.h | 5 +++++ src/src/DataStructs_templ/SettingsStruct.cpp | 10 ++++++++++ src/src/Helpers/Networking.cpp | 7 ++++--- src/src/WebServer/AdvancedConfigPage.cpp | 2 ++ 7 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/source/Plugin/P000_events.repl b/docs/source/Plugin/P000_events.repl index d17237959..043f4d32e 100644 --- a/docs/source/Plugin/P000_events.repl +++ b/docs/source/Plugin/P000_events.repl @@ -127,6 +127,23 @@ GPIO,2,0 endon + " + " + ``http#hostname=404`` + Added: 2022/07/23 + Triggered as a "return value" when performing a HTTP call to some host. + The event value is the HTTP return code. + The ``hostname`` is replaced by the hostname used in the HTTP call. + "," + + .. code-block:: none + + on http#192.168.1.2 do + if %eventvalue1%!=200 + LogEntry,"HTTP error: %eventvalue1% to: %eventpar%: + endif + endon + " " ``MQTT#Connected`` diff --git a/docs/source/Rules/Rules.rst b/docs/source/Rules/Rules.rst index 22c14699e..4e0d676e5 100644 --- a/docs/source/Rules/Rules.rst +++ b/docs/source/Rules/Rules.rst @@ -1813,6 +1813,14 @@ There is the following workaround: SendToHTTP 192.168.0.243,8080,/json.htm?type=param=switchlight&command&idx=174&switchcmd=On +Added: 2022/07/23 + +* ``SendToHTTP`` can now also be called with a full URL starting with ``http://``, so no longer the host, port and uri have to be separated. (it is still possible of course) +* HTTP return value will be made available as event to be evaluated in the rules. Example event: ``http#hostname=404`` +* Calls made to a HTTP server can now also follow redirects. (GET and HEAD calls only) This has to be enabled in Tools->Advanced page. +* Host name can contain user credentials. For example: ``http://username:pass@hostname:portnr/foo.html`` +* HTTP user credentials now can handle Basic Auth and Digest Auth. + Dew Point for temp/humidity sensors (BME280 for example) -------------------------------------------------------- diff --git a/docs/source/Tools/Tools.rst b/docs/source/Tools/Tools.rst index 5c1a9779d..93bdae656 100644 --- a/docs/source/Tools/Tools.rst +++ b/docs/source/Tools/Tools.rst @@ -230,6 +230,7 @@ Rules Settings * Allow Rules Event Reorder - It is best to have the rules blocks for the most frequently occuring events placed at the top of the first rules file. (also for frequently happening events, which you don't want to act on) The cached event positions can be reordered in memory based on how often an event was matched. (Enabled by default, Added 2022/04/17, disabled 2022/06/24) * Tolerant last parameter - When checked, the last parameter of a command will have less strict parsing. * SendToHTTP wait for ack - When checked, the command SendToHTTP will wait for an acknowledgement from the server. +* SendToHTTP Follow Redirects - When checked, HTTP calls may follow redirects. Strict RFC2616, only requests using GET or HEAD methods will be redirected (using the same method), since the RFC requires end-user confirmation in other cases. Time Source ----------- diff --git a/src/src/DataStructs/SettingsStruct.h b/src/src/DataStructs/SettingsStruct.h index 19c559226..501212f0f 100644 --- a/src/src/DataStructs/SettingsStruct.h +++ b/src/src/DataStructs/SettingsStruct.h @@ -136,6 +136,11 @@ class SettingsStruct_tmpl bool AllowOTAUnlimited() const; void AllowOTAUnlimited(bool value); + // Default behavior is to not allow following redirects + bool SendToHTTP_follow_redirects() const; + void SendToHTTP_follow_redirects(bool value); + + // Flag indicating whether all task values should be sent in a single event or one event per task value (default behavior) bool CombineTaskValues_SingleEvent(taskIndex_t taskIndex) const; void CombineTaskValues_SingleEvent(taskIndex_t taskIndex, bool value); diff --git a/src/src/DataStructs_templ/SettingsStruct.cpp b/src/src/DataStructs_templ/SettingsStruct.cpp index 28425d28c..798111392 100644 --- a/src/src/DataStructs_templ/SettingsStruct.cpp +++ b/src/src/DataStructs_templ/SettingsStruct.cpp @@ -295,6 +295,16 @@ void SettingsStruct_tmpl::AllowOTAUnlimited(bool value) { bitWrite(VariousBits1, 26, value); } +template +bool SettingsStruct_tmpl::SendToHTTP_follow_redirects() const { + return bitRead(VariousBits1, 27); +} + +template +void SettingsStruct_tmpl::SendToHTTP_follow_redirects(bool value) { + bitWrite(VariousBits1, 27, value); +} + template ExtTimeSource_e SettingsStruct_tmpl::ExtTimeSource() const { return static_cast(ExternalTimeSource >> 1); diff --git a/src/src/Helpers/Networking.cpp b/src/src/Helpers/Networking.cpp index 16b347323..0621cbcf3 100644 --- a/src/src/Helpers/Networking.cpp +++ b/src/src/Helpers/Networking.cpp @@ -1390,9 +1390,10 @@ int http_authenticate(const String& logIdentifier, http.setTimeout(timeout); http.setUserAgent(get_user_agent_string()); - // FIXME TD-er: Must make this configurable - http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); - http.setRedirectLimit(2); + if (Settings.SendToHTTP_follow_redirects()) { + http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); + http.setRedirectLimit(2); + } #ifdef MUSTFIX_CLIENT_TIMEOUT_IN_SECONDS diff --git a/src/src/WebServer/AdvancedConfigPage.cpp b/src/src/WebServer/AdvancedConfigPage.cpp index 098a3737f..71dfae19a 100644 --- a/src/src/WebServer/AdvancedConfigPage.cpp +++ b/src/src/WebServer/AdvancedConfigPage.cpp @@ -88,6 +88,7 @@ void handle_advanced() { #endif // WEBSERVER_NEW_RULES Settings.TolerantLastArgParse(isFormItemChecked(F("tolerantargparse"))); Settings.SendToHttp_ack(isFormItemChecked(F("sendtohttp_ack"))); + Settings.SendToHTTP_follow_redirects(isFormItemChecked(F("sendtohttp_redir"))); Settings.ForceWiFi_bg_mode(isFormItemChecked(LabelType::FORCE_WIFI_BG)); Settings.WiFiRestart_connection_lost(isFormItemChecked(LabelType::RESTART_WIFI_LOST_CONN)); Settings.EcoPowerMode(isFormItemChecked(LabelType::CPU_ECO_MODE)); @@ -144,6 +145,7 @@ void handle_advanced() { addFormCheckBox(F("Tolerant last parameter"), F("tolerantargparse"), Settings.TolerantLastArgParse()); addFormNote(F("Perform less strict parsing on last argument of some commands (e.g. publish and sendToHttp)")); addFormCheckBox(F("SendToHTTP wait for ack"), F("sendtohttp_ack"), Settings.SendToHttp_ack()); + addFormCheckBox(F("SendToHTTP Follow Redirects"), F("sendtohttp_redir"), Settings.SendToHTTP_follow_redirects()); /* // MQTT settings now moved to the controller settings. From 8a83d352d7facafe08abbe4be05e7931b688c943 Mon Sep 17 00:00:00 2001 From: fmuntean Date: Sat, 23 Jul 2022 16:24:40 -0400 Subject: [PATCH 14/14] [Fix] Notification Page to display GPIO --- src/src/WebServer/NotificationPage.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/src/WebServer/NotificationPage.cpp b/src/src/WebServer/NotificationPage.cpp index 4e70930e0..9cb0c2c5d 100644 --- a/src/src/WebServer/NotificationPage.cpp +++ b/src/src/WebServer/NotificationPage.cpp @@ -65,8 +65,8 @@ void handle_notifications() { NPlugin_ptr[NotificationProtocolIndex](NPlugin::Function::NPLUGIN_WEBFORM_SAVE, 0, dummyString); } NotificationSettings.Port = getFormItemInt(F("port"), 0); - NotificationSettings.Pin1 = getFormItemInt(F("pin1"), 0); - NotificationSettings.Pin2 = getFormItemInt(F("pin2"), 0); + NotificationSettings.Pin1 = getFormItemInt(F("pin1"), -1); + NotificationSettings.Pin2 = getFormItemInt(F("pin2"), -1); Settings.NotificationEnabled[notificationindex] = isFormItemChecked(F("notificationenabled")); strncpy_webserver_arg(NotificationSettings.Domain, F("domain")); strncpy_webserver_arg(NotificationSettings.Server, F("server")); @@ -140,10 +140,9 @@ void handle_notifications() { html_TD(); addHtml(NotificationSettings.Server); html_TD(); - addHtmlInt(NotificationSettings.Port); - } - else - { + if (NotificationSettings.Port){ + addHtmlInt(NotificationSettings.Port); + } else { //MFD: we display the GPIO addGpioHtml(NotificationSettings.Pin1); @@ -152,6 +151,9 @@ void handle_notifications() { html_BR(); addGpioHtml(NotificationSettings.Pin2); } + } + } + else{ html_TD(3); } }