automatically updated release notes for mega-20180910

Bugfix/command SendToHTTP  & Domoticz IDX > 9999

README.md

@@ -1,9 +1,11 @@
+|Latest Nightly  | Build Status | Patreon | Ko-Fi |
+|-------|-------|-------|-------|
+| [![GitHub version](https://img.shields.io/github/release/letscontrolit/ESPEasy/all.svg)](https://github.com/letscontrolit/ESPEasy/releases/latest) | [![Build Status](https://travis-ci.org/letscontrolit/ESPEasy.svg?branch=mega)](https://travis-ci.org/letscontrolit/ESPEasy) | [![donate](https://img.shields.io/badge/donate-Patreon-blue.svg)](https://www.patreon.com/GrovkillenTDer) | [![donate](https://img.shields.io/badge/donate-KoFi-blue.svg)](https://ko-fi.com/grovkillentder) |
+
+For ways to *support* us, see [this announcement on the forum](https://www.letscontrolit.com/forum/viewtopic.php?f=14&t=5787), or have a look at the [Patreon](https://www.patreon.com/GrovkillenTDer) or [Ko-Fi](https://ko-fi.com/grovkillentder) links above.
+
 # ESPEasy (development branch)
 
-Badge  | Info  |
--------|-------|
-[![Build Status](https://travis-ci.org/letscontrolit/ESPEasy.svg?branch=mega)](https://travis-ci.org/letscontrolit/ESPEasy) | build status |
-[![GitHub version](https://img.shields.io/github/release/letscontrolit/ESPEasy/all.svg)](https://github.com/letscontrolit/ESPEasy/releases/latest) | latest nightly |
 
 Introduction and wiki: https://www.letscontrolit.com/wiki/index.php/ESPEasy#Introduction
 

dist/Release_notes.txt

@@ -1,3 +1,93 @@
+-------------------------------------------------
+Changes in release mega-20180910 (since mega-20180909)
+-------------------------------------------------
+
+Release date: Mon Sep 10 04:00:22 CEST 2018
+
+TD-er (2):
+      [HTTP] Command SendToHttp cannot connect
+      [Domoticz IDX] Allow for IDX of > 9999
+
+sakinit (1):
+      Reduced buffer size to prevent stack overflow
+
+
+-------------------------------------------------
+Changes in release mega-20180909 (since mega-20180908)
+-------------------------------------------------
+
+Release date: Sun Sep  9 04:00:14 CEST 2018
+
+Gijs Noorlander (1):
+      Revert [WDT] Add watchdog feed to backgroundtasks()
+
+TD-er (5):
+      [#1720] WiFi read all lines from slow servers
+      Add forward declaration for void backgroundtasks()
+      [Serial Log] Limit serial log to prevent buffer overflow
+      [ESP32] Fix Build, ESP32 doesn't have the availableForWrite
+      [Serial log] Use buffer to prevent Serial buffer to overflow
+
+
+-------------------------------------------------
+Changes in release mega-20180908 (since mega-20180904)
+-------------------------------------------------
+
+Release date: Sat Sep  8 04:00:20 CEST 2018
+
+Gijs Noorlander (1):
+      Update README.md
+
+TD-er (37):
+      [controller delay] Let controllers handle their own delay and buffer
+      [MQTT] Add controller delay queue for MQTT
+      [Controller] Make ControllerDelayHandler templated and comment
+      [C001] Let C001 controller use queue
+      [C003] Let C003 controller use queue
+      [C004] Let C004 controller use queue and simplify using macros
+      Added `USES_Cxxx` checks
+      [C007] Let C007 controller use queue
+      [MQTT] Use set controller settings on MQTT controllers
+      [Webserver] Add fields for new Controller settings
+      [C008] Let C008 use queue and some bugfixes
+      [#1666] Moved code duplication for HTTP requests to special functions
+      [C009] Removed port number from HTTP host header
+      [C009] Controller C009 now also uses Delay Queue
+      [Client Connect] Check for success on connecting with client.
+      [C009] Remove 'no port' option in HTTP packet
+      [HTTP Auth] Now using proper index for retrieving credentials
+      [HTTP] Add user agent in request header
+      [C010] Controller C010 now also uses Delay Queue
+      [Cleanup] Merge code duplication between C008 and C010
+      [C011] Controller C011 now also uses Delay Queue
+      [Controller] Moved the 'magic' default and max values to defines
+      [C012] Controller C012 now also uses Delay Queue
+      [C013] Remove some obsolete function
+      [Nodes] Make Nodes list a map with dynamic size
+      [Nodes] Highlight current node in the list of nodes
+      [Cleanup] Memory optimisation in available controllers and plugins
+      [Nodes] Fix String copy to properly read node names
+      [MQTT] Give MQTT state in log when connect state changes.
+      [SPIFFS] Add SPIFFS size + usage to sysinfo page
+      [Watchdog] Add watchdog feed to backgroundtasks() function
+      [MQTT] Force MQTT client reconnect when wifi reconnects
+      [WDT] Add yield() and runbackgroundtasks() when parsing rules
+      [#1707] Getting logs from the /log even when password is not entered
+      [#1714] Parse notification sender to allow some sender formatting
+      [Scheduler] Do not schedule task device timer for tasks without interval
+      [Queue] Add memory check to determine full queue
+
+
+-------------------------------------------------
+Changes in release mega-20180904 (since mega-20180903)
+-------------------------------------------------
+
+Release date: Tue Sep  4 04:00:34 CEST 2018
+
+TD-er (1):
+      [#1702] Check for duplicate names bug
+
+
 -------------------------------------------------
 Changes in release mega-20180903 (since mega-20180826)
 -------------------------------------------------

src/Commands/Common.h

@@ -18,36 +18,6 @@ bool IsNumeric(char * source)
 	return result;
 }
 
-bool safeReadStringUntil(Stream &input, String &str, char terminator, unsigned int maxSize = 1024, unsigned int timeout = 1000)
-{
-	int c;
-	const unsigned long timer = millis() + timeout;
-	str = "";
-
-	do {
-		//read character
-		c = input.read();
-		if (c >= 0) {
-			//found terminator, we're ok
-			if (c == terminator) {
-				return(true);
-			}
-			//found character, add to string
-			else{
-				str += char(c);
-				//string at max size?
-				if (str.length() >= maxSize) {
-					addLog(LOG_LEVEL_ERROR, F("Not enough bufferspace to read all input data!"));
-					return(false);
-				}
-			}
-		}
-		yield();
-	} while (!timeOutReached(timer));
-
-	addLog(LOG_LEVEL_ERROR, F("Timeout while reading input data!"));
-	return(false);
-}
 
 String Command_GetORSetIP(struct EventStruct *event,
 	      const __FlashStringHelper *targetDescription,

src/Commands/HTTP.h

@@ -23,32 +23,17 @@ String Command_HTTP_SendToHTTP(struct EventStruct *event, const char* Line)
 			addLog(LOG_LEVEL_DEBUG, log);
 		}
 		WiFiClient client;
-		if (client.connect(host.c_str(), port.toInt())) {
-			String reply = F("GET ");
-			reply += path;
-			reply += F(" HTTP/1.1\r\n");
-			reply += F("Host: ");
-			reply += host;
-			reply += F("\r\n");
-			reply += F("Connection: close\r\n\r\n");
-			addLog(LOG_LEVEL_DEBUG, reply);
-			client.print(reply);
-
-			unsigned long timer = millis() + 200;
-			while (!client.available() && !timeOutReached(timer))
-				delay(1);
-
-			while (client.available()) {
-				// String line = client.readStringUntil('\n');
-				String line;
-				safeReadStringUntil(client, line, '\n');
-
-				if (line.substring(0, 15) == F("HTTP/1.1 200 OK"))
-					addLog(LOG_LEVEL_DEBUG, line);
-				delay(1);
+		const int port_int = port.toInt();
+		const bool connected = client.connect(host.c_str(), port_int) == 1;
+		if (connected) {
+			String hostportString = host;
+			if (port_int != 0 && port_int != 80) {
+				hostportString += ':';
+				hostportString += port_int;
 			}
-			client.flush();
-			client.stop();
+			String request = do_create_http_request(hostportString, F("GET"), path);
+			addLog(LOG_LEVEL_DEBUG, request);
+			send_via_http(F("Command_HTTP_SendToHTTP"), client, request);
 		}
 	}
 	return return_command_success();

src/Controller.ino

@@ -15,6 +15,8 @@ void sendData(struct EventStruct *event)
 //  if (!Settings.TaskDeviceSendData[event->TaskIndex])
 //    return false;
 
+/*
+  // Disabed for now, using buffers at controller side.
   if (Settings.MessageDelay != 0)
   {
     const long dif = timePassedSince(lastSend);
@@ -31,6 +33,7 @@ void sendData(struct EventStruct *event)
       //   backgroundtasks();
     }
   }
+*/
 
   LoadTaskSettings(event->TaskIndex); // could have changed during background tasks.
 
@@ -39,21 +42,25 @@ void sendData(struct EventStruct *event)
     event->ControllerIndex = x;
     event->idx = Settings.TaskDeviceID[x][event->TaskIndex];
     if (Settings.TaskDeviceSendData[event->ControllerIndex][event->TaskIndex] &&
-        Settings.ControllerEnabled[event->ControllerIndex] && Settings.Protocol[event->ControllerIndex])
+        Settings.ControllerEnabled[event->ControllerIndex] &&
+        Settings.Protocol[event->ControllerIndex])
     {
       event->ProtocolIndex = getProtocolIndex(Settings.Protocol[event->ControllerIndex]);
       if (validUserVar(event)) {
         CPlugin_ptr[event->ProtocolIndex](CPLUGIN_PROTOCOL_SEND, event, dummyString);
       } else {
-        String log = F("Invalid value detected for controller ");
-        String controllerName;
-        CPlugin_ptr[event->ProtocolIndex](CPLUGIN_GET_DEVICENAME, event, controllerName);
-        log += controllerName;
-        addLog(LOG_LEVEL_DEBUG, log);
+        if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
+          String log = F("Invalid value detected for controller ");
+          String controllerName;
+          CPlugin_ptr[event->ProtocolIndex](CPLUGIN_GET_DEVICENAME, event, controllerName);
+          log += controllerName;
+          addLog(LOG_LEVEL_DEBUG, log);
+        }
       }
     }
   }
 
+  // FIXME TD-er: This PLUGIN_EVENT_OUT seems to be unused.
   PluginCall(PLUGIN_EVENT_OUT, event, dummyString);
   lastSend = millis();
   STOP_TIMER(SEND_DATA_STATS);
@@ -265,12 +272,33 @@ void SendStatus(byte source, String status)
 
 boolean MQTTpublish(int controller_idx, const char* topic, const char* payload, boolean retained)
 {
-  if (MQTTclient.publish(topic, payload, retained)) {
+  const bool success = MQTTDelayHandler.addToQueue(MQTT_queue_element(controller_idx, topic, payload, retained));
+  scheduleNextMQTTdelayQueue();
+  return success;
+}
+
+void scheduleNextMQTTdelayQueue() {
+  scheduleNextDelayQueue(TIMER_MQTT_DELAY_QUEUE, MQTTDelayHandler.getNextScheduleTime());
+}
+
+void processMQTTdelayQueue() {
+  START_TIMER;
+  MQTT_queue_element element;
+  if (!MQTTDelayHandler.getNext(element)) return;
+  if (MQTTclient.publish(element._topic.c_str(), element._payload.c_str(), element._retained)) {
     setIntervalTimerOverride(TIMER_MQTT, 10); // Make sure the MQTT is being processed as soon as possible.
-    return true;
+    MQTTDelayHandler.markProcessed(true);
+  } else {
+    MQTTDelayHandler.markProcessed(false);
+    if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
+      String log = F("MQTT : process MQTT queue not published, ");
+      log += MQTTDelayHandler.sendQueue.size();
+      log += F(" items left in queue");
+      addLog(LOG_LEVEL_DEBUG, log);
+    }
   }
-  addLog(LOG_LEVEL_DEBUG, F("MQTT : publish failed"));
-  return false;
+  scheduleNextMQTTdelayQueue();
+  STOP_TIMER(MQTT_DELAY_QUEUE);
 }
 
 /*********************************************************************************************\

src/ESPEasy-Globals.h

@@ -11,7 +11,6 @@
 // Set default configuration settings if you want (not mandatory)
 // You can always change these during runtime and save to eeprom
 // After loading firmware, issue a 'reset' command to load the defaults.
-
 // --- Basic Config Settings ------------------------------------------------------------------------
 #define DEFAULT_NAME        "ESP_Easy"                  // Enter your device friendly name
 #define UNIT                            0                                       // Unit Number
@@ -202,6 +201,29 @@
 #define TIMER_30SEC                         4
 #define TIMER_MQTT                          5
 #define TIMER_STATISTICS                    6
+#define TIMER_MQTT_DELAY_QUEUE              7
+#define TIMER_C001_DELAY_QUEUE              8
+#define TIMER_C003_DELAY_QUEUE              9
+#define TIMER_C004_DELAY_QUEUE             10
+#define TIMER_C007_DELAY_QUEUE             11
+#define TIMER_C008_DELAY_QUEUE             12
+#define TIMER_C009_DELAY_QUEUE             13
+#define TIMER_C010_DELAY_QUEUE             14
+#define TIMER_C011_DELAY_QUEUE             15
+#define TIMER_C012_DELAY_QUEUE             16
+#define TIMER_C013_DELAY_QUEUE             17
+
+// Minimum delay between messages for a controller to send in msec.
+#define CONTROLLER_DELAY_QUEUE_DELAY_MAX   3600000
+#define CONTROLLER_DELAY_QUEUE_DELAY_DFLT  100
+// Queue length for controller messages not yet sent.
+#define CONTROLLER_DELAY_QUEUE_DEPTH_MAX   25
+#define CONTROLLER_DELAY_QUEUE_DEPTH_DFLT  10
+// Number of retries to send a message by a controller.
+// N.B. Retries without a connection to wifi do not count as retry.
+#define CONTROLLER_DELAY_QUEUE_RETRY_MAX   10
+#define CONTROLLER_DELAY_QUEUE_RETRY_DFLT  10
+
 
 #define PLUGIN_INIT_ALL                     1
 #define PLUGIN_INIT                         2
@@ -295,7 +317,7 @@
 #define PLUGIN_EXTRACONFIGVAR_MAX          16
 #define CPLUGIN_MAX                        16
 #define NPLUGIN_MAX                         4
-#define UNIT_MAX                           32 // Only relevant for UDP unicast message 'sweeps' and the nodelist.
+#define UNIT_MAX                          254 // unit 255 = broadcast
 #define RULES_TIMER_MAX                     8
 #define PINSTATE_TABLE_MAX                 32
 #define RULES_MAX_SIZE                   2048
@@ -335,6 +357,9 @@
 #define SENSOR_TYPE_LONG                   20
 #define SENSOR_TYPE_WIND                   21
 
+#define UNIT_NUMBER_MAX                  9999  // Stored in Settings.Unit
+#define DOMOTICZ_MAX_IDX            999999999  // Looks like it is an unsigned int, so could be up to 4 bln.
+
 #define VALUE_SOURCE_SYSTEM                 1
 #define VALUE_SOURCE_SERIAL                 2
 #define VALUE_SOURCE_HTTP                   3
@@ -370,6 +395,16 @@
   #define CONFIG_FILE_SIZE               131072
 #endif
 
+// Forward declaration
+void scheduleNextDelayQueue(unsigned long id, unsigned long nextTime);
+String LoadControllerSettings(int ControllerIndex, byte* memAddress, int datasize);
+String get_formatted_Controller_number(int controller_index);
+bool loglevelActiveFor(byte logLevel);
+void addToLog(byte loglevel, const String& string);
+void addToLog(byte logLevel, const __FlashStringHelper* flashString);
+void statusLED(boolean traffic);
+void backgroundtasks();
+
 enum SettingsType {
   BasicSettings_Type = 0,
   TaskSettings_Type,
@@ -412,6 +447,7 @@ bool showSettingsFileLayout = false;
 #include "I2CTypes.h"
 #include <I2Cdev.h>
 #include <map>
+#include <deque>
 
 #define FS_NO_GLOBALS
 #if defined(ESP8266)
@@ -550,6 +586,9 @@ class TimingStats;
 #define LOADFILE_STATS  0
 #define LOOP_STATS      1
 
+/*********************************************************************************************\
+ * CRCStruct
+\*********************************************************************************************/
 struct CRCStruct{
   char compileTimeMD5[16+32+1]= "MD5_MD5_MD5_MD5_BoundariesOfTheSegmentsGoHere...";
   char binaryFilename[32+32+1]= "ThisIsTheDummyPlaceHolderForTheBinaryFilename64ByteLongFilenames";
@@ -580,7 +619,9 @@ int16_t I2C_readS16_reg(uint8_t i2caddr, byte reg);
 int16_t I2C_readS16_LE_reg(uint8_t i2caddr, byte reg);
 I2Cdev i2cdev;
 
-
+/*********************************************************************************************\
+ * SecurityStruct
+\*********************************************************************************************/
 struct SecurityStruct
 {
   SecurityStruct() {
@@ -612,6 +653,9 @@ struct SecurityStruct
   uint8_t       md5[16];
 } SecuritySettings;
 
+/*********************************************************************************************\
+ * SettingsStruct
+\*********************************************************************************************/
 struct SettingsStruct
 {
   SettingsStruct() {
@@ -794,9 +838,14 @@ struct SettingsStruct
 //  uint8_t       md5[16];
 } Settings;
 
+
+/*********************************************************************************************\
+ * ControllerSettingsStruct
+\*********************************************************************************************/
 struct ControllerSettingsStruct
 {
-  ControllerSettingsStruct() : UseDNS(false), Port(0) {
+  ControllerSettingsStruct() : UseDNS(false), Port(0),
+      MinimalTimeBetweenMessages(100), MaxQueueDepth(10), MaxRetry(10), DeleteOldest(false) {
     for (byte i = 0; i < 4; ++i) {
       IP[i] = 0;
     }
@@ -816,6 +865,10 @@ struct ControllerSettingsStruct
   char          MQTTLwtTopic[129];
   char          LWTMessageConnect[129];
   char          LWTMessageDisconnect[129];
+  unsigned int  MinimalTimeBetweenMessages;
+  unsigned int  MaxQueueDepth;
+  unsigned int  MaxRetry;
+  boolean       DeleteOldest; // Action to perform when buffer full, delete oldest, or ignore newest.
 
   IPAddress getIP() const {
     IPAddress host(IP[0], IP[1], IP[2], IP[3]);
@@ -866,21 +919,22 @@ struct ControllerSettingsStruct
     return false;
   }
 
+  // Returns 1 if successful, 0 if there was a problem resolving the hostname or port
   int beginPacket(WiFiUDP &client) {
     if (!checkHostReachable(true)) {
       return 0; // Host not reachable
     }
     byte retry = 2;
     int connected = 0;
-    while (retry > 0 && !connected) {
+    while (retry > 0 && connected == 0) {
       --retry;
       connected = client.beginPacket(getIP(), Port);
       if (connected != 0) return connected;
       if (!checkHostReachable(false))
-        return false;
+        return 0;
       delay(10);
     }
-    return false;
+    return 0;
   }
 
   String getHostPortString() const {
@@ -915,6 +969,10 @@ private:
 
 };
 
+
+/*********************************************************************************************\
+ * NotificationSettingsStruct
+\*********************************************************************************************/
 struct NotificationSettingsStruct
 {
   NotificationSettingsStruct() : Port(0), Pin1(0), Pin2(0) {
@@ -942,6 +1000,10 @@ struct NotificationSettingsStruct
   //its safe to extend this struct, up to 4096 bytes, default values in config are 0
 };
 
+/*********************************************************************************************\
+ * ExtraTaskSettingsStruct
+\*********************************************************************************************/
+
 // This is only used by some plugins to store extra settings like formula descriptions.
 // These settings can only be active for one plugin, meaning they have to be loaded
 // over and over again from flash when another active plugin uses these values.
@@ -1015,6 +1077,9 @@ struct ExtraTaskSettingsStruct
   int16_t TaskDevicePluginConfig[PLUGIN_EXTRACONFIGVAR_MAX];
 } ExtraTaskSettings;
 
+/*********************************************************************************************\
+ * EventStruct
+\*********************************************************************************************/
 struct EventStruct
 {
   EventStruct() :
@@ -1058,6 +1123,10 @@ struct EventStruct
   byte *Data;
 };
 
+
+/*********************************************************************************************\
+ * LogStruct
+\*********************************************************************************************/
 #define LOG_STRUCT_MESSAGE_SIZE 128
 #ifdef ESP32
   #define LOG_STRUCT_MESSAGE_LINES 30
@@ -1176,17 +1245,23 @@ struct LogStruct {
 
 } Logging;
 
+std::deque<char> serialLogBuffer;
+unsigned long last_serial_log_emptied = 0;
+
 byte highest_active_log_level = 0;
 bool log_to_serial_disabled = false;
 // Do this in a template to prevent casting to String when not needed.
 #define addLog(L,S) if (loglevelActiveFor(L)) { addToLog(L,S); }
 
+/*********************************************************************************************\
+ * DeviceStruct
+\*********************************************************************************************/
 struct DeviceStruct
 {
   DeviceStruct() :
-    Number(0), Type(0), VType(0), Ports(0),
+    Number(0), Type(0), VType(0), Ports(0), ValueCount(0),
     PullUpOption(false), InverseLogicOption(false), FormulaOption(false),
-    ValueCount(0), Custom(false), SendDataOption(false), GlobalSyncOption(false),
+    Custom(false), SendDataOption(false), GlobalSyncOption(false),
     TimerOption(false), TimerOptional(false), DecimalsOnly(false) {}
 
   bool connectedToGPIOpins() {
@@ -1198,33 +1273,45 @@ struct DeviceStruct
   byte Type;    // How the device is connected. e.g. DEVICE_TYPE_SINGLE => connected through 1 datapin
   byte VType;   // Type of value the plugin will return, used only for Domoticz
   byte Ports;   // Port to use when device has multiple I/O pins  (N.B. not used much)
+  byte ValueCount;            // The number of output values of a plugin. The value should match the number of keys PLUGIN_VALUENAME1_xxx
   boolean PullUpOption;       // Allow to set internal pull-up resistors.
   boolean InverseLogicOption; // Allow to invert the boolean state (e.g. a switch)
   boolean FormulaOption;      // Allow to enter a formula to convert values during read. (not possible with Custom enabled)
-  byte ValueCount;            // The number of output values of a plugin. The value should match the number of keys PLUGIN_VALUENAME1_xxx
   boolean Custom;
   boolean SendDataOption;     // Allow to send data to a controller.
   boolean GlobalSyncOption;   // No longer used. Was used for ESPeasy values sync between nodes
   boolean TimerOption;        // Allow to set the "Interval" timer for the plugin.
   boolean TimerOptional;      // When taskdevice timer is not set and not optional, use default "Interval" delay (Settings.Delay)
   boolean DecimalsOnly;       // Allow to set the number of decimals (otherwise treated a 0 decimals)
-} Device[DEVICES_MAX + 1]; // 1 more because first device is empty device
+};
+typedef std::vector<DeviceStruct> DeviceVector;
+DeviceVector Device;
 
+
+/*********************************************************************************************\
+ * ProtocolStruct
+\*********************************************************************************************/
 struct ProtocolStruct
 {
   ProtocolStruct() :
-    Number(0), usesMQTT(false), usesAccount(false), usesPassword(false),
-    defaultPort(0), usesTemplate(false), usesID(false), Custom(false) {}
+    defaultPort(0), Number(0), usesMQTT(false), usesAccount(false), usesPassword(false),
+    usesTemplate(false), usesID(false), Custom(false) {}
+  uint16_t defaultPort;
   byte Number;
   boolean usesMQTT;
   boolean usesAccount;
   boolean usesPassword;
-  int defaultPort;
   boolean usesTemplate;
   boolean usesID;
   boolean Custom;
-} Protocol[CPLUGIN_MAX];
+};
+typedef std::vector<ProtocolStruct> ProtocolVector;
+ProtocolVector Protocol;
 
+
+/*********************************************************************************************\
+ * NotificationStruct
+\*********************************************************************************************/
 struct NotificationStruct
 {
   NotificationStruct() :
@@ -1234,20 +1321,29 @@ struct NotificationStruct
   byte usesGPIO;
 } Notification[NPLUGIN_MAX];
 
+
+/*********************************************************************************************\
+ * NodeStruct
+\*********************************************************************************************/
 struct NodeStruct
 {
   NodeStruct() :
-    age(0), build(0), nodeName(NULL), nodeType(0)
+    build(0), age(0), nodeType(0)
     {
       for (byte i = 0; i < 4; ++i) ip[i] = 0;
     }
+  String nodeName;
   byte ip[4];
-  byte age;
   uint16_t build;
-  char* nodeName;
+  byte age;
   byte nodeType;
-} Nodes[UNIT_MAX];
+};
+typedef std::map<byte, NodeStruct> NodesMap;
+NodesMap Nodes;
 
+/*********************************************************************************************\
+ * systemTimerStruct
+\*********************************************************************************************/
 struct systemTimerStruct
 {
   systemTimerStruct() :
@@ -1264,6 +1360,9 @@ struct systemTimerStruct
 };
 std::map<unsigned long, systemTimerStruct> systemTimers;
 
+/*********************************************************************************************\
+ * pinStatesStruct
+\*********************************************************************************************/
 struct pinStatesStruct
 {
   byte plugin;
@@ -1277,6 +1376,9 @@ struct pinStatesStruct
 #define RTC_BASE_STRUCT 64
 #define RTC_BASE_USERVAR 74
 
+/*********************************************************************************************\
+ * RTCStruct
+\*********************************************************************************************/
 //max 40 bytes: ( 74 - 64 ) * 4
 struct RTCStruct
 {
@@ -1301,6 +1403,10 @@ String printWebString = "";
 boolean printToWebJSON = false;
 
 float UserVar[VARS_PER_TASK * TASKS_MAX];
+
+/*********************************************************************************************\
+ * rulesTimerStruct
+\*********************************************************************************************/
 struct rulesTimerStatus
 {
   unsigned long timestamp;
@@ -1474,6 +1580,9 @@ boolean       UseRTOSMultitasking;
 
 void (*MainLoopCall_ptr)(void);
 
+/*********************************************************************************************\
+ * TimingStats
+\*********************************************************************************************/
 class TimingStats {
     public:
       TimingStats() : _timeTotal(0.0), _count(0), _maxVal(0), _minVal(4294967295) {}
@@ -1619,6 +1728,21 @@ unsigned long timediff_cpu_cycles_total = 0;
 #define PROC_SYS_TIMER        9
 #define SET_NEW_TIMER        10
 #define TIME_DIFF_COMPUTE    11
+#define MQTT_DELAY_QUEUE     12
+#define C001_DELAY_QUEUE     13
+#define C002_DELAY_QUEUE     14
+#define C003_DELAY_QUEUE     15
+#define C004_DELAY_QUEUE     16
+#define C005_DELAY_QUEUE     17
+#define C006_DELAY_QUEUE     18
+#define C007_DELAY_QUEUE     19
+#define C008_DELAY_QUEUE     20
+#define C009_DELAY_QUEUE     21
+#define C010_DELAY_QUEUE     22
+#define C011_DELAY_QUEUE     23
+#define C012_DELAY_QUEUE     24
+#define C013_DELAY_QUEUE     25
+
 
 
 
@@ -1644,6 +1768,25 @@ String getMiscStatsName(int stat) {
         case PROC_SYS_TIMER:        return F("proc_system_timer() ");
         case SET_NEW_TIMER:         return F("setNewTimerAt()     ");
         case TIME_DIFF_COMPUTE:     return F("timeDiff()          ");
+        case MQTT_DELAY_QUEUE:      return F("Delay queue     MQTT");
+        case C001_DELAY_QUEUE:
+        case C002_DELAY_QUEUE:
+        case C003_DELAY_QUEUE:
+        case C004_DELAY_QUEUE:
+        case C005_DELAY_QUEUE:
+        case C006_DELAY_QUEUE:
+        case C007_DELAY_QUEUE:
+        case C008_DELAY_QUEUE:
+        case C009_DELAY_QUEUE:
+        case C010_DELAY_QUEUE:
+        case C011_DELAY_QUEUE:
+        case C012_DELAY_QUEUE:
+        case C013_DELAY_QUEUE:
+        {
+          String result = F("Delay queue     ");
+          result += get_formatted_Controller_number(static_cast<int>(stat - C001_DELAY_QUEUE + 1));
+          return result;
+        }
     }
     return F("Unknown");
 }

src/ESPEasy.ino

@@ -83,6 +83,8 @@
 // Define globals before plugin sets to allow a personal override of the selected plugins
 #include "ESPEasy-Globals.h"
 #include "define_plugin_sets.h"
+// Plugin helper needs the defined controller sets, thus include after 'define_plugin_sets.h'
+#include "_CPlugin_Helper.h"
 
 // Blynk_get prototype
 boolean Blynk_get(const String& command, byte controllerIndex,float *data = NULL );
@@ -462,7 +464,7 @@ void loop()
   if (firstLoopConnectionsEstablished) {
      firstLoop = false;
      timerAwakeFromDeepSleep = millis(); // Allow to run for "awake" number of seconds, now we have wifi.
-     schedule_all_task_device_timers();
+     // schedule_all_task_device_timers(); Disabled for now, since we are now using queues for controllers.
    }
 
   // Deep sleep mode, just run all tasks one (more) time and go back to sleep as fast as possible
@@ -529,11 +531,34 @@ void runPeriodicalMQTT() {
   }
 }
 
+String getMQTT_state() {
+  switch (MQTTclient.state()) {
+    case MQTT_CONNECTION_TIMEOUT     : return F("Connection timeout");
+    case MQTT_CONNECTION_LOST        : return F("Connection lost");
+    case MQTT_CONNECT_FAILED         : return F("Connect failed");
+    case MQTT_DISCONNECTED           : return F("Disconnected");
+    case MQTT_CONNECTED              : return F("Connected");
+    case MQTT_CONNECT_BAD_PROTOCOL   : return F("Connect bad protocol");
+    case MQTT_CONNECT_BAD_CLIENT_ID  : return F("Connect bad client_id");
+    case MQTT_CONNECT_UNAVAILABLE    : return F("Connect unavailable");
+    case MQTT_CONNECT_BAD_CREDENTIALS: return F("Connect bad credentials");
+    case MQTT_CONNECT_UNAUTHORIZED   : return F("Connect unauthorized");
+    default: return "";
+  }
+}
+
 void updateMQTTclient_connected() {
   if (MQTTclient_connected != MQTTclient.connected()) {
     MQTTclient_connected = !MQTTclient_connected;
-    if (!MQTTclient_connected)
-      addLog(LOG_LEVEL_ERROR, F("MQTT : Connection lost"));
+    if (!MQTTclient_connected) {
+      if (loglevelActiveFor(LOG_LEVEL_ERROR)) {
+        String connectionError = F("MQTT : Connection lost, state: ");
+        connectionError += getMQTT_state();
+        addLog(LOG_LEVEL_ERROR, connectionError);
+      }
+    } else {
+      schedule_all_tasks_using_MQTT_controller();
+    }
     if (Settings.UseRules) {
       String event = MQTTclient_connected ? F("MQTT#Connected") : F("MQTT#Disconnected");
       rulesProcessing(event);
@@ -803,6 +828,16 @@ void backgroundtasks()
   //checkRAM(F("backgroundtasks"));
   //always start with a yield
   yield();
+/*
+  // Remove this watchdog feed for now.
+  // See https://github.com/letscontrolit/ESPEasy/issues/1722#issuecomment-419659193
+  
+  #ifdef ESP32
+  // Have to find a similar function to call ESP32's esp_task_wdt_feed();
+  #else
+  ESP.wdtFeed();
+  #endif
+*/
 
   //prevent recursion!
   if (runningBackgroundTasks)
@@ -814,7 +849,7 @@ void backgroundtasks()
   #if defined(ESP8266)
     tcpCleanup();
   #endif
-
+  process_serialLogBuffer();
   if(!UseRTOSMultitasking){
     if (Settings.UseSerial)
       if (Serial.available())

src/ESPEasyWifi.ino

@@ -126,6 +126,7 @@ void processGotIP() {
     initTime();
   }
   mqtt_reconnect_count = 0;
+  MQTTclient_should_reconnect = true;
   timermqtt_interval = 100;
   setIntervalTimer(TIMER_MQTT);
   if (Settings.UseRules)

src/Misc.ino

@@ -404,7 +404,7 @@ String checkTaskSettings(byte taskIndex) {
     }
   }
   for (int i = 0; i < TASKS_MAX; ++i) {
-    if (i != taskIndex) {
+    if (i != taskIndex && Settings.TaskDeviceEnabled[i]) {
       LoadTaskSettings(i);
       if (ExtraTaskSettings.TaskDeviceName[0] != 0) {
         if (strcasecmp(ExtraTaskSettings.TaskDeviceName, deviceName.c_str()) == 0) {
@@ -1075,7 +1075,7 @@ String getLogLevelDisplayString(byte index, int& logLevel) {
   }
 }
 
-void addToLog(byte loglevel, String& string)
+void addToLog(byte loglevel, const String& string)
 {
   addToLog(loglevel, string.c_str());
 }
@@ -1162,10 +1162,23 @@ boolean loglevelActive(byte logLevel, byte logLevelSettings) {
 
 void addToLog(byte logLevel, const char *line)
 {
+  const size_t line_length = strlen(line);
   if (loglevelActiveFor(LOG_TO_SERIAL, logLevel)) {
-    Serial.print(millis());
-    Serial.print(F(" : "));
-    Serial.println(line);
+    int roomLeft = ESP.getFreeHeap() - 5000;
+    if (roomLeft > 0) {
+      String timestamp_log(millis());
+      timestamp_log += F(" : ");
+      for (size_t i = 0; i < timestamp_log.length(); ++i) {
+        serialLogBuffer.push_back(timestamp_log[i]);
+      }
+      size_t pos = 0;
+      while (pos < line_length && pos < static_cast<size_t>(roomLeft)) {
+        serialLogBuffer.push_back(line[pos]);
+        ++pos;
+      }
+      serialLogBuffer.push_back('\r');
+      serialLogBuffer.push_back('\n');
+    }
   }
   if (loglevelActiveFor(LOG_TO_SYSLOG, logLevel)) {
     syslog(logLevel, line);
@@ -1184,6 +1197,25 @@ void addToLog(byte logLevel, const char *line)
 #endif
 }
 
+void process_serialLogBuffer() {
+  if (serialLogBuffer.size() == 0) return;
+  if (timePassedSince(last_serial_log_emptied) > 10000) {
+    last_serial_log_emptied = millis();
+    serialLogBuffer.clear();
+    return;
+  }
+  size_t snip = 128; // Some default, ESP32 doesn't have the availableForWrite function yet.
+#if defined(ESP8266)
+  snip = Serial.availableForWrite();
+#endif
+  if (snip > 0) last_serial_log_emptied = millis();
+  size_t bytes_to_write = serialLogBuffer.size();
+  if (snip < bytes_to_write) bytes_to_write = snip;
+  for (size_t i = 0; i < bytes_to_write; ++i) {
+    Serial.write(serialLogBuffer.front());
+    serialLogBuffer.pop_front();
+  }
+}
 
 /********************************************************************************************\
   Delayed reboot, in case of issues, do not reboot with high frequency as it might not help...
@@ -1973,6 +2005,7 @@ void rulesProcessing(String& event)
     log += F(" milliSeconds");
     addLog(LOG_LEVEL_DEBUG, log);
   }
+  backgroundtasks();
 
 }
 
@@ -2035,6 +2068,7 @@ String rulesProcessingFile(String fileName, String& event)
             match, codeBlock, isCommand,
             conditional, condition,
             ifBranche, ifBrancheJustMatch);
+          yield();
         }
 
         line = "";

src/Networking.ino

@@ -107,20 +107,21 @@ void checkUDP()
                 for (byte x = 0; x < 4; x++)
                   ip[x] = packetBuffer[x + 8];
 
-                if (unit < UNIT_MAX)
-                {
+                Nodes[unit].age = 0; // Create a new element when not present
+                NodesMap::iterator it = Nodes.find(unit);
+                if (it != Nodes.end()) {
                   for (byte x = 0; x < 4; x++)
-                    Nodes[unit].ip[x] = packetBuffer[x + 8];
-                  Nodes[unit].age = 0; // reset 'age counter'
+                    it->second.ip[x] = packetBuffer[x + 8];
+                  it->second.age = 0; // reset 'age counter'
                   if (len >= 41) // extended packet size
                   {
-                    // FIXME TD-er: Memory leak waiting to happen.
-                    Nodes[unit].build = packetBuffer[13] + 256*packetBuffer[14];
-                    if (Nodes[unit].nodeName==0)
-                        Nodes[unit].nodeName =  (char *)malloc(26);
-                    memcpy(Nodes[unit].nodeName,reinterpret_cast<byte*>(&packetBuffer[15]), 25);
-                    Nodes[unit].nodeName[25]=0;
-                    Nodes[unit].nodeType = packetBuffer[40];
+                    it->second.build = packetBuffer[13] + 256*packetBuffer[14];
+                    char tmpNodeName[26] = {0};
+                    memcpy(&tmpNodeName[0], reinterpret_cast<byte*>(&packetBuffer[15]), 25);
+                    tmpNodeName[25] = 0;
+                    it->second.nodeName = tmpNodeName;
+                    it->second.nodeName.trim();
+                    it->second.nodeType = packetBuffer[40];
                   }
                 }
 
@@ -166,17 +167,17 @@ void SendUDPCommand(byte destUnit, char* data, byte dataLength)
   if (!WiFiConnected(100)) {
     return;
   }
-  byte firstUnit = 1;
-  byte lastUnit = UNIT_MAX - 1;
   if (destUnit != 0)
   {
-    firstUnit = destUnit;
-    lastUnit = destUnit;
-  }
-  for (int x = firstUnit; x <= lastUnit; x++)
-  {
-    sendUDP(x, (byte*)data, dataLength);
+    sendUDP(destUnit, (byte*)data, dataLength);
     delay(10);
+  } else {
+    for (NodesMap::iterator it = Nodes.begin(); it != Nodes.end(); ++it) {
+      if (it->first != Settings.Unit) {
+        sendUDP(it->first, (byte*)data, dataLength);
+        delay(10);
+      }
+    }
   }
   delay(50);
 }
@@ -190,9 +191,18 @@ void sendUDP(byte unit, byte* data, byte size)
   if (!WiFiConnected(100)) {
     return;
   }
-  if (unit != 255)
-    if (Nodes[unit].ip[0] == 0)
+
+  IPAddress remoteNodeIP;
+  if (unit == 255)
+    remoteNodeIP = {255,255,255,255};
+  else {
+    NodesMap::iterator it = Nodes.find(unit);
+    if (it == Nodes.end())
       return;
+    if (it->second.ip[0] == 0)
+      return;
+    remoteNodeIP = it->second.ip;
+  }
 
   if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) {
     String log = F("UDP  : Send UDP message to ");
@@ -201,12 +211,6 @@ void sendUDP(byte unit, byte* data, byte size)
   }
 
   statusLED(true);
-
-  IPAddress remoteNodeIP;
-  if (unit == 255)
-    remoteNodeIP = {255,255,255,255};
-  else
-    remoteNodeIP = Nodes[unit].ip;
   portUDP.beginPacket(remoteNodeIP, Settings.UDPPort);
   portUDP.write(data, size);
   portUDP.endPacket();
@@ -218,14 +222,17 @@ void sendUDP(byte unit, byte* data, byte size)
   \*********************************************************************************************/
 void refreshNodeList()
 {
-  for (byte counter = 0; counter < UNIT_MAX; counter++)
-  {
-    if (Nodes[counter].ip[0] != 0)
-    {
-      Nodes[counter].age++;  // increment age counter
-      if (Nodes[counter].age > 10) // if entry to old, clear this node ip from the list.
-        for (byte x = 0; x < 4; x++)
-          Nodes[counter].ip[x] = 0;
+  for (NodesMap::iterator it = Nodes.begin(); it != Nodes.end(); ) {
+    bool mustRemove = true;
+    if (it->second.ip[0] != 0) {
+      if (it->second.age < 10) {
+        it->second.age++;
+        mustRemove = false;
+        ++it;
+      }
+    }
+    if (mustRemove) {
+      it = Nodes.erase(it);
     }
   }
 }
@@ -277,15 +284,17 @@ void sendSysInfoUDP(byte repeats)
       delay(500);
   }
 
-  // store my own info also in the list...
-  if (Settings.Unit < UNIT_MAX)
+  Nodes[Settings.Unit].age = 0; // Create new node when not already present.
+  // store my own info also in the list
+  NodesMap::iterator it = Nodes.find(Settings.Unit);
+  if (it != Nodes.end())
   {
     IPAddress ip = WiFi.localIP();
     for (byte x = 0; x < 4; x++)
-      Nodes[Settings.Unit].ip[x] = ip[x];
-    Nodes[Settings.Unit].age = 0;
-    Nodes[Settings.Unit].build = Settings.Build;
-    Nodes[Settings.Unit].nodeType = NODE_TYPE_ID;
+      it->second.ip[x] = ip[x];
+    it->second.age = 0;
+    it->second.build = Settings.Build;
+    it->second.nodeType = NODE_TYPE_ID;
   }
 }
 

src/Scheduler.ino

@@ -46,6 +46,7 @@ void handle_schedule() {
     // No id ready to run right now.
     // Events are not that important to run immediately.
     // Make sure normal scheduled jobs run at higher priority.
+    backgroundtasks();
     process_system_event_queue();
     return;
   }
@@ -75,12 +76,23 @@ void setIntervalTimer(unsigned long id) {
   setIntervalTimer(id, millis());
 }
 
+void setIntervalTimerAt(unsigned long id, unsigned long newtimer) {
+  setNewTimerAt(getMixedId(CONST_INTERVAL_TIMER, id), newtimer);
+}
+
 void setIntervalTimerOverride(unsigned long id, unsigned long msecFromNow) {
   unsigned long timer = millis();
   setNextTimeInterval(timer, msecFromNow);
   setNewTimerAt(getMixedId(CONST_INTERVAL_TIMER, id), timer);
 }
 
+void scheduleNextDelayQueue(unsigned long id, unsigned long nextTime) {
+  if (nextTime != 0) {
+    // Schedule for next process run.
+    setIntervalTimerAt(id, nextTime);
+  }
+}
+
 void setIntervalTimer(unsigned long id, unsigned long lasttimer) {
   // Set the initial timers for the regular runs
   unsigned long interval = 0;
@@ -91,6 +103,20 @@ void setIntervalTimer(unsigned long id, unsigned long lasttimer) {
     case TIMER_30SEC:      interval = 30000; break;
     case TIMER_MQTT:       interval = timermqtt_interval; break;
     case TIMER_STATISTICS: interval = 30000; break;
+    // Fall-through for all DelayQueue, which are just the fall-back timers.
+    // The timers for all delay queues will be set according to their own settings as long as there is something to process.
+    case TIMER_MQTT_DELAY_QUEUE:
+    case TIMER_C001_DELAY_QUEUE:
+    case TIMER_C003_DELAY_QUEUE:
+    case TIMER_C004_DELAY_QUEUE:
+    case TIMER_C007_DELAY_QUEUE:
+    case TIMER_C008_DELAY_QUEUE:
+    case TIMER_C009_DELAY_QUEUE:
+    case TIMER_C010_DELAY_QUEUE:
+    case TIMER_C011_DELAY_QUEUE:
+    case TIMER_C012_DELAY_QUEUE:
+    case TIMER_C013_DELAY_QUEUE:
+      interval = 1000; break;
   }
   unsigned long timer = lasttimer;
   setNextTimeInterval(timer, interval);
@@ -98,6 +124,8 @@ void setIntervalTimer(unsigned long id, unsigned long lasttimer) {
 }
 
 void process_interval_timer(unsigned long id, unsigned long lasttimer) {
+  // Set the interval timer now, it may be altered by the commands below.
+  // This is the default next-run-time.
   setIntervalTimer(id, lasttimer);
   switch (id) {
     case TIMER_20MSEC:
@@ -119,6 +147,63 @@ void process_interval_timer(unsigned long id, unsigned long lasttimer) {
     case TIMER_STATISTICS:
       logTimerStatistics();
       break;
+    case TIMER_MQTT_DELAY_QUEUE:
+      processMQTTdelayQueue();
+      break;
+  #ifdef USES_C001
+    case TIMER_C001_DELAY_QUEUE:
+      process_c001_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C003
+    case TIMER_C003_DELAY_QUEUE:
+      process_c003_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C004
+    case TIMER_C004_DELAY_QUEUE:
+      process_c004_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C007
+    case TIMER_C007_DELAY_QUEUE:
+      process_c007_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C008
+    case TIMER_C008_DELAY_QUEUE:
+      process_c008_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C009
+    case TIMER_C009_DELAY_QUEUE:
+      process_c009_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C010
+    case TIMER_C010_DELAY_QUEUE:
+      process_c010_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C011
+    case TIMER_C011_DELAY_QUEUE:
+      process_c011_delay_queue();
+      break;
+  #endif
+  #ifdef USES_C012
+    case TIMER_C012_DELAY_QUEUE:
+      process_c012_delay_queue();
+      break;
+  #endif
+/*
+  #ifdef USES_C013
+    case TIMER_C013_DELAY_QUEUE:
+      process_c013_delay_queue();
+      break;
+  #endif
+*/
+// When extending this, also extend in _CPlugin_Helper.h
+// Look for DEFINE_Cxxx_DELAY_QUEUE_MACRO
   }
 }
 
@@ -213,18 +298,37 @@ void schedule_all_task_device_timers() {
   }
 }
 
-void schedule_task_device_timer(unsigned long task_index, unsigned long runAt) {
-/*
-  String log = F("schedule_task_device_timer: task: ");
-  log += task_index;
-  log += F(" @ ");
-  log += runAt;
-  if (Settings.TaskDeviceEnabled[task_index]) {
-    log += F(" (enabled)");
+void schedule_all_tasks_using_MQTT_controller() {
+  int ControllerIndex = firstEnabledMQTTController();
+  if (ControllerIndex < 0) return;
+  for (byte task = 0; task < TASKS_MAX; task++) {
+    if (Settings.TaskDeviceSendData[ControllerIndex][task] &&
+        Settings.ControllerEnabled[ControllerIndex] &&
+        Settings.Protocol[ControllerIndex])
+    {
+      schedule_task_device_timer_at_init(task);
+    }
   }
-  addLog(LOG_LEVEL_INFO, log);
-*/
+}
+
+void schedule_task_device_timer(unsigned long task_index, unsigned long runAt) {
+  /*
+    String log = F("schedule_task_device_timer: task: ");
+    log += task_index;
+    log += F(" @ ");
+    log += runAt;
+    if (Settings.TaskDeviceEnabled[task_index]) {
+      log += F(" (enabled)");
+    }
+    addLog(LOG_LEVEL_INFO, log);
+  */
+
   if (task_index >= TASKS_MAX) return;
+  byte DeviceIndex = getDeviceIndex(Settings.TaskDeviceNumber[task_index]);
+  if (!Device[DeviceIndex].TimerOption) return;
+  if (Device[DeviceIndex].TimerOptional && Settings.TaskDeviceTimer[task_index] == 0) {
+    return;
+  }
   if (Settings.TaskDeviceEnabled[task_index]) {
     setNewTimerAt(getMixedId(TASK_DEVICE_TIMER, task_index), runAt);
   }

src/WebServer.ino

@@ -900,25 +900,31 @@ void handle_root() {
     addButton(F("sysinfo"), F("More info"));
 
     TXBuffer += F("</table><BR><BR><table class='multirow'><TR><TH>Node List:<TH>Name<TH>Build<TH>Type<TH>IP<TH>Age");
-    for (byte x = 0; x < UNIT_MAX; x++)
+    for (NodesMap::iterator it = Nodes.begin(); it != Nodes.end(); ++it)
     {
-      if (Nodes[x].ip[0] != 0)
+      if (it->second.ip[0] != 0)
       {
         char url[80];
-        sprintf_P(url, PSTR("<a class='button link' href='http://%u.%u.%u.%u'>%u.%u.%u.%u</a>"), Nodes[x].ip[0], Nodes[x].ip[1], Nodes[x].ip[2], Nodes[x].ip[3], Nodes[x].ip[0], Nodes[x].ip[1], Nodes[x].ip[2], Nodes[x].ip[3]);
-        html_TR_TD(); TXBuffer += F("Unit ");
-        TXBuffer += String(x);
-        html_TD();
-        if (x != Settings.Unit)
-          TXBuffer += Nodes[x].nodeName;
+        sprintf_P(url, PSTR("<a class='button link' href='http://%u.%u.%u.%u'>%u.%u.%u.%u</a>"), it->second.ip[0], it->second.ip[1], it->second.ip[2], it->second.ip[3], it->second.ip[0], it->second.ip[1], it->second.ip[2], it->second.ip[3]);
+        bool isThisUnit = it->first == Settings.Unit;
+        if (isThisUnit)
+          html_TR_TD_highlight();
         else
+          html_TR_TD();
+
+        TXBuffer += F("Unit ");
+        TXBuffer += String(it->first);
+        html_TD();
+        if (isThisUnit)
           TXBuffer += Settings.Name;
+        else
+          TXBuffer += it->second.nodeName;
         html_TD();
-        if (Nodes[x].build)
-          TXBuffer += String(Nodes[x].build);
+        if (it->second.build)
+          TXBuffer += String(it->second.build);
         html_TD();
-        if (Nodes[x].nodeType)
-          switch (Nodes[x].nodeType)
+        if (it->second.nodeType)
+          switch (it->second.nodeType)
           {
             case NODE_TYPE_ID_ESP_EASY_STD:
               TXBuffer += F("ESP Easy");
@@ -939,7 +945,7 @@ void handle_root() {
         html_TD();
         TXBuffer += url;
         html_TD();
-        TXBuffer += String( Nodes[x].age);
+        TXBuffer += String( it->second.age);
       }
     }
 
@@ -1076,7 +1082,7 @@ void handle_config() {
   Settings.Name[25] = 0;
   SecuritySettings.Password[25] = 0;
   addFormTextBox( F("Unit Name"), F("name"), Settings.Name, 25);
-  addFormNumericBox( F("Unit Number"), F("unit"), Settings.Unit, 0, 9999);
+  addFormNumericBox( F("Unit Number"), F("unit"), Settings.Unit, 0, UNIT_NUMBER_MAX);
   addFormPasswordBox(F("Admin Password"), F("password"), SecuritySettings.Password, 25);
 
   addFormSubHeader(F("Wifi Settings"));
@@ -1164,6 +1170,10 @@ void handle_controllers() {
   String MQTTLwtTopic = WebServer.arg(F("mqttlwttopic"));
   String lwtmessageconnect = WebServer.arg(F("lwtmessageconnect"));
   String lwtmessagedisconnect = WebServer.arg(F("lwtmessagedisconnect"));
+  const int minimumsendinterval = getFormItemInt(F("minimumsendinterval"), 100);
+  const int maxqueuedepth = getFormItemInt(F("maxqueuedepth"), 10);
+  const int maxretry = getFormItemInt(F("maxretry"), 10);
+  String deleteoldest = WebServer.arg(F("deleteoldest"));
 
 
   //submitted data
@@ -1182,6 +1192,8 @@ void handle_controllers() {
         //reset (some) default-settings
         byte ProtocolIndex = getProtocolIndex(Settings.Protocol[controllerindex]);
         ControllerSettings.Port = Protocol[ProtocolIndex].defaultPort;
+        ControllerSettings.MinimalTimeBetweenMessages = 100;
+//        ControllerSettings.MaxQueueDepth = 0;
         if (Protocol[ProtocolIndex].usesTemplate)
           CPlugin_ptr[ProtocolIndex](CPLUGIN_PROTOCOL_TEMPLATE, &TempEvent, dummyString);
         strncpy(ControllerSettings.Subscribe, TempEvent.String1.c_str(), sizeof(ControllerSettings.Subscribe));
@@ -1241,6 +1253,11 @@ void handle_controllers() {
         strncpy(ControllerSettings.MQTTLwtTopic, MQTTLwtTopic.c_str(), sizeof(ControllerSettings.MQTTLwtTopic));
         strncpy(ControllerSettings.LWTMessageConnect, lwtmessageconnect.c_str(), sizeof(ControllerSettings.LWTMessageConnect));
         strncpy(ControllerSettings.LWTMessageDisconnect, lwtmessagedisconnect.c_str(), sizeof(ControllerSettings.LWTMessageDisconnect));
+        ControllerSettings.MinimalTimeBetweenMessages = minimumsendinterval;
+        ControllerSettings.MaxQueueDepth = maxqueuedepth;
+        ControllerSettings.MaxRetry = maxretry;
+        ControllerSettings.DeleteOldest = deleteoldest.toInt();
+
 
         CPlugin_ptr[ProtocolIndex](CPLUGIN_INIT, &TempEvent, dummyString);
       }
@@ -1321,12 +1338,17 @@ void handle_controllers() {
       options[0] = F("Use IP address");
       options[1] = F("Use Hostname");
 
+      byte choice_delete_oldest = ControllerSettings.DeleteOldest;
+      String options_delete_oldest[2];
+      options_delete_oldest[0] = F("Ignore New");
+      options_delete_oldest[1] = F("Delete Oldest");
+
+
       byte ProtocolIndex = getProtocolIndex(Settings.Protocol[controllerindex]);
       if (!Protocol[ProtocolIndex].Custom)
       {
 
         addFormSelector(F("Locate Controller"), F("usedns"), 2, options, NULL, NULL, choice, true);
-
         if (ControllerSettings.UseDNS)
         {
           addFormTextBox( F("Controller Hostname"), F("controllerhostname"), ControllerSettings.HostName, sizeof(ControllerSettings.HostName)-1);
@@ -1337,15 +1359,21 @@ void handle_controllers() {
         }
 
         addFormNumericBox( F("Controller Port"), F("controllerport"), ControllerSettings.Port, 1, 65535);
+        addFormNumericBox( F("Minimum Send Interval"), F("minimumsendinterval"), ControllerSettings.MinimalTimeBetweenMessages, 1, CONTROLLER_DELAY_QUEUE_DELAY_MAX);
+        addUnit(F("ms"));
+        addFormNumericBox( F("Max Queue Depth"), F("maxqueuedepth"), ControllerSettings.MaxQueueDepth, 1, CONTROLLER_DELAY_QUEUE_DEPTH_MAX);
+        addFormNumericBox( F("Max Retries"), F("maxretry"), ControllerSettings.MaxRetry, 1, CONTROLLER_DELAY_QUEUE_RETRY_MAX);
+        addFormSelector(F("Full Queue Action"), F("deleteoldest"), 2, options_delete_oldest, NULL, NULL, choice_delete_oldest, true);
+
 
         if (Protocol[ProtocolIndex].usesAccount)
         {
-           String protoDisplayName;
+          String protoDisplayName;
           if (!getControllerProtocolDisplayName(ProtocolIndex, CONTROLLER_USER, protoDisplayName)) {
             protoDisplayName = F("Controller User");
           }
           addFormTextBox(protoDisplayName, F("controlleruser"), SecuritySettings.ControllerUser[controllerindex], sizeof(SecuritySettings.ControllerUser[0])-1);
-         }
+        }
         if (Protocol[ProtocolIndex].usesPassword)
         {
           String protoDisplayName;
@@ -1428,6 +1456,11 @@ void handle_controllers() {
 // Flash strings are not checked for duplication.
 //********************************************************************************
 
+void html_TR_TD_highlight() {
+  TXBuffer += F("<TR class=\"highlight\">");
+  html_TD();
+}
+
 void html_TR_TD() {
   TXBuffer += F("<TR>");
   html_TD();
@@ -1882,6 +1915,7 @@ void handle_devices() {
       PluginCall(PLUGIN_EXIT, &TempEvent, dummyString);
 
       taskClear(taskIndex, false); // clear settings, but do not save
+      ClearCustomTaskSettings(taskIndex);
 
       Settings.TaskDeviceNumber[taskIndex] = taskdevicenumber;
       if (taskdevicenumber != 0) // set default values if a new device has been selected
@@ -1889,8 +1923,9 @@ void handle_devices() {
         //NOTE: do not enable task by default. allow user to enter sensible valus first and let him enable it when ready.
         if (ExtraTaskSettings.TaskDeviceValueNames[0][0] == 0) // if field set empty, reload defaults
           PluginCall(PLUGIN_GET_DEVICEVALUENAMES, &TempEvent, dummyString); //the plugin should populate ExtraTaskSettings with its default values.
-
-        ClearCustomTaskSettings(taskIndex);
+      } else {
+        SaveTaskSettings(taskIndex);
+        SaveSettings();
       }
     }
     else if (taskdevicenumber != 0) //save settings
@@ -2248,7 +2283,7 @@ void handle_devices() {
               html_TR_TD(); TXBuffer += F("IDX:<TD>");
               id = F("TDID");   //="taskdeviceid"
               id += controllerNr + 1;
-              addNumericBox(id, Settings.TaskDeviceID[controllerNr][taskIndex], 0, 999999999); // Looks like it is an unsigned int, so could be up to 4 bln.
+              addNumericBox(id, Settings.TaskDeviceID[controllerNr][taskIndex], 0, DOMOTICZ_MAX_IDX);
             }
           }
         }
@@ -3084,6 +3119,7 @@ void handle_log() {
 // Web Interface JSON log page
 //********************************************************************************
 void handle_log_JSON() {
+  if (!isLoggedIn()) return;
   TXBuffer.startJsonStream();
   String webrequest = WebServer.arg(F("view"));
   TXBuffer += F("{\"Log\": {");
@@ -3754,14 +3790,14 @@ void handle_json()
     }
     if(showNodes) {
       bool comma_between=false;
-      for (byte x = 0; x < UNIT_MAX; x++)
+      for (NodesMap::iterator it = Nodes.begin(); it != Nodes.end(); ++it)
       {
-        if (Nodes[x].ip[0] != 0)
+        if (it->second.ip[0] != 0)
         {
 
           char ip[20];
 
-          sprintf_P(ip, PSTR("%u.%u.%u.%u"), Nodes[x].ip[0], Nodes[x].ip[1], Nodes[x].ip[2], Nodes[x].ip[3]);
+          sprintf_P(ip, PSTR("%u.%u.%u.%u"), it->second.ip[0], it->second.ip[1], it->second.ip[2], it->second.ip[3]);
 
           if( comma_between ) {
             TXBuffer += F(",");
@@ -3771,17 +3807,17 @@ void handle_json()
           }
 
           TXBuffer += F("{");
-          stream_next_json_object_value(F("nr"), String(x));
+          stream_next_json_object_value(F("nr"), String(it->first));
           stream_next_json_object_value(F("name"),
-              (x != Settings.Unit) ? Nodes[x].nodeName : Settings.Name);
+              (it->first != Settings.Unit) ? it->second.nodeName : Settings.Name);
 
-          if (Nodes[x].build) {
-            stream_next_json_object_value(F("build"), String(Nodes[x].build));
+          if (it->second.build) {
+            stream_next_json_object_value(F("build"), String(it->second.build));
           }
 
-          if (Nodes[x].nodeType) {
+          if (it->second.nodeType) {
             String platform;
-            switch (Nodes[x].nodeType)
+            switch (it->second.nodeType)
             {
               case NODE_TYPE_ID_ESP_EASY_STD:     platform = F("ESP Easy");      break;
               case NODE_TYPE_ID_ESP_EASYM_STD:    platform = F("ESP Easy Mega"); break;
@@ -3793,7 +3829,7 @@ void handle_json()
               stream_next_json_object_value(F("platform"), platform);
           }
           stream_next_json_object_value(F("ip"), ip);
-          stream_last_json_object_value(F("age"),  String( Nodes[x].age ));
+          stream_last_json_object_value(F("age"),  String( it->second.age ));
         } // if node info exists
       } // for loop
       if(comma_between) {
@@ -4379,16 +4415,19 @@ boolean handle_custom(String path) {
     if(!unit) unit = btnunit; // unit element prevails, if not used then set to btnunit
     if (unit && unit != Settings.Unit)
     {
-      TXBuffer.startStream();
-      sendHeadandTail(F("TmplDsh"),_HEAD);
-      char url[40];
-      sprintf_P(url, PSTR("http://%u.%u.%u.%u/dashboard.esp"), Nodes[unit].ip[0], Nodes[unit].ip[1], Nodes[unit].ip[2], Nodes[unit].ip[3]);
-      TXBuffer += F("<meta http-equiv=\"refresh\" content=\"0; URL=");
-      TXBuffer += url;
-      TXBuffer += F("\">");
-      sendHeadandTail(F("TmplDsh"),_TAIL);
-      TXBuffer.endStream();
-      return true;
+      NodesMap::iterator it = Nodes.find(unit);
+      if (it != Nodes.end()) {
+        TXBuffer.startStream();
+        sendHeadandTail(F("TmplDsh"),_HEAD);
+        char url[40];
+        sprintf_P(url, PSTR("http://%u.%u.%u.%u/dashboard.esp"), it->second.ip[0], it->second.ip[1], it->second.ip[2], it->second.ip[3]);
+        TXBuffer += F("<meta http-equiv=\"refresh\" content=\"0; URL=");
+        TXBuffer += url;
+        TXBuffer += F("\">");
+        sendHeadandTail(F("TmplDsh"),_TAIL);
+        TXBuffer.endStream();
+        return true;
+      }
     }
 
     TXBuffer.startStream();
@@ -4402,16 +4441,16 @@ boolean handle_custom(String path) {
     // create unit selector dropdown
     addSelector_Head(F("unit"), true);
     byte choice = Settings.Unit;
-    for (byte x = 0; x < UNIT_MAX; x++)
+    for (NodesMap::iterator it = Nodes.begin(); it != Nodes.end(); ++it)
     {
-      if (Nodes[x].ip[0] != 0 || x == Settings.Unit)
+      if (it->second.ip[0] != 0 || it->first == Settings.Unit)
       {
-        String name = String(x) + F(" - ");
-        if (x != Settings.Unit)
-          name += Nodes[x].nodeName;
+        String name = String(it->first) + F(" - ");
+        if (it->first != Settings.Unit)
+          name += it->second.nodeName;
         else
           name += Settings.Name;
-        addSelector_Item(name, x, choice == x, false, F(""));
+        addSelector_Item(name, it->first, choice == it->first, false, F(""));
       }
     }
     addSelector_Foot();
@@ -4419,10 +4458,19 @@ boolean handle_custom(String path) {
     // create <> navigation buttons
     byte prev=Settings.Unit;
     byte next=Settings.Unit;
-    for (byte x = Settings.Unit-1; x > 0; x--)
-      if (Nodes[x].ip[0] != 0) {prev = x; break;}
-    for (byte x = Settings.Unit+1; x < UNIT_MAX; x++)
-      if (Nodes[x].ip[0] != 0) {next = x; break;}
+    NodesMap::iterator it;
+    for (byte x = Settings.Unit-1; x > 0; x--) {
+      it = Nodes.find(x);
+      if (it != Nodes.end()) {
+        if (it->second.ip[0] != 0) {prev = x; break;}
+      }
+    }
+    for (byte x = Settings.Unit+1; x < UNIT_MAX; x++) {
+      it = Nodes.find(x);
+      if (it != Nodes.end()) {
+        if (it->second.ip[0] != 0) {next = x; break;}
+      }
+    }
 
     TXBuffer += F("<a class='button link' href=");
     TXBuffer += path;
@@ -5352,6 +5400,18 @@ void handle_sysinfo() {
    TXBuffer += F(" kB free)");
   #endif
 
+  html_TR_TD(); TXBuffer += F("SPIFFS Size<TD>");
+  {
+  #if defined(ESP8266)
+    fs::FSInfo fs_info;
+    SPIFFS.info(fs_info);
+    TXBuffer += fs_info.totalBytes / 1024;
+    TXBuffer += F(" kB (");
+    TXBuffer += (fs_info.totalBytes - fs_info.usedBytes) / 1024;
+    TXBuffer += F(" kB free)");
+  #endif
+  }
+
   if (showSettingsFileLayout) {
     addTableSeparator(F("Settings Files"), 2, 3);
     html_TR_TD();

src/WebStaticData.h

@@ -236,6 +236,8 @@ static const char pgDefaultCSS[] PROGMEM = {
     "table.multirow tr {padding: 4px; }"
       "table.multirow tr:nth-child(even){background-color: #DEE6FF; }"
     "table.multirow {color: #000; width: 100%; min-width: 420px; border-collapse: collapse; }"
+    // Highlight row
+    "tr.highlight td { background-color: #dbff0075; }"    
     // inside a form
     ".note {color: #444; font-style: italic; }"
     //header with title and menu

src/_C001.ino

@@ -31,49 +31,20 @@ boolean CPlugin_001(byte function, struct EventStruct *event, String& string)
         break;
       }
 
+    case CPLUGIN_INIT:
+      {
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        C001_DelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
+
     case CPLUGIN_PROTOCOL_SEND:
       {
         if (event->idx != 0)
         {
-          if (!WiFiConnected(100)) {
-            success = false;
-            break;
-          }
-          ControllerSettingsStruct ControllerSettings;
-          LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof (ControllerSettings));
-
-          String authHeader = "";
-          if ((SecuritySettings.ControllerUser[event->ControllerIndex][0] != 0) && (SecuritySettings.ControllerPassword[event->ControllerIndex][0] != 0))
-          {
-            base64 encoder;
-            String auth = SecuritySettings.ControllerUser[event->ControllerIndex];
-            auth += ":";
-            auth += SecuritySettings.ControllerPassword[event->ControllerIndex];
-            authHeader = F("Authorization: Basic ");
-            authHeader += encoder.encode(auth);
-            authHeader += F(" \r\n");
-          }
-
-          // boolean success = false;
-          addLog(LOG_LEVEL_DEBUG, String(F("HTTP : connecting to "))+ControllerSettings.getHostPortString());
-
-
-          // Use WiFiClient class to create TCP connections
-          WiFiClient client;
-          if (!ControllerSettings.connectToHost(client))
-          {
-            connectionFailures++;
-
-            addLog(LOG_LEVEL_ERROR, F("HTTP : connection failed"));
-            return false;
-          }
-          statusLED(true);
-          if (connectionFailures)
-            connectionFailures--;
-
           // We now create a URI for the request
-          String url = F("/json.htm?type=command&param=udevice&idx=");
-          url += event->idx;
+          String url;
 
           switch (event->sensorType)
           {
@@ -125,38 +96,8 @@ boolean CPlugin_001(byte function, struct EventStruct *event, String& string)
             url += mapVccToDomoticz();
           #endif
 
-          // This will send the request to the server
-          String request = F("GET ");
-          request += url;
-          request += F(" HTTP/1.1\r\n");
-          request += F("Host: ");
-          request += ControllerSettings.getHost();
-          request += F("\r\n");
-          request += authHeader;
-          request += F("Connection: close\r\n\r\n");
-          client.print(request);
-
-          unsigned long timer = millis() + 200;
-          while (!client.available() && !timeOutReached(timer))
-            yield();
-
-          // Read all the lines of the reply from server and log them
-          while (client.available()) {
-            // String line = client.readStringUntil('\n');
-            String line;
-            safeReadStringUntil(client, line, '\n');
-            addLog(LOG_LEVEL_DEBUG_MORE, line);
-            if (line.startsWith(F("HTTP/1.1 200 OK")) )
-            {
-              addLog(LOG_LEVEL_DEBUG, F("HTTP : Success"));
-              success = true;
-            }
-            yield();
-          }
-          addLog(LOG_LEVEL_DEBUG, F("HTTP : closing connection"));
-
-          client.flush();
-          client.stop();
+          success = C001_DelayHandler.addToQueue(C001_queue_element(event->ControllerIndex, url));
+          scheduleNextDelayQueue(TIMER_C001_DELAY_QUEUE, C001_DelayHandler.getNextScheduleTime());
         } // if ixd !=0
         else
         {
@@ -167,4 +108,17 @@ boolean CPlugin_001(byte function, struct EventStruct *event, String& string)
   }
   return success;
 }
+
+bool do_process_c001_delay_queue(int controller_number, const C001_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  WiFiClient client;
+  if (!try_connect_host(controller_number, client, ControllerSettings))
+    return false;
+
+  // This will send the request to the server
+  String request = create_http_request_auth(controller_number, element.controller_idx, ControllerSettings, F("GET"), element.txt);
+
+  addLog(LOG_LEVEL_DEBUG, element.txt);
+  return send_via_http(controller_number, client, request);
+}
+
 #endif

src/_C002.ino

@@ -33,6 +33,14 @@ boolean CPlugin_002(byte function, struct EventStruct *event, String& string)
         break;
       }
 
+    case CPLUGIN_INIT:
+      {
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        MQTTDelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
+
     case CPLUGIN_PROTOCOL_TEMPLATE:
       {
         event->String1 = F("domoticz/out");
@@ -139,10 +147,12 @@ boolean CPlugin_002(byte function, struct EventStruct *event, String& string)
         {
           ControllerSettingsStruct ControllerSettings;
           LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+/*
           if (!ControllerSettings.checkHostReachable(true)) {
             success = false;
             break;
           }
+*/
           StaticJsonBuffer<200> jsonBuffer;
 
           JsonObject& root = jsonBuffer.createObject();

src/_C003.ino

@@ -30,77 +30,24 @@ boolean CPlugin_003(byte function, struct EventStruct *event, String& string)
         break;
       }
 
-    case CPLUGIN_PROTOCOL_SEND:
+    case CPLUGIN_INIT:
       {
         ControllerSettingsStruct ControllerSettings;
         LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        C003_DelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
 
-        boolean success = false;
-        char log[80];
-        addLog(LOG_LEVEL_DEBUG, String(F("TELNT : connecting to ")) + ControllerSettings.getHostPortString());
-        // Use WiFiClient class to create TCP connections
-        WiFiClient client;
-        if (!ControllerSettings.connectToHost(client))
-        {
-          connectionFailures++;
-          strcpy_P(log, PSTR("TELNT: connection failed"));
-          addLog(LOG_LEVEL_ERROR, log);
-          return false;
-        }
-        statusLED(true);
-        if (connectionFailures)
-          connectionFailures--;
-
+    case CPLUGIN_PROTOCOL_SEND:
+      {
         // We now create a URI for the request
         String url = F("variableset ");
         url += event->idx;
         url += ",";
         url += formatUserVarNoCheck(event, 0);
         url += "\n";
-
-        // strcpy_P(log, PSTR("TELNT: Sending enter"));
-        // addLog(LOG_LEVEL_ERROR, log);
-        client.print(" \n");
-
-        unsigned long timer = millis() + 200;
-        while (!client.available() && !timeOutReached(timer))
-          delay(1);
-
-        timer = millis() + 1000;
-        while (client.available() && !timeOutReached(timer) && !success)
-        {
-
-          //   String line = client.readStringUntil('\n');
-          String line;
-          safeReadStringUntil(client, line, '\n');
-
-          if (line.startsWith(F("Enter your password:")))
-          {
-            success = true;
-            strcpy_P(log, PSTR("TELNT: Password request ok"));
-            addLog(LOG_LEVEL_DEBUG, log);
-          }
-          delay(1);
-        }
-
-        strcpy_P(log, PSTR("TELNT: Sending pw"));
-        addLog(LOG_LEVEL_DEBUG, log);
-        client.println(SecuritySettings.ControllerPassword[event->ControllerIndex]);
-        delay(100);
-        while (client.available())
-          client.read();
-
-        strcpy_P(log, PSTR("TELNT: Sending cmd"));
-        addLog(LOG_LEVEL_DEBUG, log);
-        client.print(url);
-        delay(10);
-        while (client.available())
-          client.read();
-
-        strcpy_P(log, PSTR("TELNT: closing connection"));
-        addLog(LOG_LEVEL_DEBUG, log);
-
-        client.stop();
+        success = C003_DelayHandler.addToQueue(C003_queue_element(event->ControllerIndex, url));
+        scheduleNextDelayQueue(TIMER_C003_DELAY_QUEUE, C003_DelayHandler.getNextScheduleTime());
 
         break;
       }
@@ -108,4 +55,67 @@ boolean CPlugin_003(byte function, struct EventStruct *event, String& string)
   }
   return success;
 }
+
+bool do_process_c003_delay_queue(int controller_number, const C003_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  boolean success = false;
+  char log[80];
+  addLog(LOG_LEVEL_DEBUG, String(F("TELNT : connecting to ")) + ControllerSettings.getHostPortString());
+  // Use WiFiClient class to create TCP connections
+  WiFiClient client;
+  if (!ControllerSettings.connectToHost(client))
+  {
+    connectionFailures++;
+    strcpy_P(log, PSTR("TELNT: connection failed"));
+    addLog(LOG_LEVEL_ERROR, log);
+    return success;
+  }
+  statusLED(true);
+  if (connectionFailures)
+    connectionFailures--;
+
+  // strcpy_P(log, PSTR("TELNT: Sending enter"));
+  // addLog(LOG_LEVEL_ERROR, log);
+  client.print(" \n");
+
+  unsigned long timer = millis() + 200;
+  while (!client.available() && !timeOutReached(timer))
+    delay(1);
+
+  timer = millis() + 1000;
+  while (client_available(client) && !timeOutReached(timer) && !success)
+  {
+
+    //   String line = client.readStringUntil('\n');
+    String line;
+    safeReadStringUntil(client, line, '\n');
+
+    if (line.startsWith(F("Enter your password:")))
+    {
+      success = true;
+      strcpy_P(log, PSTR("TELNT: Password request ok"));
+      addLog(LOG_LEVEL_DEBUG, log);
+    }
+    delay(1);
+  }
+
+  strcpy_P(log, PSTR("TELNT: Sending pw"));
+  addLog(LOG_LEVEL_DEBUG, log);
+  client.println(SecuritySettings.ControllerPassword[element.controller_idx]);
+  delay(100);
+  while (client.available())
+    client.read();
+
+  strcpy_P(log, PSTR("TELNT: Sending cmd"));
+  addLog(LOG_LEVEL_DEBUG, log);
+  client.print(element.txt);
+  delay(10);
+  while (client_available(client))
+    client.read();
+
+  strcpy_P(log, PSTR("TELNT: closing connection"));
+  addLog(LOG_LEVEL_DEBUG, log);
+
+  client.stop();
+  return success;
+}
 #endif

src/_C004.ino

@@ -30,6 +30,14 @@ boolean CPlugin_004(byte function, struct EventStruct *event, String& string)
         break;
       }
 
+    case CPLUGIN_INIT:
+      {
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        C004_DelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
+
     case CPLUGIN_GET_PROTOCOL_DISPLAY_NAME:
       {
         success = true;
@@ -48,92 +56,44 @@ boolean CPlugin_004(byte function, struct EventStruct *event, String& string)
 
     case CPLUGIN_PROTOCOL_SEND:
       {
-        ControllerSettingsStruct ControllerSettings;
-        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        success = C004_DelayHandler.addToQueue(C004_queue_element(event));
+        scheduleNextDelayQueue(TIMER_C004_DELAY_QUEUE, C004_DelayHandler.getNextScheduleTime());
 
-        // boolean success = false;
-        addLog(LOG_LEVEL_DEBUG, String(F("HTTP : connecting to "))+ControllerSettings.getHostPortString());
-        char log[80];
-        // Use WiFiClient class to create TCP connections
-        WiFiClient client;
-        if (!ControllerSettings.connectToHost(client))
-        {
-          connectionFailures++;
-          if (loglevelActiveFor(LOG_LEVEL_ERROR)) {
-            strcpy_P(log, PSTR("HTTP : connection failed"));
-            addLog(LOG_LEVEL_ERROR, log);
-          }
-          return false;
-        }
-        statusLED(true);
-        if (connectionFailures)
-          connectionFailures--;
-
-        String postDataStr = F("api_key=");
-        postDataStr += SecuritySettings.ControllerPassword[event->ControllerIndex]; // used for API key
-
-        byte valueCount = getValueCountFromSensorType(event->sensorType);
-        for (byte x = 0; x < valueCount; x++)
-        {
-          postDataStr += F("&field");
-          postDataStr += event->idx + x;
-          postDataStr += "=";
-          postDataStr += formatUserVarNoCheck(event, x);
-        }
-        String hostName = F("api.thingspeak.com"); // PM_CZ: HTTP requests must contain host headers.
-        if (ControllerSettings.UseDNS)
-          hostName = ControllerSettings.HostName;
-
-        String postStr = F("POST /update HTTP/1.1\r\n");
-        postStr += F("Host: ");
-        postStr += hostName;
-        postStr += F("\r\n");
-        postStr += F("Connection: close\r\n");
-
-        postStr += F("Content-Type: application/x-www-form-urlencoded\r\n");
-        postStr += F("Content-Length: ");
-        postStr += postDataStr.length();
-        postStr += F("\r\n\r\n");
-        postStr += postDataStr;
-
-        // This will send the request to the server
-        client.print(postStr);
-
-        unsigned long timer = millis() + 200;
-        while (!client.available() && !timeOutReached(timer))
-          delay(1);
-
-        // Read all the lines of the reply from server and print them to Serial
-        while (client.available()) {
-          //   String line = client.readStringUntil('\n');
-          String line;
-          safeReadStringUntil(client, line, '\n');
-
-          if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) {
-            line.toCharArray(log, 80);
-            addLog(LOG_LEVEL_DEBUG_MORE, log);
-          }
-          if (line.substring(0, 15) == F("HTTP/1.1 200 OK"))
-          {
-            if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
-              strcpy_P(log, PSTR("HTTP : Success!"));
-              addLog(LOG_LEVEL_DEBUG, log);
-            }
-            success = true;
-          }
-          delay(1);
-        }
-        if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
-          strcpy_P(log, PSTR("HTTP : closing connection"));
-          addLog(LOG_LEVEL_DEBUG, log);
-        }
-
-        client.flush();
-        client.stop();
         break;
       }
 
   }
   return success;
 }
+
+bool do_process_c004_delay_queue(int controller_number, const C004_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  WiFiClient client;
+  if (!try_connect_host(controller_number, client, ControllerSettings))
+    return false;
+
+  String postDataStr = F("api_key=");
+  postDataStr += SecuritySettings.ControllerPassword[element.controller_idx]; // used for API key
+
+  byte valueCount = getValueCountFromSensorType(element.sensorType);
+  for (byte x = 0; x < valueCount; x++)
+  {
+    postDataStr += F("&field");
+    postDataStr += element.idx + x;
+    postDataStr += "=";
+    postDataStr += formatUserVarNoCheck(element.TaskIndex, x);
+  }
+  String hostName = F("api.thingspeak.com"); // PM_CZ: HTTP requests must contain host headers.
+  if (ControllerSettings.UseDNS)
+    hostName = ControllerSettings.HostName;
+
+  String postStr = do_create_http_request(
+    hostName, F("POST"),
+    F("/update"), // uri
+    "",           // auth_header
+    F("Content-Type: application/x-www-form-urlencoded\r\n"),
+    postDataStr.length());
+  postStr += postDataStr;
+
+  return send_via_http(controller_number, client, postStr);
+}
 #endif

src/_C005.ino

@@ -31,6 +31,14 @@ boolean CPlugin_005(byte function, struct EventStruct *event, String& string)
         break;
       }
 
+    case CPLUGIN_INIT:
+      {
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        MQTTDelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
+
     case CPLUGIN_PROTOCOL_TEMPLATE:
       {
         event->String1 = F("/%sysname%/#");

src/_C006.ino

@@ -31,6 +31,14 @@ boolean CPlugin_006(byte function, struct EventStruct *event, String& string)
         break;
       }
 
+    case CPLUGIN_INIT:
+      {
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        MQTTDelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
+
     case CPLUGIN_PROTOCOL_TEMPLATE:
       {
         event->String1 = F("/Home/#");

src/_C007.ino

@@ -30,99 +30,54 @@ boolean CPlugin_007(byte function, struct EventStruct *event, String& string)
         break;
       }
 
+    case CPLUGIN_INIT:
+      {
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        C007_DelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
+
     case CPLUGIN_PROTOCOL_SEND:
       {
-        if (!WiFiConnected(100)) {
-          success = false;
-          break;
-        }
         const byte valueCount = getValueCountFromSensorType(event->sensorType);
         if (valueCount == 0 || valueCount > 3) {
           addLog(LOG_LEVEL_ERROR, F("emoncms : Unknown sensortype or too many sensor values"));
           break;
         }
-
-        ControllerSettingsStruct ControllerSettings;
-        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
-
-        // boolean success = false;
-        addLog(LOG_LEVEL_DEBUG, String(F("HTTP : connecting to "))+ControllerSettings.getHostPortString());
-        char log[80];
-        // Use WiFiClient class to create TCP connections
-        WiFiClient client;
-        if (!ControllerSettings.connectToHost(client))
-        {
-          connectionFailures++;
-          strcpy_P(log, PSTR("HTTP : connection failed"));
-          addLog(LOG_LEVEL_ERROR, log);
-          return false;
-        }
-        statusLED(true);
-        if (connectionFailures)
-          connectionFailures--;
-
-        String postDataStr = F("GET /emoncms/input/post.json?node=");
-
-        postDataStr += Settings.Unit;
-        postDataStr += F("&json=");
-
-        for (byte i = 0; i < valueCount; ++i) {
-          postDataStr += (i == 0) ? F("{") : F(",");
-          postDataStr += F("field");
-          postDataStr += event->idx + i;
-          postDataStr += ":";
-          postDataStr += formatUserVarNoCheck(event, i);
-        }
-        postDataStr += "}";
-        postDataStr += F("&apikey=");
-        postDataStr += SecuritySettings.ControllerPassword[event->ControllerIndex]; // "0UDNN17RW6XAS2E5" // api key
-
-        String postStr = F(" HTTP/1.1\r\n");
-        postStr += F("Host: ");
-        postStr += ControllerSettings.getHost();
-        postStr += F("\r\n");
-        postStr += F("Connection: close\r\n");
-        postStr += F("\r\n");
-
-        postDataStr += postStr;
-
-        if (Settings.SerialLogLevel >= LOG_LEVEL_DEBUG_MORE)
-          Serial.println(postDataStr);
-
-        // This will send the request to the server
-        client.print(postDataStr);
-
-        unsigned long timer = millis() + 200;
-        while (!client.available() && !timeOutReached(timer))
-          delay(1);
-
-        // Read all the lines of the reply from server and print them to Serial
-        while (client.available()) {
-          //   String line = client.readStringUntil('\n');
-          String line;
-          safeReadStringUntil(client, line, '\n');
-
-          if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) {
-            line.toCharArray(log, 80);
-            addLog(LOG_LEVEL_DEBUG_MORE, log);
-          }
-          if (line.substring(0, 15) == F("HTTP/1.1 200 OK"))
-          {
-            strcpy_P(log, PSTR("HTTP : Success!"));
-            addLog(LOG_LEVEL_DEBUG, log);
-            success = true;
-          }
-          delay(1);
-        }
-        strcpy_P(log, PSTR("HTTP : closing connection"));
-        addLog(LOG_LEVEL_DEBUG, log);
-
-        client.flush();
-        client.stop();
+        success = C007_DelayHandler.addToQueue(C007_queue_element(event));
+        scheduleNextDelayQueue(TIMER_C007_DELAY_QUEUE, C007_DelayHandler.getNextScheduleTime());
         break;
       }
 
   }
   return success;
 }
+
+bool do_process_c007_delay_queue(int controller_number, const C007_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  WiFiClient client;
+  if (!try_connect_host(controller_number, client, ControllerSettings))
+    return false;
+
+  String url = F("/emoncms/input/post.json?node=");
+  url += Settings.Unit;
+  url += F("&json=");
+  const byte valueCount = getValueCountFromSensorType(element.sensorType);
+  for (byte i = 0; i < valueCount; ++i) {
+    url += (i == 0) ? F("{") : F(",");
+    url += F("field");
+    url += element.idx + i;
+    url += ":";
+    url += formatUserVarNoCheck(element.TaskIndex, i);
+  }
+  url += "}";
+  url += F("&apikey=");
+  url += SecuritySettings.ControllerPassword[element.controller_idx]; // "0UDNN17RW6XAS2E5" // api key
+
+  if (Settings.SerialLogLevel >= LOG_LEVEL_DEBUG_MORE)
+    Serial.println(url);
+
+  return send_via_http(controller_number, client,
+    create_http_get_request(controller_number, ControllerSettings, url));
+}
 #endif

src/_C008.ino

@@ -32,6 +32,14 @@ boolean CPlugin_008(byte function, struct EventStruct *event, String& string)
         break;
       }
 
+    case CPLUGIN_INIT:
+      {
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+        C008_DelayHandler.configureControllerSettings(ControllerSettings);
+        break;
+      }
+
     case CPLUGIN_PROTOCOL_TEMPLATE:
       {
         event->String1 = "";
@@ -41,21 +49,31 @@ boolean CPlugin_008(byte function, struct EventStruct *event, String& string)
 
     case CPLUGIN_PROTOCOL_SEND:
       {
+        // Collect the values at the same run, to make sure all are from the same sample
         byte valueCount = getValueCountFromSensorType(event->sensorType);
+        C008_queue_element element(event, valueCount);
+        if (ExtraTaskSettings.TaskDeviceValueNames[0][0] == 0)
+          PluginCall(PLUGIN_GET_DEVICEVALUENAMES, event, dummyString);
+
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+
         for (byte x = 0; x < valueCount; x++)
         {
           bool isvalid;
           String formattedValue = formatUserVar(event, x, isvalid);
-          if (isvalid)
-            HTTPSend(event, x, formattedValue);
-          if (valueCount > 1)
-          {
-            delayBackground(Settings.MessageDelay);
-            // unsigned long timer = millis() + Settings.MessageDelay;
-            // while (!timeOutReached(timer))
-            //   backgroundtasks();
+          if (isvalid) {
+            element.txt[x] = "/";
+            element.txt[x] += ControllerSettings.Publish;
+            parseControllerVariables(element.txt[x], event, true);
+
+            element.txt[x].replace(F("%valname%"), URLEncode(ExtraTaskSettings.TaskDeviceValueNames[x]));
+            element.txt[x].replace(F("%value%"), formattedValue);
+            addLog(LOG_LEVEL_DEBUG_MORE, element.txt[x]);
           }
         }
+        success = C008_DelayHandler.addToQueue(element);
+        scheduleNextDelayQueue(TIMER_C008_DELAY_QUEUE, C008_DelayHandler.getNextScheduleTime());
         break;
       }
 
@@ -63,90 +81,23 @@ boolean CPlugin_008(byte function, struct EventStruct *event, String& string)
   return success;
 }
 
-
 //********************************************************************************
 // Generic HTTP get request
 //********************************************************************************
-boolean HTTPSend(struct EventStruct *event, byte varIndex, const String& formattedValue)
-{
-  if (!WiFiConnected(100)) {
-    return false;
-  }
-  ControllerSettingsStruct ControllerSettings;
-  LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
-
-  String authHeader = "";
-  if ((SecuritySettings.ControllerUser[event->ControllerIndex][0] != 0) && (SecuritySettings.ControllerPassword[event->ControllerIndex][0] != 0))
-  {
-    base64 encoder;
-    String auth = SecuritySettings.ControllerUser[event->ControllerIndex];
-    auth += ":";
-    auth += SecuritySettings.ControllerPassword[event->ControllerIndex];
-    authHeader = F("Authorization: Basic ");
-    authHeader += encoder.encode(auth) + " \r\n";
+bool do_process_c008_delay_queue(int controller_number, const C008_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  while (element.txt[element.valuesSent] == "") {
+    // A non valid value, which we are not going to send.
+    // Increase sent counter until a valid value is found.
+    if (element.checkDone(true))
+      return true;
   }
 
-  // boolean success = false;
-  addLog(LOG_LEVEL_DEBUG, String(F("HTTP : connecting to "))+ControllerSettings.getHostPortString());
-
-  // Use WiFiClient class to create TCP connections
   WiFiClient client;
-  if (!ControllerSettings.connectToHost(client))
-  {
-    connectionFailures++;
-    addLog(LOG_LEVEL_ERROR, F("HTTP : connection failed"));
+  if (!try_connect_host(controller_number, client, ControllerSettings))
     return false;
-  }
-  statusLED(true);
-  if (connectionFailures)
-    connectionFailures--;
 
-  if (ExtraTaskSettings.TaskDeviceValueNames[0][0] == 0)
-    PluginCall(PLUGIN_GET_DEVICEVALUENAMES, event, dummyString);
-
-  String url = "/";
-  url += ControllerSettings.Publish;
-  parseControllerVariables(url, event, true);
-
-  url.replace(F("%valname%"), URLEncode(ExtraTaskSettings.TaskDeviceValueNames[varIndex]));
-  url.replace(F("%value%"), formattedValue);
-
-  // url.toCharArray(log, 80);
-  addLog(LOG_LEVEL_DEBUG_MORE, url);
-
-  // This will send the request to the server
-  client.print(String(F("GET ")) + url + F(" HTTP/1.1\r\n") +
-               F("Host: ") + ControllerSettings.getHost() + F("\r\n") + authHeader +
-               F("Connection: close\r\n\r\n"));
-
-  unsigned long timer = millis() + 200;
-  while (!client.available() && !timeOutReached(timer))
-    yield();
-
-  // Read all the lines of the reply from server and print them to Serial
-  while (client.available()) {
-    // String line = client.readStringUntil('\n');
-    String line;
-    safeReadStringUntil(client, line, '\n');
-
-    // line.toCharArray(log, 80);
-    addLog(LOG_LEVEL_DEBUG_MORE, line);
-    if (line.startsWith(F("HTTP/1.1 200 OK")))
-    {
-      // strcpy_P(log, PSTR("HTTP : Success!"));
-      // addLog(LOG_LEVEL_DEBUG, log);
-      addLog(LOG_LEVEL_DEBUG, F("HTTP : Success!"));
-      // success = true;
-    }
-    delay(1);
-  }
-  // strcpy_P(log, PSTR("HTTP : closing connection"));
-  // addLog(LOG_LEVEL_DEBUG, log);
-  addLog(LOG_LEVEL_DEBUG, F("HTTP : closing connection"));
-
-  client.flush();
-  client.stop();
-
-  return(true);
+  String request = create_http_request_auth(controller_number, element.controller_idx, ControllerSettings, F("GET"), element.txt[element.valuesSent]);
+  return element.checkDone(send_via_http(controller_number, client, request));
 }
+
 #endif

src/_C009.ino

@@ -107,89 +107,30 @@ boolean CPlugin_009(byte function, struct EventStruct *event, String& string)
         }
 
         // Create json buffer
-        // char buffer[root.measureLength() +1];
-        // root.printTo(buffer, sizeof(buffer));
         String jsonString;
         root.printTo(jsonString);
-        // Push data to server
-        FHEMHTTPsend(url, jsonString, event->ControllerIndex);
+
+        success = C009_DelayHandler.addToQueue(C009_queue_element(event->ControllerIndex, url, jsonString));
+        scheduleNextDelayQueue(TIMER_C009_DELAY_QUEUE, C009_DelayHandler.getNextScheduleTime());
         break;
       }
   }
   return success;
 }
 
-
-//********************************************************************************
-// FHEM HTTP request
-//********************************************************************************
-//TODO: create a generic HTTPSend function that we use in all the controllers. lots of code duplication here
-void FHEMHTTPsend(String & url, String & buffer, byte index)
-{
-  ControllerSettingsStruct ControllerSettings;
-  LoadControllerSettings(index, (byte*)&ControllerSettings, sizeof(ControllerSettings));
-
-  // boolean success = false;
-
-  String authHeader = "";
-  if ((SecuritySettings.ControllerUser[index][0] != 0) && (SecuritySettings.ControllerPassword[index][0] != 0)) {
-    base64 encoder;
-    String auth = SecuritySettings.ControllerUser[index];
-    auth += ":";
-    auth += SecuritySettings.ControllerPassword[index];
-    authHeader = String(F("Authorization: Basic ")) + encoder.encode(auth) + " \r\n";
-  }
-
-  addLog(LOG_LEVEL_DEBUG, String(F("HTTP : connecting to "))+ControllerSettings.getHostPortString());
-
-  // Use WiFiClient class to create TCP connections
+/*********************************************************************************************\
+ * FHEM HTTP request
+\*********************************************************************************************/
+bool do_process_c009_delay_queue(int controller_number, const C009_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
   WiFiClient client;
-  if (!ControllerSettings.connectToHost(client)) {
-    connectionFailures++;
-    // strcpy_P(log, PSTR("HTTP : connection failed"));
-    addLog(LOG_LEVEL_ERROR, F("HTTP : connection failed"));
-    return;
-  }
+  if (!try_connect_host(controller_number, client, ControllerSettings))
+    return false;
 
-  statusLED(true);
-  if (connectionFailures)
-    connectionFailures--;
+  String request = create_http_request_auth(
+      controller_number, element.controller_idx, ControllerSettings,
+      F("POST"), element.url, element.json.length());
+  request += element.json;
 
-  // This will send the request to the server
-  int len = buffer.length();
-  client.print(String("POST ") + url + F(" HTTP/1.1\r\n") +
-              F("Content-Length: ")+ len + F("\r\n") +
-              F("Host: ") + ControllerSettings.getHost() + F("\r\n") + authHeader +
-              F("Connection: close\r\n\r\n")
-              + buffer);
-
-  unsigned long timer = millis() + 200;
-  while (!client.available() && !timeOutReached(timer))
-    yield();
-
-  // Read all the lines of the reply from server and print them to Serial
-  while (client.available()) {
-    // String line = client.readStringUntil('\n');
-    String line;
-    safeReadStringUntil(client, line, '\n');
-
-    // String helper = line;
-    // line.toCharArray(log, 80);
-    addLog(LOG_LEVEL_DEBUG_MORE, line);
-
-    if (line.startsWith(F("HTTP/1.1 200 OK"))) {
-      // strcpy_P(log, PSTR("HTTP : Success"));
-      addLog(LOG_LEVEL_DEBUG_MORE, F("HTTP : Success"));
-      // success = true;
-    }
-    else if (line.startsWith(F("HTTP/1.1 4"))) {
-      addLog(LOG_LEVEL_ERROR, String(F("HTTP : Error: "))+line);
-    }
-    yield();
-  }
-  // strcpy_P(log, PSTR("HTTP : closing connection"));
-  addLog(LOG_LEVEL_DEBUG, F("HTTP : closing connection"));
-  client.flush();
-  client.stop();
+  return send_via_http(controller_number, client, request);
 }
 #endif

src/_C010.ino

@@ -41,20 +41,32 @@ boolean CPlugin_010(byte function, struct EventStruct *event, String& string)
     case CPLUGIN_PROTOCOL_SEND:
       {
         byte valueCount = getValueCountFromSensorType(event->sensorType);
+        C010_queue_element element(event, valueCount);
+        if (ExtraTaskSettings.TaskDeviceValueNames[0][0] == 0)
+          PluginCall(PLUGIN_GET_DEVICEVALUENAMES, event, dummyString);
+
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+
         for (byte x = 0; x < valueCount; x++)
         {
           bool isvalid;
           String formattedValue = formatUserVar(event, x, isvalid);
-          if (isvalid)
-            C010_Send(event, x, formattedValue);
-          if (valueCount > 1)
-          {
-            delayBackground(Settings.MessageDelay);
-            // unsigned long timer = millis() + Settings.MessageDelay;
-            // while (!timeOutReached(timer))
-            //   backgroundtasks();
+          if (isvalid) {
+            element.txt[x] = "";
+            element.txt[x] += ControllerSettings.Publish;
+            parseControllerVariables(element.txt[x], event, false);
+            element.txt[x].replace(F("%valname%"), ExtraTaskSettings.TaskDeviceValueNames[x]);
+            element.txt[x].replace(F("%value%"), formattedValue);
+            if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) {
+              char log[80];
+              element.txt[x].toCharArray(log, 80);
+              addLog(LOG_LEVEL_DEBUG_MORE, log);
+            }
           }
         }
+        success = C010_DelayHandler.addToQueue(element);
+        scheduleNextDelayQueue(TIMER_C010_DELAY_QUEUE, C010_DelayHandler.getNextScheduleTime());
         break;
       }
 
@@ -66,34 +78,20 @@ boolean CPlugin_010(byte function, struct EventStruct *event, String& string)
 //********************************************************************************
 // Generic UDP message
 //********************************************************************************
-void C010_Send(struct EventStruct *event, byte varIndex, const String& formattedValue)
-{
-  ControllerSettingsStruct ControllerSettings;
-  LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
-
-  // boolean success = false;
-  addLog(LOG_LEVEL_DEBUG, String(F("UDP  : sending to ")) + ControllerSettings.getHostPortString());
-  statusLED(true);
-
-  if (ExtraTaskSettings.TaskDeviceValueNames[0][0] == 0)
-    PluginCall(PLUGIN_GET_DEVICEVALUENAMES, event, dummyString);
-
-  String msg = "";
-  msg += ControllerSettings.Publish;
-  parseControllerVariables(msg, event, false);
-  msg.replace(F("%valname%"), ExtraTaskSettings.TaskDeviceValueNames[varIndex]);
-  msg.replace(F("%value%"), formattedValue);
-
-  if (wifiStatus == ESPEASY_WIFI_SERVICES_INITIALIZED) {
-    ControllerSettings.beginPacket(portUDP);
-    portUDP.write((uint8_t*)msg.c_str(),msg.length());
-    portUDP.endPacket();
+bool do_process_c010_delay_queue(int controller_number, const C010_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  while (element.txt[element.valuesSent] == "") {
+    // A non valid value, which we are not going to send.
+    // Increase sent counter until a valid value is found.
+    if (element.checkDone(true))
+      return true;
   }
 
-  if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) {
-    char log[80];
-    msg.toCharArray(log, 80);
-    addLog(LOG_LEVEL_DEBUG_MORE, log);
-  }
+  if (!try_connect_host(controller_number, portUDP, ControllerSettings))
+    return false;
+
+  portUDP.write(
+    (uint8_t*)element.txt[element.valuesSent].c_str(),
+              element.txt[element.valuesSent].length());
+  return element.checkDone(portUDP.endPacket() != 0);
 }
 #endif

src/_C011.ino

@@ -117,68 +117,53 @@ boolean CPlugin_011(byte function, struct EventStruct *event, String& string)
 
     case CPLUGIN_PROTOCOL_SEND:
       {
-      	HTTPSend011(event);
+      	success = Create_schedule_HTTP_C011(event);
+        break;
       }
 
   }
   return success;
 }
 
+//********************************************************************************
+// Generic HTTP request
+//********************************************************************************
+bool do_process_c011_delay_queue(int controller_number, const C011_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  WiFiClient client;
+  if (!try_connect_host(controller_number, client, ControllerSettings))
+    return false;
+
+  return send_via_http(controller_number, client, element.txt);
+}
+
+
 
 //********************************************************************************
-// Generic HTTP get request
+// Create request
 //********************************************************************************
-boolean HTTPSend011(struct EventStruct *event)
+boolean Create_schedule_HTTP_C011(struct EventStruct *event)
 {
+  int controller_number = CPLUGIN_ID_011;
   if (!WiFiConnected(100)) {
     return false;
   }
   ControllerSettingsStruct ControllerSettings;
   LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
 
-  String authHeader = "";
-  if ((SecuritySettings.ControllerUser[event->ControllerIndex][0] != 0) && (SecuritySettings.ControllerPassword[event->ControllerIndex][0] != 0))
-  {
-    base64 encoder;
-    String auth = SecuritySettings.ControllerUser[event->ControllerIndex];
-    auth += ":";
-    auth += SecuritySettings.ControllerPassword[event->ControllerIndex];
-    authHeader = F("Authorization: Basic ");
-    authHeader += encoder.encode(auth);
-    authHeader += F(" \r\n");
-  }
-
   C011_ConfigStruct customConfig;
   LoadCustomControllerSettings(event->ControllerIndex,(byte*)&customConfig, sizeof(customConfig));
   customConfig.zero_last();
 
-  boolean success = false;
-  addLog(LOG_LEVEL_DEBUG, String(F("HTTP : connecting to "))+
-      ControllerSettings.getHostPortString());
-
-  // Use WiFiClient class to create TCP connections
   WiFiClient client;
-  if (!ControllerSettings.connectToHost(client))
-  {
-    connectionFailures++;
-    addLog(LOG_LEVEL_ERROR, F("HTTP : connection failed"));
+  if (!try_connect_host(controller_number, client, ControllerSettings))
     return false;
-  }
-  statusLED(true);
-  if (connectionFailures)
-    connectionFailures--;
 
   if (ExtraTaskSettings.TaskDeviceValueNames[0][0] == 0)
     PluginCall(PLUGIN_GET_DEVICEVALUENAMES, event, dummyString);
 
-  String payload = String(customConfig.HttpMethod) + " /";
-  payload += customConfig.HttpUri;
-  payload += F(" HTTP/1.1\r\n");
-  payload += F("Host: ");
-  payload += ControllerSettings.getHostPortString();
-  payload += F("\r\n");
-  payload += authHeader;
-  payload += F("Connection: close\r\n");
+  String payload = create_http_request_auth(
+    controller_number, event->ControllerIndex, ControllerSettings,
+    String(customConfig.HttpMethod), customConfig.HttpUri);
 
   if (strlen(customConfig.HttpHeader) > 0)
     payload += customConfig.HttpHeader;
@@ -195,36 +180,9 @@ boolean HTTPSend011(struct EventStruct *event)
   }
   payload += F("\r\n");
 
-  // This will send the request to the server
-  client.print(payload);
-  addLog(LOG_LEVEL_DEBUG_MORE, payload);
-
-  unsigned long timer = millis() + 200;
-  while (!client.available() && !timeOutReached(timer))
-    yield();
-
-  // Read all the lines of the reply from server and print them to Serial
-  while (client.available()) {
-    // String line = client.readStringUntil('\n');
-    String line;
-    safeReadStringUntil(client, line, '\n');
-
-
-    // line.toCharArray(log, 80);
-    addLog(LOG_LEVEL_DEBUG_MORE, line);
-    if (line.startsWith(F("HTTP/1.1 2")))
-    {
-      addLog(LOG_LEVEL_DEBUG, F("HTTP : Success!"));
-      success = true;
-    }
-    yield();
-  }
-  addLog(LOG_LEVEL_DEBUG, F("HTTP : closing connection"));
-
-  client.flush();
-  client.stop();
-
-  return(success);
+  bool success = C011_DelayHandler.addToQueue(C011_queue_element(event->ControllerIndex, payload));
+  scheduleNextDelayQueue(TIMER_C011_DELAY_QUEUE, C011_DelayHandler.getNextScheduleTime());
+  return success;
 }
 
 // parses the string and returns only the the number of name/values we want

src/_C012.ino

@@ -34,34 +34,48 @@ boolean CPlugin_012(byte function, struct EventStruct *event, String& string)
 
      case CPLUGIN_PROTOCOL_SEND:
       {
-        if (wifiStatus != ESPEASY_WIFI_SERVICES_INITIALIZED) {
-          success = false;
-          break;
-        }
+        // Collect the values at the same run, to make sure all are from the same sample
+        byte valueCount = getValueCountFromSensorType(event->sensorType);
+        C012_queue_element element(event, valueCount);
+        if (ExtraTaskSettings.TaskDeviceValueNames[0][0] == 0)
+          PluginCall(PLUGIN_GET_DEVICEVALUENAMES, event, dummyString);
 
-        String postDataStr = F("");
-        const byte valueCount = getValueCountFromSensorType(event->sensorType);
-        success = CPlugin_012_send(event, valueCount);
+        ControllerSettingsStruct ControllerSettings;
+        LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
+
+        for (byte x = 0; x < valueCount; x++)
+        {
+          bool isvalid;
+          String formattedValue = formatUserVar(event, x, isvalid);
+          if (isvalid) {
+            element.txt[x] = F("update/V");
+            element.txt[x] += event->idx + x;
+            element.txt[x] += F("?value=");
+            element.txt[x] += formattedValue;
+            addLog(LOG_LEVEL_DEBUG_MORE, element.txt[x]);
+          }
+        }
+        success = C012_DelayHandler.addToQueue(element);
+        scheduleNextDelayQueue(TIMER_C012_DELAY_QUEUE, C012_DelayHandler.getNextScheduleTime());
         break;
       }
   }
   return success;
 }
 
-boolean CPlugin_012_send(struct EventStruct *event, int nrValues) {
-  String postDataStr = F("");
-  boolean success = true;
-  for (int i = 0; i < nrValues && success; ++i) {
-    postDataStr = F("update/V") ;
-    postDataStr += event->idx + i;
-    postDataStr += F("?value=");
-    postDataStr += formatUserVarNoCheck(event, i);
-    success = Blynk_get(postDataStr, event->ControllerIndex);
+//********************************************************************************
+// Process Queued Blynk request, with data set to NULL
+//********************************************************************************
+bool do_process_c012_delay_queue(int controller_number, const C012_queue_element& element, ControllerSettingsStruct& ControllerSettings) {
+  while (element.txt[element.valuesSent] == "") {
+    // A non valid value, which we are not going to send.
+    // Increase sent counter until a valid value is found.
+    if (element.checkDone(true))
+      return true;
   }
-  return success;
+  return element.checkDone(Blynk_get(element.txt[element.valuesSent], element.controller_idx));
 }
 
-
 boolean Blynk_get(const String& command, byte controllerIndex, float *data )
 {
   if (wifiStatus != ESPEASY_WIFI_SERVICES_INITIALIZED) {
@@ -70,16 +84,16 @@ boolean Blynk_get(const String& command, byte controllerIndex, float *data )
 
   ControllerSettingsStruct ControllerSettings;
   LoadControllerSettings(controllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
-  // Use WiFiClient class to create TCP connections
-  WiFiClient client;
-  if ((SecuritySettings.ControllerPassword[controllerIndex][0] == 0) || !ControllerSettings.connectToHost(client))
-  {
-    connectionFailures++;
-    addLog(LOG_LEVEL_ERROR, F("Blynk : connection failed"));
+
+  if ((SecuritySettings.ControllerPassword[controllerIndex][0] == 0)) {
+    addLog(LOG_LEVEL_ERROR, F("Blynk : No password set"));
     return false;
   }
-  if (connectionFailures)
-    connectionFailures--;
+
+  WiFiClient client;
+  if (!try_connect_host(CPLUGIN_ID_012, client, ControllerSettings))
+    return false;
+
 
   // We now create a URI for the request
   char request[300] = {0};
@@ -99,7 +113,7 @@ boolean Blynk_get(const String& command, byte controllerIndex, float *data )
   char log[80] = {0};
 
   // Read all the lines of the reply from server and log them
-  while (client.available()) {
+  while (client_available(client)) {
     String line;
     safeReadStringUntil(client, line, '\n');
     addLog(LOG_LEVEL_DEBUG_MORE, line);
@@ -134,13 +148,13 @@ boolean Blynk_get(const String& command, byte controllerIndex, float *data )
     }
     yield();
   }
-  strcpy_P(log, PSTR("HTTP : closing connection"));
+  strcpy_P(log, PSTR("HTTP : closing connection (012)"));
   addLog(LOG_LEVEL_DEBUG, log);
 
   client.flush();
   client.stop();
 
-  // important - backgroudtasks - free mem
+  // important - backgroundtasks - free mem
   timer = millis() + Settings.MessageDelay;
   while (!timeOutReached(timer))
               backgroundtasks();

src/_C013.ino

@@ -80,7 +80,7 @@ boolean CPlugin_013(byte function, struct EventStruct *event, String& string)
 
     case CPLUGIN_PROTOCOL_SEND:
       {
-        C013_Send(event, 0, UserVar[event->BaseVarIndex], 0);
+        C013_SendUDPTaskData(0, event->TaskIndex, event->TaskIndex);
         break;
       }
 
@@ -98,14 +98,6 @@ boolean CPlugin_013(byte function, struct EventStruct *event, String& string)
 //********************************************************************************
 // Generic UDP message
 //********************************************************************************
-void C013_Send(struct EventStruct *event, byte varIndex, float value, unsigned long longValue)
-{
-  ControllerSettingsStruct ControllerSettings;
-  LoadControllerSettings(event->ControllerIndex, (byte*)&ControllerSettings, sizeof(ControllerSettings));
-  statusLED(true);
-  C013_SendUDPTaskData(0, event->TaskIndex, event->TaskIndex);
-}
-
 void C013_SendUDPTaskInfo(byte destUnit, byte sourceTaskIndex, byte destTaskIndex)
 {
   if (!WiFiConnected(100)) {
@@ -121,19 +113,18 @@ void C013_SendUDPTaskInfo(byte destUnit, byte sourceTaskIndex, byte destTaskInde
   for (byte x = 0; x < VARS_PER_TASK; x++)
     strcpy(infoReply.ValueNames[x], ExtraTaskSettings.TaskDeviceValueNames[x]);
 
-  byte firstUnit = 1;
-  byte lastUnit = UNIT_MAX - 1;
   if (destUnit != 0)
   {
-    firstUnit = destUnit;
-    lastUnit = destUnit;
-  }
-  for (byte x = firstUnit; x <= lastUnit; x++)
-  {
-    if (x != Settings.Unit){
-      infoReply.destUnit = x;
-      C013_sendUDP(x, (byte*)&infoReply, sizeof(infoStruct));
-      delay(10);
+    infoReply.destUnit = destUnit;
+    C013_sendUDP(destUnit, (byte*)&infoReply, sizeof(infoStruct));
+    delay(10);
+  } else {
+    for (NodesMap::iterator it = Nodes.begin(); it != Nodes.end(); ++it) {
+      if (it->first != Settings.Unit) {
+        infoReply.destUnit = it->first;
+        C013_sendUDP(it->first, (byte*)&infoReply, sizeof(infoStruct));
+        delay(10);
+      }
     }
   }
   delay(50);
@@ -151,19 +142,18 @@ void C013_SendUDPTaskData(byte destUnit, byte sourceTaskIndex, byte destTaskInde
   for (byte x = 0; x < VARS_PER_TASK; x++)
     dataReply.Values[x] = UserVar[dataReply.sourceTaskIndex * VARS_PER_TASK + x];
 
-  byte firstUnit = 1;
-  byte lastUnit = UNIT_MAX - 1;
   if (destUnit != 0)
   {
-    firstUnit = destUnit;
-    lastUnit = destUnit;
-  }
-  for (byte x = firstUnit; x <= lastUnit; x++)
-  {
-    if (x != Settings.Unit){
-      dataReply.destUnit = x;
-      C013_sendUDP(x, (byte*) &dataReply, sizeof(dataStruct));
-      delay(10);
+    dataReply.destUnit = destUnit;
+    C013_sendUDP(destUnit, (byte*) &dataReply, sizeof(dataStruct));
+    delay(10);
+  } else {
+    for (NodesMap::iterator it = Nodes.begin(); it != Nodes.end(); ++it) {
+      if (it->first != Settings.Unit) {
+        dataReply.destUnit = it->first;
+        C013_sendUDP(it->first, (byte*) &dataReply, sizeof(dataStruct));
+        delay(10);
+      }
     }
   }
   delay(50);
@@ -177,9 +167,14 @@ void C013_sendUDP(byte unit, byte* data, byte size)
   if (!WiFiConnected(100)) {
     return;
   }
-  if (unit != 255)
-    if (Nodes[unit].ip[0] == 0)
+  NodesMap::iterator it;
+  if (unit != 255) {
+    it = Nodes.find(unit);
+    if (it == Nodes.end())
       return;
+    if (it->second.ip[0] == 0)
+      return;
+  }
   if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) {
     String log = F("C013 : Send UDP message to ");
     log += unit;
@@ -192,7 +187,7 @@ void C013_sendUDP(byte unit, byte* data, byte size)
   if (unit == 255)
     remoteNodeIP = {255, 255, 255, 255};
   else
-    remoteNodeIP = Nodes[unit].ip;
+    remoteNodeIP = it->second.ip;
   if (!beginWiFiUDP_randomPort(C013_portUDP)) return;
   if (C013_portUDP.beginPacket(remoteNodeIP, Settings.UDPPort) == 0) return;
   C013_portUDP.write(data, size);

src/_CPlugin_Helper.h

@@ -0,0 +1,671 @@
+#ifndef CPLUGIN_HELPER_H
+#define CPLUGIN_HELPER_H CPLUGIN_HELPER_H
+
+// These element classes should be defined as class, to be used as template.
+
+/*********************************************************************************************\
+ * MQTT_queue_element for all MQTT base controllers
+\*********************************************************************************************/
+class MQTT_queue_element {
+public:
+  MQTT_queue_element() : controller_idx(0), _retained(false) {}
+
+  MQTT_queue_element(int ctrl_idx,
+    const String& topic, const String& payload, boolean retained) :
+    controller_idx(ctrl_idx), _topic(topic), _payload(payload), _retained(retained)
+     {}
+
+  size_t getSize() const {
+    return sizeof(this) + _topic.length() + _payload.length();
+  }
+
+  int controller_idx;
+  String _topic;
+  String _payload;
+  boolean _retained;
+};
+
+/*********************************************************************************************\
+ * Simple queue element, only storing controller index and some String
+\*********************************************************************************************/
+class simple_queue_element_string_only {
+public:
+  simple_queue_element_string_only() : controller_idx(0) {}
+  simple_queue_element_string_only(int ctrl_idx, const String& req) :
+       controller_idx(ctrl_idx), txt(req) {}
+
+  size_t getSize() const {
+    return sizeof(this) + txt.length();
+  }
+
+  int controller_idx;
+  String txt;
+};
+
+
+/*********************************************************************************************\
+ * C001_queue_element for queueing requests for C001.
+\*********************************************************************************************/
+#define C001_queue_element simple_queue_element_string_only
+
+/*********************************************************************************************\
+ * C003_queue_element for queueing requests for C003 Nodo Telnet.
+\*********************************************************************************************/
+#define C003_queue_element simple_queue_element_string_only
+
+/*********************************************************************************************\
+ * C004_queue_element for queueing requests for C004 ThingSpeak.
+ *   Typical use case for Thingspeak is to only send values every N seconds/minutes.
+ *   So we just store everything needed to recreate the event when the time is ready.
+\*********************************************************************************************/
+class C004_queue_element {
+public:
+  C004_queue_element() : controller_idx(0), TaskIndex(0), idx(0), sensorType(0) {}
+  C004_queue_element(const struct EventStruct* event) :
+    controller_idx(event->ControllerIndex),
+    TaskIndex(event->TaskIndex),
+    idx(event->idx),
+    sensorType(event->sensorType) {}
+
+  size_t getSize() const {
+    return sizeof(this);
+  }
+
+  int controller_idx;
+  byte TaskIndex;
+  int idx;
+  byte sensorType;
+};
+
+/*********************************************************************************************\
+ * C007_queue_element for queueing requests for C007 Emoncms
+\*********************************************************************************************/
+class C007_queue_element {
+public:
+  C007_queue_element() : controller_idx(0), TaskIndex(0), idx(0), sensorType(0) {}
+  C007_queue_element(const struct EventStruct* event) :
+    controller_idx(event->ControllerIndex),
+    TaskIndex(event->TaskIndex),
+    idx(event->idx),
+    sensorType(event->sensorType) {}
+
+  size_t getSize() const {
+    return sizeof(this);
+  }
+
+  int controller_idx;
+  byte TaskIndex;
+  int idx;
+  byte sensorType;
+};
+
+/*********************************************************************************************\
+ * Base class for controllers that only send a single value per request and thus needs to
+ * keep track of the number of values already sent.
+\*********************************************************************************************/
+class queue_element_single_value_base {
+public:
+  queue_element_single_value_base() : controller_idx(0), TaskIndex(0), idx(0), valuesSent(0) {}
+  queue_element_single_value_base(const struct EventStruct* event, byte value_count) :
+    controller_idx(event->ControllerIndex),
+    TaskIndex(event->TaskIndex),
+    idx(event->idx),
+    valuesSent(0),
+    valueCount(value_count) {}
+
+  bool checkDone(bool succesfull) const {
+    if (succesfull) ++valuesSent;
+    return (valuesSent >= valueCount || valuesSent >= VARS_PER_TASK);
+  }
+
+  size_t getSize() const {
+    size_t total = sizeof(this);
+    for (int i = 0; i < VARS_PER_TASK; ++i) {
+      total += txt[i].length();
+    }
+    return total;
+  }
+
+  int controller_idx;
+  byte TaskIndex;
+  int idx;
+  String txt[VARS_PER_TASK];
+  mutable byte valuesSent;  // Value must be set by const function checkDone()
+  byte valueCount;
+};
+
+
+/*********************************************************************************************\
+ * C008_queue_element for queueing requests for 008: Generic HTTP
+ * Using queue_element_single_value_base
+\*********************************************************************************************/
+#define C008_queue_element queue_element_single_value_base
+
+/*********************************************************************************************\
+ * C009_queue_element for queueing requests for C009: FHEM HTTP.
+\*********************************************************************************************/
+class C009_queue_element {
+public:
+  C009_queue_element() : controller_idx(0) {}
+  C009_queue_element(int ctrl_idx, const String& URI, const String& JSON) :
+     controller_idx(ctrl_idx), url(URI), json(JSON) {}
+
+  size_t getSize() const {
+    return sizeof(this) + url.length() + json.length();
+  }
+
+  int controller_idx;
+  String url;
+  String json;
+};
+
+/*********************************************************************************************\
+ * C010_queue_element for queueing requests for 010: Generic UDP
+ * Using queue_element_single_value_base
+\*********************************************************************************************/
+#define C010_queue_element queue_element_single_value_base
+
+/*********************************************************************************************\
+ * C011_queue_element for queueing requests for 011: Generic HTTP Advanced
+\*********************************************************************************************/
+#define C011_queue_element simple_queue_element_string_only
+
+/*********************************************************************************************\
+ * C012_queue_element for queueing requests for 012: Blynk
+ * Using queue_element_single_value_base
+\*********************************************************************************************/
+#define C012_queue_element queue_element_single_value_base
+
+
+
+/*********************************************************************************************\
+ * ControllerDelayHandlerStruct
+\*********************************************************************************************/
+template<class T>
+struct ControllerDelayHandlerStruct {
+  ControllerDelayHandlerStruct() :
+      lastSend(0),
+      minTimeBetweenMessages(CONTROLLER_DELAY_QUEUE_DELAY_DFLT),
+      max_queue_depth(CONTROLLER_DELAY_QUEUE_DEPTH_DFLT),
+      attempt(0),
+      max_retries(CONTROLLER_DELAY_QUEUE_RETRY_DFLT),
+      delete_oldest(false) {}
+
+  void configureControllerSettings(const ControllerSettingsStruct& settings) {
+    minTimeBetweenMessages = settings.MinimalTimeBetweenMessages;
+    max_queue_depth = settings.MaxQueueDepth;
+    max_retries = settings.MaxRetry;
+    delete_oldest = settings.DeleteOldest;
+    // Set some sound limits when not configured
+    if (max_queue_depth == 0) max_queue_depth = CONTROLLER_DELAY_QUEUE_DEPTH_DFLT;
+    if (max_retries == 0) max_retries = CONTROLLER_DELAY_QUEUE_RETRY_DFLT;
+    if (minTimeBetweenMessages == 0) minTimeBetweenMessages = CONTROLLER_DELAY_QUEUE_DELAY_DFLT;
+    // No less than 10 msec between messages.
+    if (minTimeBetweenMessages < 10) minTimeBetweenMessages = 10;
+  }
+
+  bool queueFull(const T& element) const {
+    if (sendQueue.size() >= max_queue_depth) return true;
+
+    // Number of elements is not exceeding the limit, check memory
+    int freeHeap = ESP.getFreeHeap();
+    if (freeHeap > 5000) return false; // Memory is not an issue.
+    if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
+      String log = "Controller-";
+      log += element.controller_idx +1;
+      log += " : Memory used: ";
+      log += getQueueMemorySize();
+      log += " bytes ";
+      log += sendQueue.size();
+      log += " items ";
+      log += freeHeap;
+      log += " free";
+      addLog(LOG_LEVEL_DEBUG, log);
+    }
+    return true;
+  }
+
+  // Try to add to the queue, if permitted by "delete_oldest"
+  // Return false when no item was added.
+  bool addToQueue(const T& element) {
+    if (delete_oldest) {
+      // Force add to the queue.
+      // If max buffer is reached, the oldest in the queue (first to be served) will be removed.
+      while (queueFull(element)) {
+        sendQueue.pop_front();
+      }
+      sendQueue.emplace_back(element);
+      return true;
+    }
+    if (!queueFull(element)) {
+      sendQueue.emplace_back(element);
+      return true;
+    }
+    if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
+      String log = get_formatted_Controller_number(element.controller_idx);
+      log += " : queue full";
+      addLog(LOG_LEVEL_DEBUG, log);
+    }
+    return false;
+  }
+
+  // Get the next element.
+  // Remove front element when max_retries is reached.
+  bool getNext(T& element) {
+    if (sendQueue.empty()) return false;
+    if (attempt > max_retries) {
+      sendQueue.pop_front();
+      attempt = 0;
+      if (sendQueue.empty()) return false;
+    }
+    element = sendQueue.front();
+    return true;
+  }
+
+  // Mark as processed and return time to schedule for next process.
+  // Return 0 when nothing to process.
+  // @param remove_from_queue indicates whether the elements should be removed from the queue.
+  unsigned long markProcessed(bool remove_from_queue) {
+    if (sendQueue.empty()) return 0;
+    if (remove_from_queue) {
+      sendQueue.pop_front();
+      attempt = 0;
+    } else {
+      ++attempt;
+    }
+    lastSend = millis();
+    return getNextScheduleTime();
+  }
+
+  unsigned long getNextScheduleTime() const {
+    if (sendQueue.empty()) return 0;
+    unsigned long nextTime = lastSend + minTimeBetweenMessages;
+    if (timePassedSince(nextTime) > 0) {
+      nextTime = millis();
+    }
+    if (nextTime == 0) nextTime = 1; // Just to make sure it will be executed
+    return nextTime;
+  }
+
+  size_t getQueueMemorySize() const {
+    size_t totalSize = 0;
+    for (auto it = sendQueue.begin(); it != sendQueue.end(); ++it) {
+      totalSize += it->getSize();
+    }
+    return totalSize;
+  }
+
+  std::list<T> sendQueue;
+  unsigned long lastSend;
+  unsigned int minTimeBetweenMessages;
+  byte max_queue_depth;
+  byte attempt;
+  byte max_retries;
+  bool delete_oldest;
+};
+
+ControllerDelayHandlerStruct<MQTT_queue_element> MQTTDelayHandler;
+
+
+// This macro defines the code needed to create the 'process_c##NNN##_delay_queue()'
+// function and all needed objects and forward declarations.
+// It is a macro to prevent common typo errors.
+// This function will perform the (re)scheduling and mark if it is processed (and can be removed)
+// The controller itself must implement the 'do_process_c004_delay_queue' function to actually
+// send the data.
+// Its return value must state whether it can be marked 'Processed'.
+// N.B. some controllers only can send one value per iteration, so a returned "false" can mean it
+//      was still successful. The controller should keep track of the last value sent
+//      in the element stored in the queue.
+#define DEFINE_Cxxx_DELAY_QUEUE_MACRO(NNN, M) \
+                ControllerDelayHandlerStruct<C##NNN##_queue_element> C##NNN##_DelayHandler; \
+                bool do_process_c##NNN##_delay_queue(int controller_number, const C##NNN##_queue_element& element, ControllerSettingsStruct& ControllerSettings); \
+                void process_c##NNN##_delay_queue() { \
+                  C##NNN##_queue_element element; \
+                  if (!C##NNN##_DelayHandler.getNext(element)) return; \
+                  ControllerSettingsStruct ControllerSettings; \
+                  LoadControllerSettings(element.controller_idx, (byte*)&ControllerSettings, sizeof(ControllerSettings)); \
+                  C##NNN##_DelayHandler.configureControllerSettings(ControllerSettings); \
+                  if (!WiFiConnected(100)) { \
+                    scheduleNextDelayQueue(TIMER_C##NNN##_DELAY_QUEUE, C##NNN##_DelayHandler.getNextScheduleTime()); \
+                    return; \
+                  } \
+                  START_TIMER; \
+                  C##NNN##_DelayHandler.markProcessed(do_process_c##NNN##_delay_queue(M, element, ControllerSettings)); \
+                  STOP_TIMER(C##NNN##_DELAY_QUEUE); \
+                  scheduleNextDelayQueue(TIMER_C##NNN##_DELAY_QUEUE, C##NNN##_DelayHandler.getNextScheduleTime()); \
+                }
+
+// Define the function wrappers to handle the calling to Cxxx_DelayHandler etc.
+// If someone knows how to add leading zeros in macros, please be my guest :)
+#ifdef USES_C001
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(001, 1)
+#endif
+#ifdef USES_C003
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(003, 3)
+#endif
+#ifdef USES_C004
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(004, 4)
+#endif
+#ifdef USES_C007
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(007, 7)
+#endif
+#ifdef USES_C008
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(008, 8)
+#endif
+#ifdef USES_C009
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(009, 9)
+#endif
+#ifdef USES_C010
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(010, 10)
+#endif
+#ifdef USES_C011
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(011, 11)
+#endif
+#ifdef USES_C012
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(012, 12)
+#endif
+/*
+#ifdef USES_C013
+  DEFINE_Cxxx_DELAY_QUEUE_MACRO(013, 13)
+#endif
+*/
+// When extending this, also extend in Scheduler.ino:
+// void process_interval_timer(unsigned long id, unsigned long lasttimer)
+
+
+/*********************************************************************************************\
+ * Helper functions used in a number of controllers
+\*********************************************************************************************/
+bool safeReadStringUntil(Stream &input, String &str, char terminator, unsigned int maxSize = 1024, unsigned int timeout = 1000)
+{
+	int c;
+  const unsigned long start = millis();
+	const unsigned long timer = start + timeout;
+  unsigned long backgroundtasks_timer = start + 100;
+	str = "";
+
+	do {
+		//read character
+		c = input.read();
+		if (c >= 0) {
+			//found terminator, we're ok
+			if (c == terminator) {
+				return(true);
+			}
+			//found character, add to string
+			else{
+				str += char(c);
+				//string at max size?
+				if (str.length() >= maxSize) {
+					addLog(LOG_LEVEL_ERROR, F("Not enough bufferspace to read all input data!"));
+					return(false);
+				}
+			}
+		}
+    // We must run the backgroundtasks every now and then.
+    if (timeOutReached(backgroundtasks_timer)) {
+      backgroundtasks_timer += 100;
+      backgroundtasks();
+    } else {
+      yield();
+    }
+	} while (!timeOutReached(timer));
+
+	addLog(LOG_LEVEL_ERROR, F("Timeout while reading input data!"));
+	return(false);
+}
+
+bool valid_controller_number(int controller_number) {
+  if (controller_number < 0) return false;
+  return true;
+//  return getProtocolIndex(controller_number) <= protocolCount;
+}
+
+String get_formatted_Controller_number(int controller_number) {
+  if (!valid_controller_number(controller_number)) {
+    return F("C---");
+  }
+  String result = F("C");
+  if (controller_number < 100) result += '0';
+  if (controller_number < 10) result += '0';
+  result += controller_number;
+  return result;
+}
+
+String get_auth_header(int controller_index) {
+  String authHeader = "";
+  if (controller_index < CONTROLLER_MAX) {
+    if ((SecuritySettings.ControllerUser[controller_index][0] != 0) &&
+        (SecuritySettings.ControllerPassword[controller_index][0] != 0))
+    {
+      base64 encoder;
+      String auth = SecuritySettings.ControllerUser[controller_index];
+      auth += ":";
+      auth += SecuritySettings.ControllerPassword[controller_index];
+      authHeader = F("Authorization: Basic ");
+      authHeader += encoder.encode(auth);
+      authHeader += F(" \r\n");
+    }
+  } else {
+    addLog(LOG_LEVEL_ERROR, F("Invalid controller index"));
+  }
+  return authHeader;
+}
+
+String get_user_agent_request_header_field() {
+  static unsigned int agent_size = 20;
+  String request;
+  request.reserve(agent_size);
+  request = F("User-Agent: ");
+  request += F("ESP Easy/");
+  request += BUILD;
+  request += '/';
+  request += String(CRCValues.compileDate);
+  request += ' ';
+  request += String(CRCValues.compileTime);
+  request += F("\r\n");
+  agent_size = request.length();
+  return request;
+}
+
+String do_create_http_request(
+    const String& hostportString,
+    const String& method, const String& uri,
+    const String& auth_header, const String& additional_options,
+    int content_length) {
+  int estimated_size = hostportString.length() + method.length()
+                       + uri.length() + auth_header.length()
+                       + additional_options.length()
+                       + 42;
+  if (content_length >= 0) estimated_size += 25;
+  String request;
+  request.reserve(estimated_size);
+  request += method;
+  request += ' ';
+  if (!uri.startsWith("/")) request += '/';
+  request += uri;
+  request += F(" HTTP/1.1\r\n");
+  if (content_length >= 0) {
+    request += F("Content-Length: ");
+    request += content_length;
+    request += F("\r\n");
+  }
+  request += F("Host: ");
+  request += hostportString;
+  request += F("\r\n");
+  request += auth_header;
+  request += additional_options;
+  request += get_user_agent_request_header_field();
+  request += F("Connection: close\r\n");
+  request += F("\r\n");
+  addLog(LOG_LEVEL_DEBUG, request);
+  return request;
+}
+
+String do_create_http_request(
+    const String& hostportString,
+    const String& method, const String& uri) {
+  return do_create_http_request(hostportString, method, uri,
+    "", // auth_header
+    "", // additional_options
+    -1  // content_length
+  );
+}
+
+String do_create_http_request(
+    int controller_number, ControllerSettingsStruct& ControllerSettings,
+    const String& method, const String& uri,
+    int content_length) {
+  const bool defaultport = ControllerSettings.Port == 0 || ControllerSettings.Port == 80;
+  return do_create_http_request(
+    defaultport ? ControllerSettings.getHost() : ControllerSettings.getHostPortString(),
+    method,
+    uri,
+    "", // auth_header
+    "", // additional_options
+    content_length);
+}
+
+String create_http_request_auth(
+    int controller_number, int controller_index, ControllerSettingsStruct& ControllerSettings,
+    const String& method, const String& uri,
+    int content_length) {
+  const bool defaultport = ControllerSettings.Port == 0 || ControllerSettings.Port == 80;
+  return do_create_http_request(
+    defaultport ? ControllerSettings.getHost() : ControllerSettings.getHostPortString(),
+    method,
+    uri,
+    get_auth_header(controller_index),
+    "", // additional_options
+    content_length);
+}
+
+String create_http_get_request(int controller_number, ControllerSettingsStruct& ControllerSettings,
+    const String& uri) {
+  return do_create_http_request(controller_number, ControllerSettings, F("GET"), uri, -1);
+}
+
+String create_http_request_auth(int controller_number, int controller_index, ControllerSettingsStruct& ControllerSettings,
+    const String& method, const String& uri) {
+  return create_http_request_auth(controller_number, controller_index, ControllerSettings, method, uri, -1);
+}
+
+void log_connecting_to(const String& prefix, int controller_number, ControllerSettingsStruct& ControllerSettings) {
+  if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
+    String log = prefix;
+    log += get_formatted_Controller_number(controller_number);
+    log += F(" connecting to ");
+    log += ControllerSettings.getHostPortString();
+    addLog(LOG_LEVEL_DEBUG, log);
+  }
+}
+
+void log_connecting_fail(const String& prefix, int controller_number, ControllerSettingsStruct& ControllerSettings) {
+  if (loglevelActiveFor(LOG_LEVEL_ERROR)) {
+    String log = prefix;
+    log += get_formatted_Controller_number(controller_number);
+    log += F(" connection failed");
+    addLog(LOG_LEVEL_ERROR, log);
+  }
+}
+
+bool count_connection_results(bool success, const String& prefix, int controller_number, ControllerSettingsStruct& ControllerSettings) {
+  if (!success)
+  {
+    connectionFailures++;
+    log_connecting_fail(prefix, controller_number, ControllerSettings);
+    return false;
+  }
+  statusLED(true);
+  if (connectionFailures)
+    connectionFailures--;
+  return true;
+}
+
+bool try_connect_host(int controller_number, WiFiUDP& client, ControllerSettingsStruct& ControllerSettings) {
+  log_connecting_to(F("UDP  : "), controller_number, ControllerSettings);
+  bool success = ControllerSettings.beginPacket(client) != 0;
+  return count_connection_results(
+      success,
+      F("UDP  : "), controller_number, ControllerSettings);
+}
+
+bool try_connect_host(int controller_number, WiFiClient& client, ControllerSettingsStruct& ControllerSettings) {
+  // Use WiFiClient class to create TCP connections
+  log_connecting_to(F("HTTP : "), controller_number, ControllerSettings);
+  bool success = ControllerSettings.connectToHost(client);
+  return count_connection_results(
+      success,
+      F("HTTP : "), controller_number, ControllerSettings);
+}
+
+// Use "client.available() || client.connected()" to read all lines from slow servers.
+// See: https://github.com/esp8266/Arduino/pull/5113
+//      https://github.com/esp8266/Arduino/pull/1829
+bool client_available(WiFiClient& client) {
+  return client.available() || client.connected();
+}
+
+bool send_via_http(const String& logIdentifier, WiFiClient& client, const String& postStr) {
+  bool success = false;
+  // This will send the request to the server
+  client.print(postStr);
+
+  unsigned long timer = millis() + 200;
+  while (!client.available() && !timeOutReached(timer))
+    delay(1);
+
+  // Read all the lines of the reply from server and print them to Serial
+  while (client_available(client) && !success) {
+    //   String line = client.readStringUntil('\n');
+    String line;
+    safeReadStringUntil(client, line, '\n');
+
+    if (loglevelActiveFor(LOG_LEVEL_DEBUG_MORE)) {
+      if (line.length() > 80) {
+        addLog(LOG_LEVEL_DEBUG_MORE, line.substring(0, 80));
+      } else {
+        addLog(LOG_LEVEL_DEBUG_MORE, line);
+      }
+    }
+    if (line.startsWith(F("HTTP/1.1 2")))
+    {
+      success = true;
+      if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
+        String log = F("HTTP : ");
+        log += logIdentifier;
+        log += F(" Success! ");
+        log += line;
+        addLog(LOG_LEVEL_DEBUG, log);
+      }
+    } else if (line.startsWith(F("HTTP/1.1 4"))) {
+      if (loglevelActiveFor(LOG_LEVEL_ERROR)) {
+        String log = F("HTTP : ");
+        log += logIdentifier;
+        log += F(" Error: ");
+        log += line;
+        addLog(LOG_LEVEL_ERROR, log);
+      }
+      addLog(LOG_LEVEL_DEBUG_MORE, postStr);
+    }
+    yield();
+  }
+  if (loglevelActiveFor(LOG_LEVEL_DEBUG)) {
+    String log = F("HTTP : ");
+    log += logIdentifier;
+    log += F(" closing connection");
+    addLog(LOG_LEVEL_DEBUG, log);
+  }
+
+  client.flush();
+  client.stop();
+  return success;
+}
+
+bool send_via_http(int controller_number, WiFiClient& client, const String& postStr) {
+  return send_via_http(get_formatted_Controller_number(controller_number), client, postStr);
+}
+
+
+#endif // CPLUGIN_HELPER_H

src/_N001_Email.ino

@@ -89,6 +89,24 @@ boolean NPlugin_001_send(const NotificationSettingsStruct& notificationsettings,
 			"X-Mailer: EspEasy v$espeasyversion\r\n\r\n"
 			);
 
+    String email_address = notificationsettings.Sender;
+		int pos_less = email_address.indexOf('<');
+		if (pos_less == -1) {
+			// No email address markup
+			mailheader.replace(String(F("$nodename")), Settings.Name);
+			mailheader.replace(String(F("$emailfrom")), notificationsettings.Sender);
+		} else {
+			String senderName = email_address.substring(0, pos_less);
+			senderName.replace("\"", ""); // Remove quotes
+			String address = email_address.substring(pos_less + 1);
+			address.replace("<", "");
+			address.replace(">", "");
+			address.trim();
+			senderName.trim();
+			mailheader.replace(String(F("$nodename")), senderName);
+			mailheader.replace(String(F("$emailfrom")), address);
+		}
+
 		mailheader.replace(String(F("$nodename")), Settings.Name);
 		mailheader.replace(String(F("$emailfrom")), notificationsettings.Sender);
 		mailheader.replace(String(F("$ato")), notificationsettings.Receiver);

src/_P029_Output.ino

@@ -53,7 +53,7 @@ boolean Plugin_029(byte function, struct EventStruct *event, String& string)
         addHtml(F("<TR><TD>IDX:<TD>"));
         String id = F("TDID");   //="taskdeviceid"
         id += controllerNr + 1;
-        addNumericBox(id, Settings.TaskDeviceID[controllerNr][event->TaskIndex], 0, 9999);
+        addNumericBox(id, Settings.TaskDeviceID[controllerNr][event->TaskIndex], 0, DOMOTICZ_MAX_IDX);
         success = true;
         break;
       }

src/_P044_P1WifiGateway.ino

@@ -16,7 +16,7 @@
 
 #define P044_STATUS_LED 12
 #define P044_BUFFER_SIZE 1024
-#define P044_NETBUF_SIZE 600
+#define P044_NETBUF_SIZE 128
 #define P044_DISABLED 0
 #define P044_WAITING 1
 #define P044_READING 2
@@ -31,7 +31,7 @@ boolean CRCcheck = false;
 unsigned int currCRC = 0;
 int checkI = 0;
 
-WiFiServer *P1GatewayServer;
+WiFiServer *P1GatewayServer = nullptr;
 WiFiClient P1GatewayClient;
 
 boolean Plugin_044(byte function, struct EventStruct *event, String& string)
@@ -125,7 +125,11 @@ boolean Plugin_044(byte function, struct EventStruct *event, String& string)
           #if defined(ESP32)
             Serial.begin(ExtraTaskSettings.TaskDevicePluginConfigLong[1], serialconfig);
           #endif
-          if (P1GatewayServer) P1GatewayServer->close();
+          if (P1GatewayServer)
+          {
+            P1GatewayServer->close();
+            delete P1GatewayServer;
+          }
           P1GatewayServer = new WiFiServer(ExtraTaskSettings.TaskDevicePluginConfigLong[0]);
           P1GatewayServer->begin();
 
@@ -164,7 +168,7 @@ boolean Plugin_044(byte function, struct EventStruct *event, String& string)
       {
         if (P1GatewayServer) {
           P1GatewayServer->close();
-          //FIXME: shouldnt P1P1GatewayServer be deleted?
+          delete P1GatewayServer;
           P1GatewayServer = NULL;
         }
         success = true;
@@ -175,7 +179,6 @@ boolean Plugin_044(byte function, struct EventStruct *event, String& string)
       {
         if (Plugin_044_init)
         {
-          size_t bytes_read;
           if (P1GatewayServer->hasClient())
           {
             if (P1GatewayClient) P1GatewayClient.stop();
@@ -186,24 +189,25 @@ boolean Plugin_044(byte function, struct EventStruct *event, String& string)
           if (P1GatewayClient.connected())
           {
             connectionState = 1;
-            uint8_t net_buf[P044_BUFFER_SIZE];
+            uint8_t net_buf[P044_NETBUF_SIZE];
             int count = P1GatewayClient.available();
             if (count > 0)
             {
-              if (count > P044_BUFFER_SIZE)
-                count = P044_BUFFER_SIZE;
-              bytes_read = P1GatewayClient.read(net_buf, count);
-              Serial.write(net_buf, bytes_read);
+              size_t net_bytes_read;
+              if (count > P044_NETBUF_SIZE)
+                count = P044_NETBUF_SIZE;
+              net_bytes_read = P1GatewayClient.read(net_buf, count);
+              Serial.write(net_buf, net_bytes_read);
               Serial.flush(); // Waits for the transmission of outgoing serial data to complete
 
-              if (count == P044_BUFFER_SIZE) // if we have a full buffer, drop the last position to stuff with string end marker
+              if (count == P044_NETBUF_SIZE) // if we have a full buffer, drop the last position to stuff with string end marker
               {
                 count--;
                 // and log buffer full situation
                 addLog(LOG_LEVEL_ERROR, F("P1   : Error: network buffer full!"));
               }
               net_buf[count] = 0; // before logging as a char array, zero terminate the last position to be safe.
-              char log[P044_BUFFER_SIZE + 40];
+              char log[P044_NETBUF_SIZE + 40];
               sprintf_P(log, PSTR("P1   : Error: N>: %s"), (char*)net_buf);
               addLog(LOG_LEVEL_DEBUG, log);
             }

src/__CPlugin.ino

@@ -148,11 +148,19 @@ byte CPluginCall(byte Function, struct EventStruct *event)
   {
     // Unconditional calls to all plugins
     case CPLUGIN_PROTOCOL_ADD:
-      for (x = 0; x < CPLUGIN_MAX; x++)
-        if (CPlugin_id[x] != 0){
+      for (x = 0; x < CPLUGIN_MAX; x++) {
+        if (CPlugin_id[x] != 0) {
+          const unsigned int next_ProtocolIndex = protocolCount + 2;
+          if (next_ProtocolIndex > Protocol.size()) {
+            // Increase with 8 to get some compromise between number of resizes and wasted space
+            unsigned int newSize = Protocol.size();
+            newSize = newSize + 8 - (newSize % 8);
+            Protocol.resize(newSize);
+          }
           checkRAM(F("CPluginCallADD"),x);
           CPlugin_ptr[x](Function, event, dummyString);
         }
+      }
       return true;
       break;
 

src/__Plugin.ino

@@ -1113,6 +1113,15 @@ byte PluginCall(byte Function, struct EventStruct *event, String& str)
     case PLUGIN_UNCONDITIONAL_POLL:
       for (byte x = 0; x < PLUGIN_MAX; x++) {
         if (Plugin_id[x] != 0){
+          if (Function == PLUGIN_DEVICE_ADD) {
+            const unsigned int next_DeviceIndex = deviceCount + 2;
+            if (next_DeviceIndex > Device.size()) {
+              // Increase with 16 to get some compromise between number of resizes and wasted space
+              unsigned int newSize = Device.size();
+              newSize = newSize + 16 - (newSize % 16);
+              Device.resize(newSize);
+            }
+          }
           START_TIMER;
           Plugin_ptr[x](Function, event, str);
           STOP_TIMER_TASK(x,Function);