[LoRa/TTN] Add Sample Set Initiator to mark samples of the same set

For LoRa, it may take a while to send out samples of different plugins.
This makes it hard to match samples belonging to the same set after receiving them.
The Sample Set Initiator is a task index which should always be considered as the first of a set samples sent to the controller.
For example when triggering to send a set of samples from rules, its order can be always the same. 
Setting the initiator to the first one of the sequence, the same sample set counter will be sent along with the samples.

docs/source/Controller/C018.rst

@@ -150,6 +150,8 @@ Decoding Data
 -------------
 
 
+Controller Settings
+-------------------
 
 
 

docs/source/Controller/_Controller.rst

@@ -61,8 +61,12 @@ before WiFi connection is made or during lost connection.
 - **Max Queue Depth** - Maximum length of the buffer queue to keep unsent messages.
 - **Max Retries** - Maximum number of retries to send a message.
 - **Full Queue Action** - How to handle when queue is full, ignore new or delete oldest message.
-- **Client Timeout** - Timeout in msec for an network connection used by the controller.
 - **Check Reply** - When set to false, a sent message is considered always successful.
+- **Client Timeout** - Timeout in msec for an network connection used by the controller.
+- **Sample Set Initiator** - Some controllers (e.g. C018 LoRa/TTN) can mark samples to belong to a set of samples. A new sample from set task index will increment this counter.
+  Especially useful for controllers which cannot send samples in a burst. This makes the receiving time stamp useless to detect what samples were taken around the same time.
+  The sample set counter value can help matching received samples to a single set.
+
 
 
 Sample ThingSpeak configuration
@@ -93,3 +97,5 @@ MQTT related settings
 - **Controller lwl topic** - Topic to which LWT (Last Will Testament) messages should be sent.
 - **LWT Connect Message** - Connection established message.
 - **LWT Disconnect Message** - Connection lost message (sent to broker during connect and published by broker when connection is lost)
+
+

docs/source/Controller/_controller_substitutions.repl

@@ -184,6 +184,6 @@
 .. |C018_github| replace:: C018.ino
 .. _C018_github: https://github.com/letscontrolit/ESPEasy/blob/mega/src/_C018.ino
 .. |C018_usedby| replace:: `.`
-.. |C018_shortinfo| replace:: `.`
+.. |C018_shortinfo| replace:: `Controller for the LoRaWAN/TTN network supporting RN2384 (434/868 MHz) and RN2903 (915 MHz)`
 .. |C018_maintainer| replace:: `TD-er`
 .. |C018_compileinfo| replace:: `.`

misc/TTN/packed_decoder.js

@@ -13,20 +13,20 @@ function Decoder(bytes, port) {
 
   if (port === 1) {
     // Single value
-    if (bytes.length === 8) {
-      return decode(bytes, [pluginid, uint16, uint8, int32_1e4], ['plugin_id', 'IDX', 'valuecount', 'val_1']);
+    if (bytes.length === 9) {
+      return decode(bytes, [pluginid, uint16, uint8, uint8, int32_1e4], ['plugin_id', 'IDX', 'samplesetcount', 'valuecount', 'val_1']);
     }
     // Dual value
-    if (bytes.length === 12) {
-      return decode(bytes, [pluginid, uint16, uint8, int32_1e4, int32_1e4], ['plugin_id', 'IDX', 'valuecount', 'val_1', 'val_2']);
+    if (bytes.length === 13) {
+      return decode(bytes, [pluginid, uint16, uint8, uint8, int32_1e4, int32_1e4], ['plugin_id', 'IDX', 'samplesetcount', 'valuecount', 'val_1', 'val_2']);
     }
     // Triple value
-    if (bytes.length === 16) {
-      return decode(bytes, [pluginid, uint16, uint8, int32_1e4, int32_1e4, int32_1e4], ['plugin_id', 'IDX', 'valuecount', 'val_1', 'val_2', 'val_3']);
+    if (bytes.length === 17) {
+      return decode(bytes, [pluginid, uint16, uint8, uint8, int32_1e4, int32_1e4, int32_1e4], ['plugin_id', 'IDX', 'samplesetcount', 'valuecount', 'val_1', 'val_2', 'val_3']);
     }
     // Quad value
-    if (bytes.length === 20) {
-      return decode(bytes, [pluginid, uint16, uint8, int32_1e4, int32_1e4, int32_1e4, int32_1e4], ['plugin_id', 'IDX', 'valuecount', 'val_1', 'val_2', 'val_3', 'val_4']);
+    if (bytes.length === 21) {
+      return decode(bytes, [pluginid, uint16, uint8, uint8, int32_1e4, int32_1e4, int32_1e4, int32_1e4], ['plugin_id', 'IDX', 'samplesetcount', 'valuecount', 'val_1', 'val_2', 'val_3', 'val_4']);
     }
   }
 

src/ESPEasy-Globals.h

@@ -357,7 +357,8 @@ void check_size() {
 #define CONTROLLER_LWT_CONNECT_MESSAGE      16
 #define CONTROLLER_LWT_DISCONNECT_MESSAGE   17
 #define CONTROLLER_TIMEOUT                  18
-#define CONTROLLER_ENABLED                  19  // Keep this as last, is used to loop over all parameters
+#define CONTROLLER_SAMPLE_SET_INITIATOR       19
+#define CONTROLLER_ENABLED                  20  // Keep this as last, is used to loop over all parameters
 
 #define NPLUGIN_PROTOCOL_ADD                1
 #define NPLUGIN_GET_DEVICENAME              2
@@ -1105,6 +1106,7 @@ struct ControllerSettingsStruct
     DeleteOldest = false;
     ClientTimeout = CONTROLLER_CLIENTTIMEOUT_DFLT;
     MustCheckReply = false;
+    SampleSetInitiator = 0;
     for (byte i = 0; i < 4; ++i) {
       IP[i] = 0;
     }
@@ -1131,6 +1133,7 @@ struct ControllerSettingsStruct
   boolean       DeleteOldest; // Action to perform when buffer full, delete oldest, or ignore newest.
   unsigned int  ClientTimeout;
   boolean       MustCheckReply; // When set to false, a sent message is considered always successful.
+  uint8_t       SampleSetInitiator; // The first plugin to start a sample set.
 
   void validate() {
     if (Port > 65535) Port = 0;
@@ -1616,7 +1619,8 @@ struct ProtocolStruct
 {
   ProtocolStruct() :
     defaultPort(0), Number(0), usesMQTT(false), usesAccount(false), usesPassword(false),
-    usesTemplate(false), usesID(false), Custom(false), usesHost(true), usesPort(true), usesQueue(true) {}
+    usesTemplate(false), usesID(false), Custom(false), usesHost(true), usesPort(true), 
+    usesQueue(true), usesSampleSets(false) {}
   uint16_t defaultPort;
   byte Number;
   bool usesMQTT : 1;
@@ -1628,6 +1632,7 @@ struct ProtocolStruct
   bool usesHost : 1;
   bool usesPort : 1;
   bool usesQueue : 1;
+  bool usesSampleSets : 1;
 };
 typedef std::vector<ProtocolStruct> ProtocolVector;
 ProtocolVector Protocol;

src/WebServer_ControllerPage.ino

@@ -252,6 +252,9 @@ void handle_controllers_ControllerSettingsPage(byte controllerindex)
       }
       addControllerParameterForm(ControllerSettings, controllerindex, CONTROLLER_CHECK_REPLY);
       addControllerParameterForm(ControllerSettings, controllerindex, CONTROLLER_TIMEOUT);
+      if (Protocol[ProtocolIndex].usesSampleSets) {
+        addControllerParameterForm(ControllerSettings, controllerindex, CONTROLLER_SAMPLE_SET_INITIATOR);
+      }
 
       if (Protocol[ProtocolIndex].usesAccount || Protocol[ProtocolIndex].usesPassword) {
         addTableSeparator(F("Credentials"), 2, 3);

src/WebServer_Markup_Forms.ino

@@ -78,6 +78,16 @@ void addFormFloatNumberBox(const String& label, const String& id, float value, f
   addFloatNumberBox(id, value, min, max);
 }
 
+// ********************************************************************************
+// Add a task selector form
+// ********************************************************************************
+void addTaskSelectBox(const String& label, const String& id, int choice)
+{
+  addRowLabel_tr_id(label, id);
+  addTaskSelect(id, choice);
+}
+
+
 // ********************************************************************************
 // Add a Text Box form
 // ********************************************************************************

src/_C016.ino

@@ -48,6 +48,7 @@ bool CPlugin_016(byte function, struct EventStruct *event, String& string)
         Protocol[protocolCount].usesID = false;
         Protocol[protocolCount].usesHost = false;
         Protocol[protocolCount].usesPort = false;
+        Protocol[protocolCount].usesSampleSets = false;
         break;
       }
 

src/_C018.ino

@@ -37,11 +37,12 @@ struct C018_data_struct {
     autobaud_success = false;
   }
 
-  bool init(const int8_t serial_rx, const int8_t serial_tx, unsigned long baudrate, bool joinIsOTAA) {
+  bool init(const int8_t serial_rx, const int8_t serial_tx, unsigned long baudrate, bool joinIsOTAA, uint8_t sampleSet_Initiator) {
     if ((serial_rx < 0) || (serial_tx < 0)) {
       // Both pins are needed, or else no serial possible
       return false;
     }
+    sampleSetInitiator = sampleSet_Initiator;
 
     if (isInitialized()) {
       // Check to see if serial parameters have changed.
@@ -178,6 +179,17 @@ struct C018_data_struct {
     return cacheSysVer;
   }
 
+  uint8_t getSampleSetCount() const { return sampleSetCounter; }
+
+  uint8_t getSampleSetCount(uint8_t taskIndex) {
+    if (sampleSetInitiator == taskIndex)
+    {
+      ++sampleSetCounter;
+    }
+    return sampleSetCounter;
+  }
+
+
 private:
 
   void C018_logError(const String& command) {
@@ -214,6 +226,8 @@ private:
   String         cacheDevAddr;
   String         cacheHWEUI;
   String         cacheSysVer;
+  uint8_t        sampleSetCounter = 0;
+  uint8_t        sampleSetInitiator = 0;
   bool           autobaud_success = false;
 } C018_data;
 
@@ -277,13 +291,14 @@ bool CPlugin_018(byte function, struct EventStruct *event, String& string)
   {
     case CPLUGIN_PROTOCOL_ADD:
     {
-      Protocol[++protocolCount].Number     = CPLUGIN_ID_018;
-      Protocol[protocolCount].usesMQTT     = false;
-      Protocol[protocolCount].usesAccount  = true;
-      Protocol[protocolCount].usesPassword = true;
-      Protocol[protocolCount].defaultPort  = 1;
-      Protocol[protocolCount].usesID       = true;
-      Protocol[protocolCount].usesHost     = false;
+      Protocol[++protocolCount].Number       = CPLUGIN_ID_018;
+      Protocol[protocolCount].usesMQTT       = false;
+      Protocol[protocolCount].usesAccount    = true;
+      Protocol[protocolCount].usesPassword   = true;
+      Protocol[protocolCount].defaultPort    = 1;
+      Protocol[protocolCount].usesID         = true;
+      Protocol[protocolCount].usesHost       = false;
+      Protocol[protocolCount].usesSampleSets = true;
       break;
     }
 
@@ -315,7 +330,9 @@ bool CPlugin_018(byte function, struct EventStruct *event, String& string)
       LoadCustomControllerSettings(event->ControllerIndex, (byte *)&customConfig, sizeof(customConfig));
       customConfig.validate();
 
-      C018_data.init(customConfig.rxpin, customConfig.txpin, customConfig.baudrate, customConfig.joinmethod == C018_USE_OTAA);
+      C018_data.init(customConfig.rxpin, customConfig.txpin, customConfig.baudrate, 
+                     customConfig.joinmethod == C018_USE_OTAA,
+                     ControllerSettings.SampleSetInitiator);
 
       C018_data.setFrequencyPlan(static_cast<FREQ_PLAN>(customConfig.frequencyplan));
 
@@ -402,7 +419,6 @@ bool CPlugin_018(byte function, struct EventStruct *event, String& string)
       addFormNumericBox(F("Baudrate"), F(C018_BAUDRATE_LABEL), customConfig.baudrate, 2400, 115200);
       addUnit(F("baud"));
 
-
       addTableSeparator(F("Device Status"), 2, 3);
 
       // Some information on detected device
@@ -430,6 +446,9 @@ bool CPlugin_018(byte function, struct EventStruct *event, String& string)
       addRowLabel(F("Last Command Error"));
       addHtml(C018_data.getLastErrorInvalidParam());
 
+      addRowLabel(F("Sample Set Counter"));
+      addHtml(String(C018_data.getSampleSetCount()));
+
       addRowLabel(F("Status"));
       addHtml(String(C018_data.getRawStatus()));
 
@@ -487,7 +506,7 @@ bool CPlugin_018(byte function, struct EventStruct *event, String& string)
     case CPLUGIN_PROTOCOL_SEND:
     {
       byte valueCount = getValueCountFromSensorType(event->sensorType);
-      success = C018_DelayHandler.addToQueue(C018_queue_element(event, valueCount));
+      success = C018_DelayHandler.addToQueue(C018_queue_element(event, valueCount, C018_data.getSampleSetCount(event->TaskIndex)));
       scheduleNextDelayQueue(TIMER_C018_DELAY_QUEUE, C018_DelayHandler.getNextScheduleTime());
 
       break;

src/_CPlugin_Helper.h

@@ -319,14 +319,15 @@ public:
 class C018_queue_element {
 public:
 
-  C018_queue_element() : idx(0), TaskIndex(0), sensorType(0) {}
+  C018_queue_element() : idx(0), TaskIndex(0), sensorType(0), valueCount(0), sampleSetCount(0) {}
 
-  C018_queue_element(const struct EventStruct *event, byte value_count) :
+  C018_queue_element(const struct EventStruct *event, byte value_count, uint8_t sampleSet_count) :
     controller_idx(event->ControllerIndex),
     idx(event->idx),
     TaskIndex(event->TaskIndex),
     sensorType(event->sensorType),
-    valueCount(value_count)
+    valueCount(value_count),
+    sampleSetCount(sampleSet_count)
   {
     const byte BaseVarIndex = TaskIndex * VARS_PER_TASK;
 
@@ -344,6 +345,7 @@ public:
     data[pos++] = Settings.TaskDeviceNumber[TaskIndex];
     data[pos++] = (idx & 0xFF);
     data[pos++] = ((idx >> 8) & 0xFF);
+    data[pos++] = sampleSetCount;
     data[pos++] = valueCount;
 
     for (int i = 0; i < valueCount; ++i) {
@@ -366,6 +368,7 @@ public:
   byte TaskIndex;
   byte sensorType;
   byte valueCount;
+  uint8_t sampleSetCount;
 };
 
 /*********************************************************************************************\

src/_CPlugin_Helper_webform.ino

@@ -36,6 +36,7 @@ String getControllerParameterName(byte ProtocolIndex, byte parameterIdx, bool di
     case CONTROLLER_LWT_CONNECT_MESSAGE:      name = F("LWT Connect Message");    break;
     case CONTROLLER_LWT_DISCONNECT_MESSAGE:   name = F("LWT Disconnect Message"); break;
     case CONTROLLER_TIMEOUT:                  name = F("Client Timeout");         break;
+    case CONTROLLER_SAMPLE_SET_INITIATOR:     name = F("Sample Set Initiator");     break;
 
     case CONTROLLER_ENABLED:
 
@@ -170,6 +171,9 @@ void addControllerParameterForm(const ControllerSettingsStruct& ControllerSettin
       addFormNumericBox(displayName, internalName, ControllerSettings.ClientTimeout, 10, CONTROLLER_CLIENTTIMEOUT_MAX);
       addUnit(F("ms"));
       break;
+    case CONTROLLER_SAMPLE_SET_INITIATOR:
+      addTaskSelectBox(displayName, internalName, ControllerSettings.SampleSetInitiator);
+      break;
     case CONTROLLER_ENABLED:
       addFormCheckBox(displayName, internalName, Settings.ControllerEnabled[controllerindex]);
       break;
@@ -247,6 +251,9 @@ void saveControllerParameterForm(ControllerSettingsStruct& ControllerSettings, b
     case CONTROLLER_TIMEOUT:
       ControllerSettings.ClientTimeout = getFormItemInt(internalName, ControllerSettings.ClientTimeout);
       break;
+    case CONTROLLER_SAMPLE_SET_INITIATOR:
+      ControllerSettings.SampleSetInitiator = getFormItemInt(internalName, ControllerSettings.SampleSetInitiator);
+      break;
     case CONTROLLER_ENABLED:
       Settings.ControllerEnabled[controllerindex] = isFormItemChecked(internalName);
       break;