Run every regression test in CI

Discover the C++ tests and Arduino regression sketches automatically. Run the sketches against the actual library with host serial output and timing, including marine extensions, and propagate failures and timeouts to CI.

Co-authored-by: Limor Fried <ladyada@users.noreply.github.com>

.github/workflows/githubci.yml

@@ -3,25 +3,14 @@ name: Arduino Library CI
 on: [pull_request, push, repository_dispatch]
 
 jobs:
-  nmea-tests:
-    name: NMEA regression tests
+  regression-tests:
+    name: All regression tests
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
-    - name: Run host tests with sanitizers
-      shell: bash
-      run: |
-        set -euo pipefail
-        mkdir -p "$RUNNER_TEMP/nmea-tests"
-        for test in nmea_validate nmea_fields nmea_utilities; do
-          echo "Running $test"
-          g++ -std=c++11 -Wall -Wextra -Werror -g \
-            -fsanitize=address,undefined -fno-sanitize-recover=all \
-            -fno-omit-frame-pointer -Isrc src/Adafruit_NMEA.cpp \
-            "extras/tests/$test/$test.cpp" -o "$RUNNER_TEMP/nmea-tests/$test"
-          "$RUNNER_TEMP/nmea-tests/$test"
-        done
+    - name: Run all tests with sanitizers
+      run: python3 extras/tests/run_tests.py
 
   build:
     runs-on: ubuntu-latest

extras/test_support/Arduino.cpp

@@ -0,0 +1,26 @@
+#include "Arduino.h"
+#include <chrono>
+#include <thread>
+
+HardwareSerial Serial;
+
+uint32_t millis() {
+  static const auto start = std::chrono::steady_clock::now();
+  return (uint32_t)std::chrono::duration_cast<std::chrono::milliseconds>(
+             std::chrono::steady_clock::now() - start)
+      .count();
+}
+
+void delay(uint32_t milliseconds) {
+  std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
+}
+
+void yield() { std::this_thread::yield(); }
+
+void unexpectedHardware() {
+  fputs("FAIL: hardware access is not supported by host tests\n", stderr);
+  abort();
+}
+
+void pinMode(int, int) { unexpectedHardware(); }
+void digitalWrite(int, int) { unexpectedHardware(); }

extras/test_support/Arduino.h

@@ -0,0 +1,118 @@
+// Minimal Arduino compatibility for host regressions, not a hardware emulator.
+#ifndef GPS_TEST_ARDUINO_H
+#define GPS_TEST_ARDUINO_H
+
+#include <algorithm>
+#include <ctype.h>
+#include <math.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+using std::max;
+using std::min;
+typedef uint8_t byte;
+
+#define PROGMEM
+#define PSTR(text) (text)
+#define strcmp_P strcmp
+#define strncmp_P strncmp
+#define strcpy_P strcpy
+#define strlen_P strlen
+#define pgm_read_byte(address) (*(const uint8_t *)(address))
+#define RAD_TO_DEG 57.295779513082320876798154814105
+#define OUTPUT 1
+#define HIGH 1
+#define LOW 0
+
+class __FlashStringHelper;
+#define F(text) ((const __FlashStringHelper *)(text))
+
+uint32_t millis();
+void delay(uint32_t milliseconds);
+void yield();
+void pinMode(int pin, int mode);
+void digitalWrite(int pin, int value);
+void unexpectedHardware();
+
+inline bool isDigit(int c) { return isdigit((unsigned char)c); }
+inline bool isAlpha(int c) { return isalpha((unsigned char)c); }
+
+class Print {
+public:
+  virtual ~Print() {}
+  virtual size_t write(uint8_t c) = 0;
+  size_t print(const char *text) {
+    size_t count = 0;
+    while (*text) {
+      count += write(*text++);
+    }
+    return count;
+  }
+  size_t print(const __FlashStringHelper *text) {
+    return print((const char *)text);
+  }
+  size_t print(char value) { return write(value); }
+  size_t print(int value) { return print((long)value); }
+  size_t print(unsigned value) { return print((unsigned long)value); }
+  size_t print(long value) {
+    char text[32];
+    snprintf(text, sizeof(text), "%ld", value);
+    return print(text);
+  }
+  size_t print(unsigned long value) {
+    char text[32];
+    snprintf(text, sizeof(text), "%lu", value);
+    return print(text);
+  }
+  size_t print(double value, int digits = 2) {
+    char text[384];
+    snprintf(text, sizeof(text), "%.*f", digits, value);
+    return print(text);
+  }
+  size_t println() { return print("\r\n"); }
+  template <typename T> size_t println(T value) {
+    return print(value) + println();
+  }
+  size_t println(double value, int digits) {
+    return print(value, digits) + println();
+  }
+  size_t println(const char *text) { return print(text) + print("\r\n"); }
+  size_t println(const __FlashStringHelper *text) {
+    return println((const char *)text);
+  }
+};
+
+class Stream : public Print {
+public:
+  virtual int available() = 0;
+  virtual int read() = 0;
+  virtual int peek() = 0;
+  virtual void flush() = 0;
+};
+
+class HardwareSerial : public Stream {
+public:
+  void begin(uint32_t) {}
+  operator bool() const { return true; }
+  int available() {
+    unexpectedHardware();
+    return 0;
+  }
+  int read() {
+    unexpectedHardware();
+    return -1;
+  }
+  int peek() {
+    unexpectedHardware();
+    return -1;
+  }
+  void flush() { fflush(stdout); }
+  size_t write(uint8_t c) { return fputc(c, stdout) == EOF ? 0 : 1; }
+};
+
+extern HardwareSerial Serial;
+
+#endif

extras/test_support/SPI.h

@@ -0,0 +1,25 @@
+#ifndef GPS_TEST_SPI_H
+#define GPS_TEST_SPI_H
+
+#include "Arduino.h"
+
+#define MSBFIRST 1
+#define SPI_MODE0 0
+
+class SPISettings {
+public:
+  SPISettings(uint32_t, uint8_t, uint8_t) {}
+};
+
+class SPIClass {
+public:
+  void begin() { unexpectedHardware(); }
+  void beginTransaction(SPISettings) { unexpectedHardware(); }
+  void endTransaction() { unexpectedHardware(); }
+  uint8_t transfer(uint8_t) {
+    unexpectedHardware();
+    return 0;
+  }
+};
+
+#endif

extras/test_support/Wire.h

@@ -0,0 +1,28 @@
+#ifndef GPS_TEST_WIRE_H
+#define GPS_TEST_WIRE_H
+
+#include "Arduino.h"
+
+class TwoWire {
+public:
+  void begin() { unexpectedHardware(); }
+  void beginTransmission(uint8_t) { unexpectedHardware(); }
+  uint8_t endTransmission(bool = true) {
+    unexpectedHardware();
+    return 4;
+  }
+  size_t write(uint8_t) {
+    unexpectedHardware();
+    return 0;
+  }
+  uint8_t requestFrom(uint8_t, uint8_t, bool) {
+    unexpectedHardware();
+    return 0;
+  }
+  int read() {
+    unexpectedHardware();
+    return -1;
+  }
+};
+
+#endif

extras/test_support/sketch_main.cpp

@@ -0,0 +1,10 @@
+// Run the same checks as the Arduino sketch and report failure to CI.
+extern bool testsPassed;
+void setup();
+void loop();
+
+int main() {
+  setup();
+  loop();
+  return testsPassed ? 0 : 1;
+}

extras/tests/README.md

@@ -0,0 +1,24 @@
+# Regression tests
+
+Run every test on Linux with Python 3 and g++:
+
+```sh
+python3 extras/tests/run_tests.py
+```
+
+The runner discovers every `.cpp` and `.ino` in the test subdirectories.
+Each C++ test supplies `main()`. Each Arduino sketch supplies `setup()`,
+`loop()`, and a `testsPassed` flag that remains false until all checks pass.
+The runner calls `setup()` and `loop()` once and returns failure when that
+flag is false. New tests are picked up automatically without editing CI.
+
+All tests use AddressSanitizer and UndefinedBehaviorSanitizer. Compiler
+warnings, failed checks, sanitizer findings, and a 30-second execution timeout
+fail the job. Other tests still run after an individual failure.
+
+The Arduino sketches use the real library sources with NMEA extensions enabled,
+including the marine and RMB checks. `extras/test_support` provides serial
+output and a real monotonic clock; the wakeup timeout test takes ten seconds.
+I2C, SPI, GPIO, and hardware serial input abort if used. This runs the parser
+and mock-stream regressions, not physical GPS/SD hardware or MCU emulation.
+The sketches remain usable on Arduino boards with the normal Arduino core.

extras/tests/marine_sources/marine_sources.ino

@@ -1,5 +1,7 @@
 #include <Adafruit_GPS.h>
 
+bool parseSentence(char *sentence, const char *source, const char *type);
+
 Adafruit_GPS GPS;
 bool testsPassed = false;
 

extras/tests/parse_str/parse_str.ino

@@ -8,6 +8,8 @@
 #include <Adafruit_GPS.h>
 #undef private
 
+bool checkField(const char *value, uint8_t capacity, const char *expected);
+
 Adafruit_GPS GPS;
 bool testsPassed = false;
 

extras/tests/run_tests.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Build and run every regression under extras/tests with Linux g++."""
+
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+
+
+def main():
+    tests_dir = Path(__file__).resolve().parent
+    root = tests_dir.parent.parent
+    support = root / "extras/test_support"
+    tests = sorted([*tests_dir.glob("*/*.cpp"), *tests_dir.glob("*/*.ino")])
+    if not tests:
+        raise RuntimeError("No regression tests found")
+
+    # Keep binaries outside the checkout, including when run locally.
+    build = Path(tempfile.mkdtemp(prefix="gps-tests-"))
+    compiler = os.environ.get("CXX", "g++")
+    flags = [
+        "-std=c++11", "-Wall", "-Wextra", "-Werror", "-g", "-pthread",
+        "-fsanitize=address,undefined", "-fno-sanitize-recover=all",
+        "-fno-omit-frame-pointer", "-I" + str(root / "src"),
+    ]
+    failures = []
+    for index, test in enumerate(tests, 1):
+        name = str(test.relative_to(tests_dir))
+        print(f"\n[{index}/{len(tests)}] {name}", flush=True)
+        binary = build / str(index)
+        if test.suffix == ".ino":
+            # Enable marine extensions so RMB and conditional marine checks run.
+            sources = [*sorted((root / "src").glob("*.cpp")),
+                       support / "Arduino.cpp", support / "sketch_main.cpp"]
+            extra_flags = ["-I" + str(support), "-DNMEA_EXTRAS=1"]
+        else:
+            sources = [root / "src/Adafruit_NMEA.cpp"]
+            extra_flags = []
+        try:
+            subprocess.run([compiler, *flags, *extra_flags,
+                            *map(str, sources), "-x", "c++", str(test),
+                            "-o", str(binary)], check=True, timeout=120)
+            subprocess.run([str(binary)], check=True, timeout=30)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
+            print(f"FAIL: {name}: {error}", flush=True)
+            failures.append(name)
+
+    print(f"\n{len(tests) - len(failures)}/{len(tests)} regressions passed.", flush=True)
+    if failures:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()