From fdaa8279b85f2f6fc10ded70a8eb94095f7580f0 Mon Sep 17 00:00:00 2001 From: ladyada Date: Tue, 15 Sep 2026 15:41:15 -0400 Subject: [PATCH] Run every regression test in CI Discover the C++ tests and Arduino regression sketches automatically. Run the sketches against the actual library with host serial output and timing, including marine extensions, and propagate failures and timeouts to CI. Co-authored-by: Limor Fried --- .github/workflows/githubci.yml | 19 +-- extras/test_support/Arduino.cpp | 26 ++++ extras/test_support/Arduino.h | 118 ++++++++++++++++++ extras/test_support/SPI.h | 25 ++++ extras/test_support/Wire.h | 28 +++++ extras/test_support/sketch_main.cpp | 10 ++ extras/tests/README.md | 24 ++++ .../tests/marine_sources/marine_sources.ino | 2 + extras/tests/parse_str/parse_str.ino | 2 + extras/tests/run_tests.py | 54 ++++++++ 10 files changed, 293 insertions(+), 15 deletions(-) create mode 100644 extras/test_support/Arduino.cpp create mode 100644 extras/test_support/Arduino.h create mode 100644 extras/test_support/SPI.h create mode 100644 extras/test_support/Wire.h create mode 100644 extras/test_support/sketch_main.cpp create mode 100644 extras/tests/README.md create mode 100644 extras/tests/run_tests.py diff --git a/.github/workflows/githubci.yml b/.github/workflows/githubci.yml index a755ca3..8c2081f 100644 --- a/.github/workflows/githubci.yml +++ b/.github/workflows/githubci.yml @@ -3,25 +3,14 @@ name: Arduino Library CI on: [pull_request, push, repository_dispatch] jobs: - nmea-tests: - name: NMEA regression tests + regression-tests: + name: All regression tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Run host tests with sanitizers - shell: bash - run: | - set -euo pipefail - mkdir -p "$RUNNER_TEMP/nmea-tests" - for test in nmea_validate nmea_fields nmea_utilities; do - echo "Running $test" - g++ -std=c++11 -Wall -Wextra -Werror -g \ - -fsanitize=address,undefined -fno-sanitize-recover=all \ - -fno-omit-frame-pointer -Isrc src/Adafruit_NMEA.cpp \ - "extras/tests/$test/$test.cpp" -o "$RUNNER_TEMP/nmea-tests/$test" - "$RUNNER_TEMP/nmea-tests/$test" - done + - name: Run all tests with sanitizers + run: python3 extras/tests/run_tests.py build: runs-on: ubuntu-latest diff --git a/extras/test_support/Arduino.cpp b/extras/test_support/Arduino.cpp new file mode 100644 index 0000000..5bcb9cb --- /dev/null +++ b/extras/test_support/Arduino.cpp @@ -0,0 +1,26 @@ +#include "Arduino.h" +#include +#include + +HardwareSerial Serial; + +uint32_t millis() { + static const auto start = std::chrono::steady_clock::now(); + return (uint32_t)std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); +} + +void delay(uint32_t milliseconds) { + std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds)); +} + +void yield() { std::this_thread::yield(); } + +void unexpectedHardware() { + fputs("FAIL: hardware access is not supported by host tests\n", stderr); + abort(); +} + +void pinMode(int, int) { unexpectedHardware(); } +void digitalWrite(int, int) { unexpectedHardware(); } diff --git a/extras/test_support/Arduino.h b/extras/test_support/Arduino.h new file mode 100644 index 0000000..759f545 --- /dev/null +++ b/extras/test_support/Arduino.h @@ -0,0 +1,118 @@ +// Minimal Arduino compatibility for host regressions, not a hardware emulator. +#ifndef GPS_TEST_ARDUINO_H +#define GPS_TEST_ARDUINO_H + +#include +#include +#include +#include +#include +#include +#include +#include + +using std::max; +using std::min; +typedef uint8_t byte; + +#define PROGMEM +#define PSTR(text) (text) +#define strcmp_P strcmp +#define strncmp_P strncmp +#define strcpy_P strcpy +#define strlen_P strlen +#define pgm_read_byte(address) (*(const uint8_t *)(address)) +#define RAD_TO_DEG 57.295779513082320876798154814105 +#define OUTPUT 1 +#define HIGH 1 +#define LOW 0 + +class __FlashStringHelper; +#define F(text) ((const __FlashStringHelper *)(text)) + +uint32_t millis(); +void delay(uint32_t milliseconds); +void yield(); +void pinMode(int pin, int mode); +void digitalWrite(int pin, int value); +void unexpectedHardware(); + +inline bool isDigit(int c) { return isdigit((unsigned char)c); } +inline bool isAlpha(int c) { return isalpha((unsigned char)c); } + +class Print { +public: + virtual ~Print() {} + virtual size_t write(uint8_t c) = 0; + size_t print(const char *text) { + size_t count = 0; + while (*text) { + count += write(*text++); + } + return count; + } + size_t print(const __FlashStringHelper *text) { + return print((const char *)text); + } + size_t print(char value) { return write(value); } + size_t print(int value) { return print((long)value); } + size_t print(unsigned value) { return print((unsigned long)value); } + size_t print(long value) { + char text[32]; + snprintf(text, sizeof(text), "%ld", value); + return print(text); + } + size_t print(unsigned long value) { + char text[32]; + snprintf(text, sizeof(text), "%lu", value); + return print(text); + } + size_t print(double value, int digits = 2) { + char text[384]; + snprintf(text, sizeof(text), "%.*f", digits, value); + return print(text); + } + size_t println() { return print("\r\n"); } + template size_t println(T value) { + return print(value) + println(); + } + size_t println(double value, int digits) { + return print(value, digits) + println(); + } + size_t println(const char *text) { return print(text) + print("\r\n"); } + size_t println(const __FlashStringHelper *text) { + return println((const char *)text); + } +}; + +class Stream : public Print { +public: + virtual int available() = 0; + virtual int read() = 0; + virtual int peek() = 0; + virtual void flush() = 0; +}; + +class HardwareSerial : public Stream { +public: + void begin(uint32_t) {} + operator bool() const { return true; } + int available() { + unexpectedHardware(); + return 0; + } + int read() { + unexpectedHardware(); + return -1; + } + int peek() { + unexpectedHardware(); + return -1; + } + void flush() { fflush(stdout); } + size_t write(uint8_t c) { return fputc(c, stdout) == EOF ? 0 : 1; } +}; + +extern HardwareSerial Serial; + +#endif diff --git a/extras/test_support/SPI.h b/extras/test_support/SPI.h new file mode 100644 index 0000000..32b411c --- /dev/null +++ b/extras/test_support/SPI.h @@ -0,0 +1,25 @@ +#ifndef GPS_TEST_SPI_H +#define GPS_TEST_SPI_H + +#include "Arduino.h" + +#define MSBFIRST 1 +#define SPI_MODE0 0 + +class SPISettings { +public: + SPISettings(uint32_t, uint8_t, uint8_t) {} +}; + +class SPIClass { +public: + void begin() { unexpectedHardware(); } + void beginTransaction(SPISettings) { unexpectedHardware(); } + void endTransaction() { unexpectedHardware(); } + uint8_t transfer(uint8_t) { + unexpectedHardware(); + return 0; + } +}; + +#endif diff --git a/extras/test_support/Wire.h b/extras/test_support/Wire.h new file mode 100644 index 0000000..e670559 --- /dev/null +++ b/extras/test_support/Wire.h @@ -0,0 +1,28 @@ +#ifndef GPS_TEST_WIRE_H +#define GPS_TEST_WIRE_H + +#include "Arduino.h" + +class TwoWire { +public: + void begin() { unexpectedHardware(); } + void beginTransmission(uint8_t) { unexpectedHardware(); } + uint8_t endTransmission(bool = true) { + unexpectedHardware(); + return 4; + } + size_t write(uint8_t) { + unexpectedHardware(); + return 0; + } + uint8_t requestFrom(uint8_t, uint8_t, bool) { + unexpectedHardware(); + return 0; + } + int read() { + unexpectedHardware(); + return -1; + } +}; + +#endif diff --git a/extras/test_support/sketch_main.cpp b/extras/test_support/sketch_main.cpp new file mode 100644 index 0000000..b52611e --- /dev/null +++ b/extras/test_support/sketch_main.cpp @@ -0,0 +1,10 @@ +// Run the same checks as the Arduino sketch and report failure to CI. +extern bool testsPassed; +void setup(); +void loop(); + +int main() { + setup(); + loop(); + return testsPassed ? 0 : 1; +} diff --git a/extras/tests/README.md b/extras/tests/README.md new file mode 100644 index 0000000..d905ad4 --- /dev/null +++ b/extras/tests/README.md @@ -0,0 +1,24 @@ +# Regression tests + +Run every test on Linux with Python 3 and g++: + +```sh +python3 extras/tests/run_tests.py +``` + +The runner discovers every `.cpp` and `.ino` in the test subdirectories. +Each C++ test supplies `main()`. Each Arduino sketch supplies `setup()`, +`loop()`, and a `testsPassed` flag that remains false until all checks pass. +The runner calls `setup()` and `loop()` once and returns failure when that +flag is false. New tests are picked up automatically without editing CI. + +All tests use AddressSanitizer and UndefinedBehaviorSanitizer. Compiler +warnings, failed checks, sanitizer findings, and a 30-second execution timeout +fail the job. Other tests still run after an individual failure. + +The Arduino sketches use the real library sources with NMEA extensions enabled, +including the marine and RMB checks. `extras/test_support` provides serial +output and a real monotonic clock; the wakeup timeout test takes ten seconds. +I2C, SPI, GPIO, and hardware serial input abort if used. This runs the parser +and mock-stream regressions, not physical GPS/SD hardware or MCU emulation. +The sketches remain usable on Arduino boards with the normal Arduino core. diff --git a/extras/tests/marine_sources/marine_sources.ino b/extras/tests/marine_sources/marine_sources.ino index f1fb048..bd4e0db 100644 --- a/extras/tests/marine_sources/marine_sources.ino +++ b/extras/tests/marine_sources/marine_sources.ino @@ -1,5 +1,7 @@ #include +bool parseSentence(char *sentence, const char *source, const char *type); + Adafruit_GPS GPS; bool testsPassed = false; diff --git a/extras/tests/parse_str/parse_str.ino b/extras/tests/parse_str/parse_str.ino index fd74f9b..eb841a9 100644 --- a/extras/tests/parse_str/parse_str.ino +++ b/extras/tests/parse_str/parse_str.ino @@ -8,6 +8,8 @@ #include #undef private +bool checkField(const char *value, uint8_t capacity, const char *expected); + Adafruit_GPS GPS; bool testsPassed = false; diff --git a/extras/tests/run_tests.py b/extras/tests/run_tests.py new file mode 100644 index 0000000..113a05b --- /dev/null +++ b/extras/tests/run_tests.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Build and run every regression under extras/tests with Linux g++.""" + +import os +from pathlib import Path +import subprocess +import tempfile + + +def main(): + tests_dir = Path(__file__).resolve().parent + root = tests_dir.parent.parent + support = root / "extras/test_support" + tests = sorted([*tests_dir.glob("*/*.cpp"), *tests_dir.glob("*/*.ino")]) + if not tests: + raise RuntimeError("No regression tests found") + + # Keep binaries outside the checkout, including when run locally. + build = Path(tempfile.mkdtemp(prefix="gps-tests-")) + compiler = os.environ.get("CXX", "g++") + flags = [ + "-std=c++11", "-Wall", "-Wextra", "-Werror", "-g", "-pthread", + "-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-omit-frame-pointer", "-I" + str(root / "src"), + ] + failures = [] + for index, test in enumerate(tests, 1): + name = str(test.relative_to(tests_dir)) + print(f"\n[{index}/{len(tests)}] {name}", flush=True) + binary = build / str(index) + if test.suffix == ".ino": + # Enable marine extensions so RMB and conditional marine checks run. + sources = [*sorted((root / "src").glob("*.cpp")), + support / "Arduino.cpp", support / "sketch_main.cpp"] + extra_flags = ["-I" + str(support), "-DNMEA_EXTRAS=1"] + else: + sources = [root / "src/Adafruit_NMEA.cpp"] + extra_flags = [] + try: + subprocess.run([compiler, *flags, *extra_flags, + *map(str, sources), "-x", "c++", str(test), + "-o", str(binary)], check=True, timeout=120) + subprocess.run([str(binary)], check=True, timeout=30) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + print(f"FAIL: {name}: {error}", flush=True) + failures.append(name) + + print(f"\n{len(tests) - len(failures)}/{len(tests)} regressions passed.", flush=True) + if failures: + raise SystemExit(1) + + +if __name__ == "__main__": + main()