Berry transpose C defines to Berry in tasmota_defines_for_berry.be (#24680)

.gitignore

@@ -27,6 +27,8 @@ unpacked_fs
 unpacked_boards
 tasmota/user/*
 tasmota/user_config_override.h
+tasmota/tasmota_defines_for_berry.h
+tasmota/tasmota_defines_for_berry.be
 tasmota/include/local_ca_data.h
 tasmota/include/local_ca_descriptor.h
 variants

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ## [15.4.0.1]
 ### Added
 - Berry add support for pre-processor
+- Berry transpose C defines to Berry in `tasmota_defines_for_berry.be`
 
 ### Breaking Changed
 

lib/libesp32/berry/TASMOTA_DEFINES_FOR_BERRY.md

@@ -0,0 +1,106 @@
+# Tasmota defines for Berry solidification
+
+## Why this exists
+
+Berry solidification compiles `.be` scripts into pre-built C structures that are
+embedded directly in the firmware. The solidifier runs at **build time** on the
+host machine (via `berry_port`, a Python re-implementation of the Berry VM).
+
+For solidified code to be correct it must know which Tasmota features are
+compiled into the firmware. For example, a solidified class that references
+`USE_MATTER_DEVICE` or a `D_JSON_*` string constant must see the same values
+that the C++ compiler sees when it builds the firmware. Without this, the
+solidified bytecode can embed wrong constants or include dead code paths.
+
+## How the dump is produced
+
+`pio-tools/dump-defines.py` runs as a **pre-script** in the PlatformIO build
+(registered in `platformio_tasmota32.ini`, before `gen-berry-structures.py`).
+It invokes the host C preprocessor in macro-dump mode:
+
+```
+xtensa-esp32-elf-g++ -E -dM -x c++
+   -DESP32 <build_flags -D entries>
+   -I include/ -I tasmota/include/ -I tasmota/
+   -include tasmota/include/tasmota.h
+   -include tasmota/my_user_config.h
+   -include include/tasmota_options.h
+   -include tasmota/include/i18n.h
+   - < /dev/null
+   > tasmota/tasmota_defines_for_berry.h
+```
+
+Key design decisions:
+
+| Decision | Reason |
+|---|---|
+| Post-script (`post:`) | `$CXX` is set to the real cross-compiler after the platform builder runs. No `-m32` hack needed. |
+| Cross-compiler (`$CXX`) | Correct int/pointer widths, endianness, and target-specific built-in macros for ESP32. |
+| `-E -dM` | Preprocessor-only mode; outputs every `#define` visible after expanding all headers. |
+| Only config/i18n headers, not `tasmota.ino` | `tasmota.ino` pulls in Arduino/framework library headers (`EEPROM.h`, `WiFiHelper.h`, …) that aren't resolvable before PlatformIO's LDF runs. Those libraries don't contribute macros the solidifier needs. |
+| Empty `sdkconfig.h` stub | `tasmota_configurations_ESP32.h` includes `sdkconfig.h` which is generated per-MCU at build time. Its `CONFIG_*` values aren't needed for Berry solidification, so an empty stub is supplied. |
+| `-D` flags from `BUILD_FLAGS` | Ensures `USE_CONFIG_OVERRIDE`, `MY_LANGUAGE`, firmware-variant flags, etc. are defined exactly as in the real build. |
+
+The output is written to `tasmota/tasmota_defines_for_berry.h` which is
+**gitignored** — it is a build artefact, regenerated on every build.
+
+## What the output looks like
+
+The file is a flat list of `#define` lines, one per macro, in the format
+produced by `gcc -dM`. Approximately **3 200 lines** for a typical
+`tasmota32` build, split roughly as:
+
+| Category | Count | Example |
+|---|---|---|
+| `USE_*` feature flags | ~220 | `#define USE_BERRY ` |
+| `D_*` strings (JSON keys, commands, sensor names) | ~2 000 | `#define D_JSON_TEMPERATURE "Temperature"` |
+| Firmware metadata | a few | `#define CODE_IMAGE_STR "tasmota32"` |
+| Language / locale | a few | `#define LANGUAGE_LCID 2057` |
+| Compiler built-ins + misc | remainder | `#define __INT_MAX__ 2147483647` |
+
+A representative sample:
+
+```c
+/* Feature flags */
+#define USE_BERRY
+#define USE_MATTER_DEVICE
+#define USE_RULES
+#define USE_TLS
+#define USE_WEBSERVER
+#define USE_IPV6 1
+#define USE_ZIGBEE_ZNP
+
+/* Language / locale */
+#define LANGUAGE_LCID 2057          /* en_GB */
+#define CODE_IMAGE_STR "tasmota32"
+
+/* JSON key strings */
+#define D_JSON_TEMPERATURE "Temperature"
+#define D_JSON_TEMPERATURE_UNIT "TempUnit"
+
+/* Command strings */
+#define D_CMND_STATUS "Status"
+
+/* Sensor name strings */
+#define D_SENSOR_SWITCH "Switch"
+```
+
+Flags that are **not** defined simply do not appear in the file. For example,
+if `USE_ZIGBEE` (the master gate) is commented out in `my_user_config.h`, there
+will be no `#define USE_ZIGBEE` line — only the unconditional sub-configuration
+constants (`USE_ZIGBEE_ZNP`, `USE_ZIGBEE_CHANNEL`, …) that are declared as
+defaults regardless of the master gate.
+
+## How the solidifier should consume this file
+
+The intended use is to parse it
+as a flat text file inside the Berry solidification toolchain so that:
+
+1. **Conditional solidification** — a `.be` class or module is only solidified
+   when the corresponding `USE_*` flag is present.
+2. **Constant folding** — `D_JSON_*` and `D_CMND_*` string constants can be
+   embedded directly into solidified bytecode rather than looked up at runtime.
+
+The file is regenerated at the start of every esp32 build, so it always
+reflects the current `user_config_override.h` and `platformio_override.ini`
+settings.

lib/libesp32/berry_animation/solidify_all_python.be

@@ -13,6 +13,8 @@ import sys
 sys.path().push('src')   # allow to import from src/embedded
 sys.path().push('src/core')   # allow to import from src/embedded
 
+import "../../../tasmota/tasmota_defines_for_berry.be" as tasmota_defines
+
 # globals that need to exist to make compilation succeed
 var globs = "path,ctypes_bytes_dyn,tasmota,ccronexpr,gpio,light,webclient,load,MD5,lv,light_state,udp,tcpclientasync,log,"
 

lib/libesp32/berry_custom/solidify_all_python.be

@@ -0,0 +1,106 @@
+#!/usr/bin/env -S PYTHONPATH=../berry python3 -m berry_port -s -g
+#
+# Berry solidify files
+
+import os
+import global
+import solidify
+import string as string2
+import re
+
+import sys
+sys.path().push('src/embedded')   # allow to import from src/embedded
+
+import "../../../tasmota/tasmota_defines_for_berry.be" as tasmota_defines
+
+# globals that need to exist to make compilation succeed
+var globs = "path,ctypes_bytes_dyn,tasmota,ccronexpr,gpio,light,webclient,load,MD5,lv,light_state,udp,tcpclientasync,"
+            "lv_clock,lv_clock_icon,lv_signal_arcs,lv_signal_bars,lv_wifi_arcs_icon,lv_wifi_arcs,"
+            "lv_wifi_bars_icon,lv_wifi_bars,"
+            "_lvgl,"
+            "int64"
+
+for g:string2.split(globs, ",")
+  global.(g) = nil
+end
+
+var prefix_dir = "src/embedded/"
+var prefix_out = "src/solidify/"
+
+def sort(l)
+  # insertion sort
+  for i:1..size(l)-1
+    var k = l[i]
+    var j = i
+    while (j > 0) && (l[j-1] > k)
+      l[j] = l[j-1]
+      j -= 1
+    end
+    l[j] = k
+  end
+  return l
+end
+
+def clean_directory(dir)
+  var file_list = os.listdir(dir)
+  for f : file_list
+    if f[0] == '.'  continue end    # ignore files starting with `.`
+    os.remove(dir + f)
+  end
+end
+
+var pattern = "#@\\s*solidify:([A-Za-z0-9_.,]+)"
+
+def parse_file(fname, prefix_out)
+  print("Parsing: ", fname)
+  var f = open(prefix_dir + fname)
+  var src = f.read()
+  f.close()
+  # try to compile
+  var compiled = compile(src)
+  compiled()      # run the compile code to instanciate the classes and modules
+  # output solidified
+  var fname_h = string2.split(fname, '.be')[0] + '.h'  # take whatever is before the first '.be'
+  var fout = open(prefix_out + "solidified_" + fname_h, "w")
+  fout.write(f"/* Solidification of {fname_h} */\n")
+  fout.write("/********************************************************************\\\n")
+  fout.write("* Generated code, don't edit                                         *\n")
+  fout.write("\\********************************************************************/\n")
+  fout.write('#include "be_constobj.h"\n')
+
+  var directives = re.searchall(pattern, src)
+  # print(directives)
+
+  for directive : directives
+    var object_list = string2.split(directive[1], ',')
+    var object_name = object_list[0]
+    var weak = (object_list.find('weak') != nil)          # do we solidify with weak strings?
+    var o = global
+    var cl_name = nil
+    var obj_name = nil
+    for subname : string2.split(object_name, '.')
+      o = o.(subname)
+      cl_name = obj_name
+      obj_name = subname
+      if   (type(o) == 'class')
+        obj_name = 'class_' + obj_name
+      elif (type(o) == 'module')
+        obj_name = 'module_' + obj_name
+      end
+    end
+    solidify.dump(o, weak, fout, cl_name)
+  end
+
+  fout.write("/********************************************************************/\n")
+  fout.write("/* End of solidification */\n")
+  fout.close()
+end
+
+clean_directory(prefix_out)
+
+var src_file_list = os.listdir(prefix_dir)
+src_file_list = sort(src_file_list)
+for src_file : src_file_list
+  if src_file[0] == '.'  continue end
+  parse_file(src_file, prefix_out)
+end

lib/libesp32/berry_matter/solidify_all_python.be

@@ -11,6 +11,8 @@ import re
 import sys
 sys.path().push('src/embedded')   # allow to import from src/embedded
 
+import "../../../tasmota/tasmota_defines_for_berry.be" as tasmota_defines
+
 # globals that need to exist to make compilation succeed
 var globs = "path,ctypes_bytes_dyn,tasmota,ccronexpr,gpio,light,webclient,load,MD5,lv,light_state,udp,tcpclientasync,log,"
             "lv_clock,lv_clock_icon,lv_signal_arcs,lv_signal_bars,lv_wifi_arcs_icon,lv_wifi_arcs,"

lib/libesp32/berry_tasmota/solidify_all_python.be

@@ -12,6 +12,8 @@ import introspect
 import sys
 sys.path().push('src/embedded')   # allow to import from src/embedded
 
+import "../../../tasmota/tasmota_defines_for_berry.be" as tasmota_defines
+
 # globals that need to exist to make compilation succeed
 var globs = "path,ctypes_bytes_dyn,tasmota,ccronexpr,gpio,light,webclient,load,MD5,lv,light_state,udp,tcpserver,log,sortedmap,"
             "lv_clock,lv_clock_icon,lv_signal_arcs,lv_signal_bars,lv_wifi_arcs_icon,lv_wifi_arcs,"

lib/libesp32_lvgl/lv_binding_berry/solidify_all_python.be

@@ -11,6 +11,8 @@ import re
 # import sys
 # sys.path().push('src/embedded')   # allow to import from src/embedded
 
+import "../../../tasmota/tasmota_defines_for_berry.be" as tasmota_defines
+
 # globals that need to exist to make compilation succeed
 var globs = "path,ctypes_bytes_dyn,tasmota,ccronexpr,gpio,light,webclient,load,MD5,lv,light_state,"
             "lv_clock,lv_clock_icon,lv_signal_arcs,lv_signal_bars,lv_wifi_arcs_icon,lv_wifi_arcs,"

lib/libesp32_lvgl/lv_haspmota/solidify_all_python.be

@@ -11,6 +11,8 @@ import re
 import sys
 sys.path().push('src/embedded')   # allow to import from src/embedded
 
+import "../../../tasmota/tasmota_defines_for_berry.be" as tasmota_defines
+
 # globals that need to exist to make compilation succeed
 var globs = "path,ctypes_bytes_dyn,tasmota,ccronexpr,gpio,light,webclient,load,MD5,lv,light_state,udp,tcpclientasync,log,"
             "lv_clock,lv_clock_icon,lv_signal_arcs,lv_signal_bars,lv_wifi_arcs_icon,lv_wifi_arcs,"

pio-tools/dump-defines.py

@@ -0,0 +1,127 @@
+# Dump preprocessor `#define` macros that matter for Berry solidification:
+#   - USE_* feature flags (from my_user_config.h, user_config_override.h,
+#                           tasmota_options.h, tasmota_configurations.h)
+#   - Language strings D_JSON_*, D_CMND_*, ... (from i18n.h + the language
+#                                                file selected via MY_LANGUAGE)
+#   - Tasmota enums (from tasmota.h)
+#
+# Output: tasmota/tasmota_defines_for_berry.h (gitignored)
+#
+# Runs as a POST-script so $CXX is already set to the real cross-compiler
+# (xtensa-esp32-elf-g++, riscv32-esp-elf-g++, etc.) by the time this
+# script executes. This guarantees correct int/pointer widths and
+# target-specific built-in macros without any -m32 hack.
+#
+# We pass `-D` flags from the project's build_flags so that USE_* gates,
+# MY_LANGUAGE, ESP32, etc. are defined the same way the real build sees
+# them.
+
+Import("env")
+
+import pathlib
+import re
+import subprocess
+
+from colorama import Fore
+
+
+_IS_ESP32 = env["PIOPLATFORM"] == "espressif32"
+
+
+def _run_dump():
+    project_dir = pathlib.Path(env.subst("$PROJECT_DIR"))
+
+    output_file = project_dir / "tasmota" / "tasmota_defines_for_berry.h"
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Working dir for the sdkconfig.h stub.
+    stub_parent = project_dir / "build_output"
+    stub_parent.mkdir(parents=True, exist_ok=True)
+
+    cxx = env.subst("$CXX")
+    if not cxx:
+        print(Fore.YELLOW + "dump-defines: $CXX not set, skipping")
+        return
+
+    forced_includes = [
+        project_dir / "tasmota" / "include" / "tasmota.h",
+        project_dir / "tasmota" / "my_user_config.h",
+        project_dir / "include" / "tasmota_options.h",
+        project_dir / "tasmota" / "include" / "i18n.h",
+    ]
+
+    # Extract -D flags from build_flags. Other flags (-I, -W, -f, ...) we
+    # don't need: our forced includes are absolute paths and we supply
+    # explicit -I for the Tasmota header roots below.
+    build_flags = env.subst(" ".join(env.get("BUILD_FLAGS") or []))
+    define_flags = re.findall(r"-D\s*\S+", build_flags)
+
+    # Also surface ESP32 unconditionally (real build always defines it via
+    # the platform) so #ifdef ESP32 in tasmota_configurations.h fires.
+    if not any(f.startswith("-DESP32") for f in define_flags):
+        define_flags.append("-DESP32")
+
+    include_flags = [
+        "-I" + str(project_dir / "include"),
+        "-I" + str(project_dir / "tasmota" / "include"),
+        "-I" + str(project_dir / "tasmota"),
+    ]
+
+    # tasmota_configurations_ESP32.h and the ESP-IDF headers pull in
+    # `sdkconfig.h`, which is generated per MCU variant by the framework
+    # and not present at pre-script time. For Berry solidification we
+    # don't need its CONFIG_* values - the macros the solidifier cares
+    # about live in Tasmota headers. Supply an empty stub so the
+    # preprocessor stops complaining.
+    stub_dir = stub_parent / "dump-defines-stubs"
+    stub_dir.mkdir(parents=True, exist_ok=True)
+    (stub_dir / "sdkconfig.h").write_text(
+        "/* auto-generated empty stub for dump-defines.py */\n"
+    )
+    # Prepend so the stub shadows any real header on CPPPATH.
+    include_flags.insert(0, "-I" + str(stub_dir))
+
+    cmd = [cxx, "-E", "-dM", "-x", "c++"]
+    cmd += define_flags
+    cmd += include_flags
+    for header in forced_includes:
+        cmd += ["-include", str(header)]
+    cmd.append("-")  # read empty stdin
+
+    print(f"Berry dumping defines: {output_file.relative_to(project_dir)}")
+
+    try:
+        with open(output_file, "w", encoding="utf-8") as out:
+            proc = subprocess.run(
+                cmd,
+                input="",
+                stdout=out,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=False,
+            )
+    except FileNotFoundError as e:
+        print(Fore.YELLOW + f"dump-defines: {cxx} not runnable ({e}), skipping")
+        return
+
+    if proc.returncode != 0:
+        print(Fore.YELLOW + f"dump-defines: {cxx} returned {proc.returncode}")
+        if proc.stderr:
+            for line in proc.stderr.splitlines()[:10]:
+                print(Fore.YELLOW + f"  {line}")
+
+
+def _action(*args, **kwargs):
+    _run_dump()
+
+
+if _IS_ESP32:
+    _run_dump()
+
+    env.AddCustomTarget(
+        name="dump_defines",
+        dependencies=None,
+        actions=[_action],
+        title="Dump preprocessor defines",
+        description="Run the preprocessor over Tasmota config/i18n headers and write macros to tasmota/tasmota_defines_for_berry.h",
+    )

pio-tools/gen-berry-defines.py

@@ -0,0 +1,126 @@
+# Parse tasmota/tasmota_defines_for_berry.h and emit
+# tasmota/tasmota_defines_for_berry.be for Berry solidification.
+#
+# Runs as a POST-script, after dump-defines.py has produced the .h file
+# and before gen-berry-structures.py runs solidification.
+#
+# Supported conversions (everything else becomes a comment):
+#   #define NAME              → preproc.define('NAME')
+#   #define NAME true/false   → preproc.define('NAME', true/false)
+#   #define NAME "string"     → preproc.define('NAME', "string")
+#   #define NAME 123          → preproc.define('NAME', 123)
+#   #define NAME 0x1A         → preproc.define('NAME', 0x1A)
+#   #define NAME -42          → preproc.define('NAME', -42)
+#
+# Ignored (kept as comments):
+#   - Names starting with '__'
+#   - Values that are expressions / identifiers / floats / other types
+
+Import("env")
+
+import pathlib
+import re
+
+from colorama import Fore
+
+# ---------------------------------------------------------------------------
+# Regex patterns
+# ---------------------------------------------------------------------------
+
+# Full #define line: captures name and optional value (everything after name)
+_RE_DEFINE = re.compile(r"^#define\s+(\S+)(?:\s+(.*?))?\s*$")
+
+# Accepted value patterns
+_RE_STRING  = re.compile(r'^"(?:[^"\\]|\\.)*"$')          # "..."
+_RE_BOOL    = re.compile(r"^(true|false)$")                # true / false
+_RE_INT     = re.compile(r"^-?\d+$")                       # decimal integer
+_RE_HEX     = re.compile(r"^0[xX][0-9A-Fa-f]+$")          # hex 0x...
+
+
+def _classify(name: str, value: str | None):
+    """
+    Returns ('emit', berry_value_str) or ('ignore', reason).
+    berry_value_str is None for bare defines (no value).
+    """
+    # Skip compiler built-ins
+    if name.startswith("__"):
+        return ("ignore", f"#define {name}" + (f" {value}" if value else ""))
+
+    # No value → bare define
+    if value is None or value == "":
+        return ("emit", None)
+
+    # true / false
+    if _RE_BOOL.match(value):
+        return ("emit", value)
+
+    # Quoted string
+    if _RE_STRING.match(value):
+        return ("emit", value)
+
+    # Decimal integer
+    if _RE_INT.match(value):
+        return ("emit", value)
+
+    # Hex integer
+    if _RE_HEX.match(value):
+        return ("emit", value)
+
+    # Anything else (expressions, identifiers, floats, …) → comment
+    return ("ignore", f"#define {name} {value}")
+
+
+def _run():
+    project_dir = pathlib.Path(env.subst("$PROJECT_DIR"))
+    input_file  = project_dir / "tasmota" / "tasmota_defines_for_berry.h"
+    output_file = project_dir / "tasmota" / "tasmota_defines_for_berry.be"
+
+    if not input_file.exists():
+        print(
+            Fore.YELLOW
+            + f"gen-berry-defines: {input_file.relative_to(project_dir)} not found, skipping"
+        )
+        return
+
+    n_emitted = 0
+    n_ignored = 0
+
+    with open(input_file, encoding="utf-8") as fh_in, \
+         open(output_file, "w", encoding="utf-8") as fh_out:
+
+        fh_out.write("#  Generated code from tasmota_defines_for_berry.h, don't edit\n")
+        fh_out.write("import preproc\n")
+        fh_out.write("\n")
+
+        for raw in fh_in:
+            line = raw.rstrip("\r\n")
+            m = _RE_DEFINE.match(line)
+            if not m:
+                # Not a #define line at all — skip silently (blank lines, etc.)
+                continue
+
+            name  = m.group(1)
+            value = m.group(2)  # may be None
+
+            action, payload = _classify(name, value)
+
+            if action == "ignore":
+                fh_out.write(f"    # ignored {payload}\n")
+                n_ignored += 1
+            else:
+                # payload is the Berry value string, or None for bare define
+                if payload is None:
+                    fh_out.write(f"preproc.define('{name}')\n")
+                else:
+                    fh_out.write(f"preproc.define('{name}', {payload})\n")
+                n_emitted += 1
+
+        fh_out.write("\n")
+        fh_out.write(f"# {n_emitted} defines emitted, {n_ignored} ignored\n")
+
+    print(
+        f"Berry converting defines: {output_file.relative_to(project_dir)}"
+        f" ({n_emitted} defines, {n_ignored} ignored)"
+    )
+
+_run()

pio-tools/gen-berry-structures.py

@@ -33,6 +33,7 @@ else:
         join(PROJECT_DIR, "lib", "libesp32", "berry_tasmota"),
         join(PROJECT_DIR, "lib", "libesp32", "berry_matter"),
         join(PROJECT_DIR, "lib", "libesp32", "berry_animation"),
+        join(PROJECT_DIR, "lib", "libesp32", "berry_custom"),
         join(PROJECT_DIR, "lib", "libesp32_lvgl", "lv_binding_berry"),
         join(PROJECT_DIR, "lib", "libesp32_lvgl", "lv_haspmota"),
     ]
@@ -54,7 +55,7 @@ else:
         if not isfile(script):
             continue
         rel_script = os.path.relpath(script, PROJECT_DIR)
-        print(f"Solidifying: {rel_script}")
+        print(f"Berry solidification: {rel_script}")
         os.chdir(solidify_dir)
         solidify_cmd = (
             env["PYTHONEXE"],
@@ -108,6 +109,8 @@ for filePath in fileList:
         # print("Deleting file : ", filePath)
     except:
         print("Error while deleting file : ", filePath)
+
+print(f"Berry coc compiler")
 cmd = (env["PYTHONEXE"],join("tools","coc","coc"),"-o","generate","src","default",join("..","berry_tasmota","src"),join("..","berry_matter","src","solidify"),join("..","berry_matter","src"),join("..","berry_custom","src","solidify"),join("..","berry_custom","src"),join("..","berry_animation","src","solidify"),join("..","berry_animation","src"),join("..","berry_tasmota","src","solidify"),join("..","berry_mapping","src"),join("..","berry_int64","src"),join("..","..","libesp32_lvgl","lv_binding_berry","src"),join("..","..","libesp32_lvgl","lv_binding_berry","src","solidify"),join("..","..","libesp32_lvgl","lv_binding_berry","generate"),join("..","..","libesp32_lvgl","lv_haspmota","src","solidify"),"-c",join("default","berry_conf.h"))
 returncode = subprocess.call(cmd, shell=False)
 os.chdir(CURRENT_DIR)

platformio_tasmota32.ini

@@ -49,7 +49,9 @@ lib_ignore                  =
                               ArduinoOTA
 extra_scripts               = pre:pio-tools/add_c_flags.py
                               pre:pio-tools/solidify-from-url.py
-                              pre:pio-tools/gen-berry-structures.py
+                              post:pio-tools/dump-defines.py
+                              post:pio-tools/gen-berry-defines.py
+                              post:pio-tools/gen-berry-structures.py
                               post:pio-tools/post_esp32.py
                               ${esp_defaults.extra_scripts}
 monitor_filters             = esp32_exception_decoder