Michel2/Tasmota
1
0
Fork 0
mirror of https://github.com/arendst/Tasmota.git synced 2026-07-27 20:05:46 +00:00
Code Issues Releases Activity

Berry optimized solidified structures for code constants and maps (#24838)

Browse Source
This commit is contained in:
s-hadinger
2026-06-14 20:59:19 +02:00
committed by GitHub
parent b29f6926e1
commit bd466806fe
31 changed files with 1613 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.
- SCD4x library FrogmoreScd40 to Sensirion arduino-i2c-scd4x v1.1.0
- SPS30 library Sensirion arduino-i2c-sps30 v1.0.1
- Code hardening replacing `strcat` and `strcpy` with safer alternatives (#24832)
- Berry optimized solidified structures for code constants and maps
### Fixed
lib/libesp32/berry/berry_port/be_api.py
+16 -43
View File
@@ -1459,9 +1459,8 @@ def be_data_reverse(vm, index):
# var_setobj(iter, BE_COMPTR, NULL);
# return btrue;
# } else if (var_islist(v)) {
# blist *list = var_toobj(v);
# bvalue *iter = be_incrtop(vm);
# var_setobj(iter, BE_COMPTR, be_list_data(list) - 1);
# var_setint(iter, -1); /* index-based: start before first element */
# return btrue;
# }
# return bfalse;
@@ -1476,46 +1475,32 @@ def be_pushiter(vm, index):
elif var_islist(v):
_ensure_stack(vm, 1)
idx = be_incrtop(vm)
# Store iterator as integer index: -1 means "before first element"
var_setobj(vm.stack[idx], BE_COMPTR, -1)
# index-based: start before first element. Storing the index instead of
# a pointer means the iterator stays valid if the list is reallocated
# (e.g. mutated) during iteration.
var_setint(vm.stack[idx], -1)
return True
return False
# static int list_next(bvm *vm)
def _list_next(vm):
# static int list_next(bvm *vm, bvalue *v)
def _list_next(vm, v):
"""Advance list iterator and push the next value. Returns 1."""
iter_val = vm.stack[be_indexof(vm, -1)]
cur = var_toobj(iter_val) # current index
next_idx = cur + 1
var_setobj(iter_val, BE_COMPTR, next_idx)
# Get the list from the stack position before the iterator
# The list is at index -2 relative to the iterator push
# But we need to find it — the caller passes the list index
# Actually, list_next in C uses the raw pointer. In Python we store
# the index into list.data. We need the list object.
# The list is at the index passed to be_iter_next, which is before the iter.
# We'll get it from the calling context. For now, push the value.
idx = var_toint(iter_val) + 1
lst = var_toobj(v)
var_setint(iter_val, idx)
_ensure_stack(vm, 1)
dst_idx = be_incrtop(vm)
dst = vm.stack[dst_idx]
# We need the list — it's stored at the index the user passed to be_iter_next
# The convention is: list at `index`, iter at top-1
# We'll store the list ref in a helper. Actually, let's look at how C does it:
# list_next just uses the iter pointer to walk through list data.
# In Python, we store the index. We need the list to get data[next_idx].
# The list is passed via the `o` parameter in be_iter_next.
# We'll handle this in be_iter_next directly.
var_setnil(dst)
var_setval(vm.stack[dst_idx], lst.data[idx])
return 1
# static bbool list_hasnext(bvm *vm, bvalue *v)
def _list_hasnext(vm, v):
"""Check if list iterator has more elements."""
iter_val = vm.stack[be_indexof(vm, -1)]
lst = var_toobj(v)
cur = var_toobj(iter_val) # current index
next_idx = cur + 1
return 0 <= next_idx < lst.count
obj = var_toobj(v)
idx = var_toint(iter_val) + 1
return 0 <= idx < obj.count
# static int map_next(bvm *vm, bvalue *v)
def _map_next(vm, v):
@@ -1551,26 +1536,14 @@ def _map_hasnext(vm, v):
# BERRY_API int be_iter_next(bvm *vm, int index)
# {
# bvalue *o = be_indexof(vm, index);
# if (var_islist(o)) { return list_next(vm); }
# if (var_islist(o)) { return list_next(vm, o); }
# else if (var_ismap(o)) { return map_next(vm, o); }
# return 0;
# }
def be_iter_next(vm, index):
o = vm.stack[be_indexof(vm, index)]
if var_islist(o):
# Inline list_next with access to the list object
iter_val = vm.stack[be_indexof(vm, -1)]
cur = var_toobj(iter_val)
next_idx = cur + 1
var_setobj(iter_val, BE_COMPTR, next_idx)
lst = var_toobj(o)
_ensure_stack(vm, 1)
dst_idx = be_incrtop(vm)
if 0 <= next_idx < lst.count:
var_setval(vm.stack[dst_idx], lst.data[next_idx])
else:
var_setnil(vm.stack[dst_idx])
return 1
return _list_next(vm, o)
elif var_ismap(o):
return _map_next(vm, o)
return 0
lib/libesp32/berry/berry_port/be_listlib.py
+17 -19
View File
@@ -639,15 +639,15 @@ def m_clear(vm):
# * directly without using by the stack. */
# bntvclos *func = var_toobj(vm->cf->func);
# bvalue *uv0 = be_ntvclos_upval(func, 0)->value; /* list value */
# bvalue *uv1 = be_ntvclos_upval(func, 1)->value; /* iter value */
# bvalue *next = cast(bvalue*, var_toobj(uv1)) + 1;
# bvalue *uv1 = be_ntvclos_upval(func, 1)->value; /* iter value (index) */
# bint idx = var_toint(uv1) + 1;
# blist *list = var_toobj(uv0);
# if (next >= be_list_end(list)) {
# if (idx >= be_list_count(list)) {
# be_stop_iteration(vm);
# }
# var_toobj(uv1) = next; /* set upvale[1] (iter value) */
# var_setint(uv1, idx); /* set upvalue[1] (iter index) */
# /* push next value to top */
# var_setval(vm->top, next);
# var_setval(vm->top, be_list_at(list, idx));
# be_incrtop(vm);
# be_return(vm);
# }
@@ -658,24 +658,22 @@ def iter_closure(vm):
func = be_obj.var_toobj(vm.stack[vm.cf.func])
uv0 = be_func.be_ntvclos_upval(func, 0).value # list value
uv1 = be_func.be_ntvclos_upval(func, 1).value # iter value
uv1 = be_func.be_ntvclos_upval(func, 1).value # iter value (index)
# In C, uv1 holds a pointer into the list data array.
# In Python, uv1.v is an integer index into list.data.
# "next = cast(bvalue*, var_toobj(uv1)) + 1" means advance the iterator.
cur_idx = be_obj.var_toobj(uv1)
next_idx = cur_idx + 1
# uv1 holds an integer index into the list. Storing an index instead of a
# pointer keeps the iterator valid if the list is reallocated (mutated)
# during iteration.
idx = be_obj.var_toint(uv1) + 1
lst = be_obj.var_toobj(uv0) # blist object
if next_idx >= lst.count:
if idx >= lst.count:
be_api.be_stop_iteration(vm)
# Update the iterator upvalue to point to next_idx
uv1.v = next_idx
uv1.type = be_obj.BE_INDEX
# Update the iterator upvalue to the new index
be_obj.var_setint(uv1, idx)
# Push the value at next_idx to top of stack
val = lst.data[next_idx]
# Push the value at idx to top of stack
val = lst.data[idx]
be_obj.var_setval(vm.stack[vm.top_idx], val)
be_api.be_incrtop(vm)
return be_api.be_returnvalue(vm)
@@ -690,7 +688,7 @@ def iter_closure(vm):
# be_pushntvclosure(vm, iter_closure, 2);
# be_getmember(vm, 1, ".p");
# be_setupval(vm, -2, 0);
# be_pushiter(vm, -1);
# be_pushint(vm, -1); /* start index before first element */
# be_setupval(vm, -3, 1);
# be_pop(vm, 2);
# be_return(vm);
@@ -700,7 +698,7 @@ def m_iter(vm):
be_api.be_pushntvclosure(vm, iter_closure, 2)
be_api.be_getmember(vm, 1, ".p")
be_api.be_setupval(vm, -2, 0)
be_api.be_pushiter(vm, -1)
be_api.be_pushint(vm, -1) # start index before first element
be_api.be_setupval(vm, -3, 1)
be_api.be_pop(vm, 2)
return be_api.be_returnvalue(vm)
lib/libesp32/berry/berry_port/be_solidifylib.py
+305 -26
View File
@@ -46,7 +46,7 @@ from berry_port.be_decoder import (
OP_CLASS, OP_LDCONST, OP_RET,
OP_GETGBL, OP_SETGBL,
)
from berry_port.berry_conf import BE_INTGER_TYPE, BE_USE_SINGLE_FLOAT
from berry_port.berry_conf import BE_INTGER_TYPE, BE_USE_SINGLE_FLOAT, BE_USE_COMPACT_KTAB, BE_USE_COMPACT_MAP
# ============================================================================
@@ -297,6 +297,41 @@ def m_solidify_map(vm, str_literal, map_, prefix_name, fout):
be_map.be_map_compact(vm, map_)
_logfmt(fout, " be_nested_map(%i,\n", map_.count)
if BE_USE_COMPACT_MAP:
_logfmt(fout, " ( (struct bmapnode*) &(const bmapnodec[]) {\n")
for i in range(map_.size):
node = map_.slots[i]
if node.key.type == BE_NIL:
continue # key not used
key_next = node.key.next
if key_next == 0xFFFFFF:
key_next = -1 # more readable
if node.key.type == BE_STRING:
key = be_string.be_str2cstr(node.key.v)
id_buf = toidentifier(key)
if not str_literal:
_logfmt(fout, " { be_ckey(%s, %i), ", id_buf, key_next)
else:
_logfmt(fout, " { be_ckey_weak(%s, %i), ", id_buf, key_next)
m_solidify_map_value(vm, str_literal, node.value, prefix_name,
be_string.be_str2cstr(node.key.v), fout)
elif node.key.type == BE_INT:
_logfmt(fout, " { be_ckey_int(%i, %i), ",
node.key.v, key_next)
m_solidify_map_value(vm, str_literal, node.value, prefix_name,
None, fout)
else:
error = "Unsupported type in key: %i" % node.key.type
be_api.be_raise(vm, "internal_error", error)
_logfmt(fout, " },\n")
_lognofmt(fout, " }))")
return
_logfmt(fout, " ( (struct bmapnode*) &(const bmapnode[]) {\n")
for i in range(map_.size):
@@ -562,6 +597,218 @@ def _solidify_instance(vm, str_literal, value, prefix_name, key, fout):
_lognofmt(fout, " ) } ))")
# value type names for compact map nodes (incl. INSTANCE/MAP/LIST which never
# occur in a function ktab)
_MAP_VALTYPE_NAMES = {
BE_NIL: "BE_NIL",
BE_BOOL: "BE_BOOL",
BE_INT: "BE_INT",
BE_INDEX: "BE_INDEX",
BE_REAL: "BE_REAL",
BE_STRING: "BE_STRING",
BE_CLOSURE: "BE_CLOSURE",
BE_CLASS: "BE_CLASS",
BE_COMPTR: "BE_COMPTR",
BE_NTVFUNC: "BE_NTVFUNC",
BE_INSTANCE: "BE_INSTANCE",
BE_MAP: "BE_MAP",
BE_LIST: "BE_LIST",
}
def m_solidify_map_value(vm, str_literal, value, prefix_name, key, fout):
"""Emit one compact map node's value: the value type byte then the value
payload. Mirrors m_solidify_map_value in be_solidifylib.c."""
be_string = _lazy_be_string()
be_api = _lazy_be_api()
be_byteslib = _lazy_be_byteslib()
type_ = var_primetype(value)
tname = _MAP_VALTYPE_NAMES.get(type_)
if tname is None:
error = "Unsupported type in compact map value: %i" % type_
be_api.be_raise(vm, "internal_error", error)
return
if var_isstatic(value):
_logfmt(fout, "%s | BE_STATIC, ", tname)
else:
_logfmt(fout, "%s, ", tname)
if type_ == BE_INSTANCE:
ins = var_toobj(value)
cl = ins._class
cl_name = be_string.be_str2cstr(cl.name)
if cl_name == "bytes":
bufptr = var_toobj(ins.members[0])
length = var_toint(ins.members[1])
if bufptr is not None and length > 0:
hex_out = be_byteslib.be_bytes_tohex(bufptr, length)
else:
hex_out = ""
_lognofmt(fout, "be_kv_bytes_instance(")
_lognofmt(fout, hex_out)
_lognofmt(fout, ")")
elif ins.super is not None or ins.sub is not None:
be_api.be_raise(vm, "internal_error",
"instance must not have a super/sub class")
else:
if cl_name == "map":
cl_ptr = "map"
elif cl_name == "list":
cl_ptr = "list"
else:
be_api.be_raise(vm, "internal_error", "unsupported class")
return
_logfmt(fout,
"be_kv_ptr(be_nested_simple_instance(&be_class_%s, {\n",
cl_ptr)
if cl_ptr == "map":
_lognofmt(fout, " be_const_map( * ")
else:
_lognofmt(fout, " be_const_list( * ")
m_solidify_bvalue(vm, str_literal, ins.members[0],
prefix_name, key, fout)
_lognofmt(fout, " ) } ))")
elif type_ == BE_MAP:
_lognofmt(fout, "be_kv_ptr(")
m_solidify_map(vm, str_literal, var_toobj(value), prefix_name, fout)
_lognofmt(fout, ")")
elif type_ == BE_LIST:
_lognofmt(fout, "be_kv_ptr(")
m_solidify_list(vm, str_literal, var_toobj(value), prefix_name, fout)
_lognofmt(fout, ")")
else:
m_solidify_kval(vm, str_literal, value, prefix_name, key, fout)
# ============================================================================
# Compact ktab emitters (BE_USE_COMPACT_KTAB) — mirror be_solidifylib.c
# ============================================================================
# static void m_solidify_kval(bvm *vm, bbool str_literal, const bvalue * value,
# const char *prefix_name, const char *key, void* fout)
# emit the `union bvaldata` payload-word initializer for one constant
def m_solidify_kval(vm, str_literal, value, prefix_name, key, fout):
"""Emit the union bvaldata payload-word initializer for one constant.
Mirrors m_solidify_kval in be_solidifylib.c. Produces be_kv_* macros."""
be_string = _lazy_be_string()
be_api = _lazy_be_api()
type_ = var_primetype(value)
if type_ == BE_NIL:
_lognofmt(fout, "be_kv_nil()")
elif type_ == BE_BOOL:
_logfmt(fout, "be_kv_bool(%i)", 1 if var_tobool(value) else 0)
elif type_ == BE_INT or type_ == BE_INDEX:
_logfmt(fout, "be_kv_int(%i)", var_toint(value))
elif type_ == BE_REAL:
real_val = var_toreal(value)
if BE_USE_SINGLE_FLOAT:
raw = struct.unpack('>I', struct.pack('>f', real_val))[0]
_logfmt(fout, "be_kv_real(0x%08X)", raw)
else:
raw = struct.unpack('<Q', struct.pack('<d', real_val))[0]
_logfmt(fout, "be_kv_real(0x%016x)", raw)
elif type_ == BE_STRING:
s = be_string.be_str2cstr(var_tostr(value))
slen = len(s)
id_buf = toidentifier(s)
# be_kv_str / be_kv_str_weak / be_kv_str_long carry the &be_const_str_
# detail inside the macro and are recognized by coc (strong / weak /
# long-bclstring) so the referenced string is registered
if slen >= 255:
_logfmt(fout, "be_kv_str_long(%s)", id_buf)
elif not str_literal:
_logfmt(fout, "be_kv_str(%s)", id_buf)
else:
_logfmt(fout, "be_kv_str_weak(%s)", id_buf)
elif type_ == BE_CLOSURE:
clo = var_toobj(value)
func_name = be_string.be_str2cstr(clo.proto.name)
func_name_id = toidentifier(func_name)
prefix = prefix_name if prefix_name else ""
sep = "_" if prefix_name else ""
_logfmt(fout, "be_kv_closure(%s%s%s_closure)",
prefix, sep, func_name_id)
elif type_ == BE_CLASS:
cl = var_toobj(value)
_logfmt(fout, "be_kv_class(be_class_%s)",
be_string.be_str2cstr(cl.name))
elif type_ == BE_COMPTR:
_logfmt(fout, "be_kv_comptr(&be_ntv_%s_%s)",
prefix_name if prefix_name else "unknown",
key if key else "unknown")
elif type_ == BE_NTVFUNC:
_logfmt(fout, "be_kv_func(be_ntv_%s_%s)",
prefix_name if prefix_name else "unknown",
key if key else "unknown")
else:
error = "Unsupported type in compact ktab constants: %i" % type_
be_api.be_raise(vm, "internal_error", error)
# static void m_solidify_ktype(bvm *vm, const bvalue * value, void* fout)
# emit the type-byte symbolic name for one constant (with BE_STATIC if set)
_KTYPE_NAMES = {
BE_NIL: "BE_NIL",
BE_BOOL: "BE_BOOL",
BE_INT: "BE_INT",
BE_INDEX: "BE_INDEX",
BE_REAL: "BE_REAL",
BE_STRING: "BE_STRING",
BE_CLOSURE: "BE_CLOSURE",
BE_CLASS: "BE_CLASS",
BE_COMPTR: "BE_COMPTR",
BE_NTVFUNC: "BE_NTVFUNC",
}
def m_solidify_ktype(vm, value, fout):
"""Emit the type-byte symbolic name for one constant (OR BE_STATIC).
Mirrors m_solidify_ktype in be_solidifylib.c."""
be_api = _lazy_be_api()
type_ = var_primetype(value)
name = _KTYPE_NAMES.get(type_)
if name is None:
error = "Unsupported type in compact ktab constants: %i" % type_
be_api.be_raise(vm, "internal_error", error)
return
if var_isstatic(value):
_logfmt(fout, "%s | BE_STATIC", name)
else:
_lognofmt(fout, name)
# static void m_solidify_proto_ktab_compact(bvm *vm, bbool str_literal,
# const bproto *pr, int indent, void* fout)
# emit a proto's constant table as two inline arrays (payload words + types)
def m_solidify_proto_ktab_compact(vm, str_literal, pr, indent, fout):
"""Emit a proto's constant table as two inline arrays (values + types)."""
_logfmt(fout, "%*s( &(const union bvaldata[%2d]) { /* constants */\n",
indent, "", pr.nconst)
for k in range(pr.nconst):
_logfmt(fout, "%*s/* K%-3d */ ", indent, "", k)
m_solidify_kval(vm, str_literal, pr.ktab[k], None, None, fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s}),\n", indent, "")
_logfmt(fout, "%*s( &(const bbyte[%2d]) { /* constant types */\n",
indent, "", pr.nconst)
for k in range(pr.nconst):
_logfmt(fout, "%*s/* K%-3d */ ", indent, "", k)
m_solidify_ktype(vm, pr.ktab[k], fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s}),\n", indent, "")
# ============================================================================
# m_solidify_closure_inner_class (from be_solidifylib.c)
@@ -677,21 +924,32 @@ def m_solidify_proto(vm, str_literal, pr, func_name, indent, prefix_name, fout):
_logfmt(fout, "%*s%d, /* has constants */\n",
indent, "", 1 if (pr.nconst > 0) else 0)
if pr.nconst > 0:
if pr.varg & BE_VA_SHARED_KTAB:
_logfmt(fout, "%*s&be_ktab_%s, /* shared constants */\n",
indent, "", prefix_name)
if BE_USE_COMPACT_KTAB:
if pr.varg & BE_VA_SHARED_KTAB:
_logfmt(fout, "%*s&be_kval_%s, &be_ktype_%s, /* shared constants */\n",
indent, "", prefix_name, prefix_name)
else:
m_solidify_proto_ktab_compact(vm, str_literal, pr, indent, fout)
else:
_logfmt(fout, "%*s( &(const bvalue[%2d]) { /* constants */\n",
indent, "", pr.nconst)
for k in range(pr.nconst):
_logfmt(fout, "%*s/* K%-3d */ ", indent, "", k)
m_solidify_bvalue(vm, str_literal, pr.ktab[k],
None, None, fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s}),\n", indent, "")
if pr.varg & BE_VA_SHARED_KTAB:
_logfmt(fout, "%*s&be_ktab_%s, /* shared constants */\n",
indent, "", prefix_name)
else:
_logfmt(fout, "%*s( &(const bvalue[%2d]) { /* constants */\n",
indent, "", pr.nconst)
for k in range(pr.nconst):
_logfmt(fout, "%*s/* K%-3d */ ", indent, "", k)
m_solidify_bvalue(vm, str_literal, pr.ktab[k],
None, None, fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s}),\n", indent, "")
else:
_logfmt(fout, "%*sNULL, /* no const */\n",
indent, "")
if BE_USE_COMPACT_KTAB:
_logfmt(fout, "%*sNULL, NULL, /* no const */\n",
indent, "")
else:
_logfmt(fout, "%*sNULL, /* no const */\n",
indent, "")
# function name
name_str = be_string.be_str2cstr(pr.name)
@@ -1162,18 +1420,39 @@ def m_compact_class(vm, str_literal, cla, fout):
# output shared ktab
indent = 0
_logfmt(fout,
"// compact class '%s' ktab size: %d, total: %d (saved %i bytes)\n",
classname, ktab_size, ktab_total,
(ktab_total - ktab_size) * 8)
_logfmt(fout, "static const bvalue be_ktab_class_%s[%i] = {\n",
classname, ktab_size)
for k in range(ktab_size):
_logfmt(fout, "%*s/* K%-3d */ ", indent + 2, "", k)
m_solidify_bvalue(vm, str_literal, ktab[k], None, None, fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s};\n", indent, "")
_lognofmt(fout, "\n")
if BE_USE_COMPACT_KTAB:
_logfmt(fout,
"// compact class '%s' ktab size: %d, total: %d (saved %i bvalues)\n",
classname, ktab_size, ktab_total,
(ktab_total - ktab_size))
_logfmt(fout, "static const union bvaldata be_kval_class_%s[%i] = {\n",
classname, ktab_size)
for k in range(ktab_size):
_logfmt(fout, "%*s/* K%-3d */ ", indent + 2, "", k)
m_solidify_kval(vm, str_literal, ktab[k], None, None, fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s};\n", indent, "")
_logfmt(fout, "static const bbyte be_ktype_class_%s[%i] = {\n",
classname, ktab_size)
for k in range(ktab_size):
_logfmt(fout, "%*s/* K%-3d */ ", indent + 2, "", k)
m_solidify_ktype(vm, ktab[k], fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s};\n", indent, "")
_lognofmt(fout, "\n")
else:
_logfmt(fout,
"// compact class '%s' ktab size: %d, total: %d (saved %i bytes)\n",
classname, ktab_size, ktab_total,
(ktab_total - ktab_size) * 8)
_logfmt(fout, "static const bvalue be_ktab_class_%s[%i] = {\n",
classname, ktab_size)
for k in range(ktab_size):
_logfmt(fout, "%*s/* K%-3d */ ", indent + 2, "", k)
m_solidify_bvalue(vm, str_literal, ktab[k], None, None, fout)
_lognofmt(fout, ",\n")
_logfmt(fout, "%*s};\n", indent, "")
_lognofmt(fout, "\n")
lib/libesp32/berry/berry_port/berry_conf.py
+27
View File
@@ -44,6 +44,33 @@ BE_INTGER_TYPE = 0
# #define BE_USE_SINGLE_FLOAT 1
BE_USE_SINGLE_FLOAT = 1
# /* Macro: BE_USE_COMPACT_KTAB
# * Store each function's constant table (ktab) as a structure-of-arrays:
# * a payload-word array (`union bvaldata`) plus a parallel type-byte array,
# * instead of an array of full `bvalue`. Saves ~37% of constant-table flash
# * on 32-bit targets. Read-only at runtime; constants are materialized on
# * access. This flag controls the *solidify output format* so that the
# * Python solidifier emits the same C source as the C implementation.
# * Must match the C build's BE_USE_COMPACT_KTAB for the generated code to
# * compile correctly.
# * Default: 0 (disabled)
# **/
# #define BE_USE_COMPACT_KTAB 0
BE_USE_COMPACT_KTAB = 1
# /* Macro: BE_USE_COMPACT_MAP
# * Store the nodes of *constant* (solidified/flash) maps in a packed
# * 12-byte layout instead of the regular 16-byte `bmapnode`, by folding the
# * value's type byte into the spare bits of the key word. Saves 25% of the
# * flash used by solidified class/module member maps on 32-bit targets.
# * Mutable runtime maps are unchanged. This flag controls the *solidify
# * output format* so the Python solidifier emits the same C source as the C
# * implementation. Must match the C build's BE_USE_COMPACT_MAP.
# * Default: 0 (disabled)
# **/
# #define BE_USE_COMPACT_MAP 0
BE_USE_COMPACT_MAP = 1
# ---------------------------------------------------------------------------
# Bytes max size
# ---------------------------------------------------------------------------
lib/libesp32/berry/default/berry_conf.h
+45
View File
@@ -60,6 +60,51 @@
**/
#define BE_USE_PRECOMPILED_OBJECT 1
/* Macro: BE_USE_COMPACT_KTAB
* Store each function's constant table (ktab) as a structure-of-arrays:
* a payload-word array (`union bvaldata`) plus a parallel type-byte
* array, instead of an array of full `bvalue` (which wastes the padding
* of the 4-byte type field). On 32-bit targets (single-precision float,
* 32-bit int) a payload word is 4 bytes, so this saves ~37% of the flash
* occupied by constant tables of solidified code (executed in place from
* flash on ESP32). Constant tables are read-only at runtime, so constants
* are materialized into a scratch value on access; live registers are
* unchanged. The representation is portable: on 64-bit hosts the payload
* word is 8 bytes and the feature still works (and can be unit-tested),
* but the flash win is only meaningful on the 32-bit target.
* Default: 0 (disabled)
**/
#ifndef BE_USE_COMPACT_KTAB
#define BE_USE_COMPACT_KTAB 1
#endif
/* Macro: BE_USE_COMPACT_MAP
* Store the nodes of *constant* (solidified/flash) maps in a packed
* 12-byte layout instead of the regular 16-byte `bmapnode`. The value's
* type byte is folded into the spare bits of the key word (`next` is
* shrunk to 16 bits), removing the 3 padding bytes of the value's type
* field. On 32-bit targets this saves 25% of the flash used by solidified
* class/module member maps (executed in place from flash on ESP32).
*
* Only read-only (const) maps use the packed layout; mutable runtime maps
* keep the regular 16-byte layout untouched, so all map mutation paths and
* their `bvalue*` return contract are unchanged. Const-map reads decode the
* packed node into a scratch on access. The discriminator at runtime is
* `gc_isconst(map)`. Limited to 65535 slots per const map (ample).
*
* The representation is portable: on 64-bit hosts the node is larger (the
* payload word is 8 bytes) and the feature still works and is testable, but
* the flash win is only meaningful on the 32-bit target.
*
* NOTE: changing this flag changes the layout of every const map emitted by
* `coc` and by solidify, so the generated headers must be regenerated
* (`make prebuild`) and `berry_port/berry_conf.py` must match.
* Default: 0 (disabled)
**/
#ifndef BE_USE_COMPACT_MAP
#define BE_USE_COMPACT_MAP 1
#endif
/* Macro: BE_DEBUG_SOURCE_FILE
* Indicate if each function remembers its source file name
* 0: do not keep the file name (saves 4 bytes per function)
lib/libesp32/berry/src/be_api.c
+10 -11
View File
@@ -913,31 +913,30 @@ BERRY_API bbool be_pushiter(bvm *vm, int index)
var_setobj(iter, BE_COMPTR, NULL);
return btrue;
} else if (var_islist(v)) {
blist *list = var_toobj(v);
bvalue *iter = be_incrtop(vm);
var_setobj(iter, BE_COMPTR, be_list_data(list) - 1);
var_setint(iter, -1); /* index-based: start before first element */
return btrue;
}
return bfalse;
}
static int list_next(bvm *vm)
static int list_next(bvm *vm, bvalue *v)
{
bvalue *iter = be_indexof(vm, -1);
bvalue *next, *dst = be_incrtop(vm);
next = cast(bvalue*, var_toobj(iter)) + 1;
var_setobj(iter, BE_COMPTR, next);
var_setval(dst, next);
bint idx = var_toint(iter) + 1;
blist *list = var_toobj(v);
var_setint(iter, idx);
bvalue *dst = be_incrtop(vm);
var_setval(dst, be_list_at(list, idx));
return 1;
}
static bbool list_hasnext(bvm *vm, bvalue *v)
{
bvalue *next;
bvalue *iter = be_indexof(vm, -1);
blist *obj = var_toobj(v);
next = cast(bvalue*, var_toobj(iter)) + 1;
return next >= be_list_data(obj) && next < be_list_end(obj);
bint idx = var_toint(iter) + 1;
return idx >= 0 && idx < be_list_count(obj);
}
static int map_next(bvm *vm, bvalue *v)
@@ -969,7 +968,7 @@ BERRY_API int be_iter_next(bvm *vm, int index)
{
bvalue *o = be_indexof(vm, index);
if (var_islist(o)) {
return list_next(vm);
return list_next(vm, o);
} else if (var_ismap(o)) {
return map_next(vm, o);
}
lib/libesp32/berry/src/be_bytecode.c
+43 -4
View File
@@ -208,17 +208,19 @@ static void save_bytecode(bvm *vm, void *fp, bproto *proto)
static void save_constants(bvm *vm, void *fp, bproto *proto)
{
bvalue *v = proto->ktab, *end;
int i;
save_long(fp, proto->nconst); /* constants count */
for (end = v + proto->nconst; v < end; ++v) {
if ((v == proto->ktab) && (proto->varg & BE_VA_STATICMETHOD) && (v->type == BE_CLASS)) {
for (i = 0; i < proto->nconst; ++i) {
bvalue v;
proto_const_get(proto, i, v);
if ((i == 0) && (proto->varg & BE_VA_STATICMETHOD) && (v.type == BE_CLASS)) {
/* implicit `_class` parameter, output nil */
bvalue v_nil;
v_nil.v.i = 0;
v_nil.type = BE_NIL;
save_value(vm, fp, &v_nil);
} else {
save_value(vm, fp, v);
save_value(vm, fp, &v);
}
}
}
@@ -474,11 +476,19 @@ static void load_class(bvm *vm, void *fp, bvalue *v, int version)
bproto *proto = (bproto*)var_toobj(value);
bbool is_method = proto->varg & BE_VA_METHOD;
if (!is_method) {
#if BE_USE_COMPACT_KTAB
if ((proto->nconst > 0) && (proto->ktype[0] == BE_NIL)) {
/* The first argument is nil so we replace with the class as implicit '_class' */
proto->ktype[0] = BE_CLASS;
proto->kval[0].p = c;
}
#else
if ((proto->nconst > 0) && (proto->ktab->type == BE_NIL)) {
/* The first argument is nil so we replace with the class as implicit '_class' */
proto->ktab->type = BE_CLASS;
proto->ktab->v.p = c;
}
#endif
}
be_class_method_bind(vm, c, name, var_toobj(value), !is_method);
} else {
@@ -548,6 +558,34 @@ static void load_constant(bvm *vm, void *fp, bproto *proto, int version)
{
int size = load_count(vm, fp, "constant"); /* nconst */
if (size) {
#if BE_USE_COMPACT_KTAB
/* Load into a temporary bvalue[] and only build the compact arrays
* once everything is loaded. The temporary array is published to the
* proto using the in-progress sentinel (`ktype == NULL` means "`kval`
* is really a `bvalue[]` of `nconst` entries", see be_gc.c), so the
* partially-loaded constants stay GC-reachable through the proto.
* This matters because loading a single constant can allocate and
* trigger a GC: e.g. a class constant loads its methods, and the class
* (already stored in the array) must not be collected while its method
* protos are being read. Loading into a non-reachable C local instead
* would let the GC free that class -> heap-use-after-free. */
bvalue *v, *end, *tmp = be_malloc(vm, sizeof(bvalue) * size);
for (v = tmp, end = tmp + size; v < end; ++v) {
v->v.i = 0;
var_setnil(v);
}
proto->kval = (union bvaldata*)tmp; /* sentinel: ktype == NULL */
proto->ktype = NULL;
proto->nconst = (int16_t)size;
for (v = tmp, end = tmp + size; v < end; ++v) {
load_value(vm, fp, v, version);
}
/* build the compact arrays from the fully-loaded temporary; the
* temporary stays reachable (sentinel) across the allocation in
* be_proto_set_ktab, then is freed */
be_proto_set_ktab(vm, proto, tmp, size);
be_free(vm, tmp, sizeof(bvalue) * size);
#else
bvalue *end, *v = be_malloc(vm, sizeof(bvalue) * size);
memset(v, 0, sizeof(bvalue) * size);
proto->ktab = v;
@@ -555,6 +593,7 @@ static void load_constant(bvm *vm, void *fp, bproto *proto, int version)
for (end = v + size; v < end; ++v) {
load_value(vm, fp, v, version);
}
#endif
}
}
lib/libesp32/berry/src/be_byteslib.c
+135
View File
@@ -1806,6 +1806,70 @@ end
/********************************************************************
** Solidified function: getbits
********************************************************************/
#if BE_USE_COMPACT_KTAB
be_local_closure(getbits, /* name */
be_nested_proto(
9, /* nstack */
3, /* argc */
0, /* varg */
0, /* has upvals */
NULL, /* no upvals */
0, /* has sup protos */
NULL, /* no sub protos */
1, /* has constants */
( &(const union bvaldata[ 5]) { /* constants */
/* K0 */ be_kv_int(0),
/* K1 */ be_kv_str(value_error),
/* K2 */ be_kv_str(length_X20in_X20bits_X20must_X20be_X20between_X200_X20and_X2032),
/* K3 */ be_kv_int(3),
/* K4 */ be_kv_int(1),
}),
( &(const bbyte[ 5]) { /* constant types */
/* K0 */ BE_INT,
/* K1 */ BE_STRING,
/* K2 */ BE_STRING,
/* K3 */ BE_INT,
/* K4 */ BE_INT,
}),
&be_const_str_getbits,
&be_const_str_solidified,
( &(const binstruction[32]) { /* code */
0x180C0500, // 0000 LE R3 R2 K0
0x740E0002, // 0001 JMPT R3 #0005
0x540E001F, // 0002 LDINT R3 32
0x240C0403, // 0003 GT R3 R2 R3
0x780E0000, // 0004 JMPF R3 #0006
0xB0060302, // 0005 RAISE 1 K1 K2
0x580C0000, // 0006 LDCONST R3 K0
0x3C100303, // 0007 SHR R4 R1 K3
0x54160007, // 0008 LDINT R5 8
0x10040205, // 0009 MOD R1 R1 R5
0x58140000, // 000A LDCONST R5 K0
0x24180500, // 000B GT R6 R2 K0
0x781A0011, // 000C JMPF R6 #001F
0x541A0007, // 000D LDINT R6 8
0x04180C01, // 000E SUB R6 R6 R1
0x241C0C02, // 000F GT R7 R6 R2
0x781E0000, // 0010 JMPF R7 #0012
0x5C180400, // 0011 MOVE R6 R2
0x381E0806, // 0012 SHL R7 K4 R6
0x041C0F04, // 0013 SUB R7 R7 K4
0x381C0E01, // 0014 SHL R7 R7 R1
0x94200004, // 0015 GETIDX R8 R0 R4
0x2C201007, // 0016 AND R8 R8 R7
0x3C201001, // 0017 SHR R8 R8 R1
0x38201005, // 0018 SHL R8 R8 R5
0x300C0608, // 0019 OR R3 R3 R8
0x00140A06, // 001A ADD R5 R5 R6
0x04080406, // 001B SUB R2 R2 R6
0x58040000, // 001C LDCONST R1 K0
0x00100904, // 001D ADD R4 R4 K4
0x7001FFEB, // 001E JMP #000B
0x80040600, // 001F RET 1 R3
})
)
);
#else
be_local_closure(getbits, /* name */
be_nested_proto(
9, /* nstack */
@@ -1861,11 +1925,81 @@ be_local_closure(getbits, /* name */
})
)
);
#endif
/*******************************************************************/
/********************************************************************
** Solidified function: setbits
********************************************************************/
#if BE_USE_COMPACT_KTAB
be_local_closure(setbits, /* name */
be_nested_proto(
10, /* nstack */
4, /* argc */
0, /* varg */
0, /* has upvals */
NULL, /* no upvals */
0, /* has sup protos */
NULL, /* no sub protos */
1, /* has constants */
( &(const union bvaldata[ 5]) { /* constants */
/* K0 */ be_kv_int(0),
/* K1 */ be_kv_str(value_error),
/* K2 */ be_kv_str(length_X20in_X20bits_X20must_X20be_X20between_X200_X20and_X2032),
/* K3 */ be_kv_int(3),
/* K4 */ be_kv_int(1),
}),
( &(const bbyte[ 5]) { /* constant types */
/* K0 */ BE_INT,
/* K1 */ BE_STRING,
/* K2 */ BE_STRING,
/* K3 */ BE_INT,
/* K4 */ BE_INT,
}),
&be_const_str_setbits,
&be_const_str_solidified,
( &(const binstruction[37]) { /* code */
0x14100500, // 0000 LT R4 R2 K0
0x74120002, // 0001 JMPT R4 #0005
0x5412001F, // 0002 LDINT R4 32
0x24100404, // 0003 GT R4 R2 R4
0x78120000, // 0004 JMPF R4 #0006
0xB0060302, // 0005 RAISE 1 K1 K2
0x60100009, // 0006 GETGBL R4 G9
0x5C140600, // 0007 MOVE R5 R3
0x7C100200, // 0008 CALL R4 1
0x5C0C0800, // 0009 MOVE R3 R4
0x3C100303, // 000A SHR R4 R1 K3
0x54160007, // 000B LDINT R5 8
0x10040205, // 000C MOD R1 R1 R5
0x24140500, // 000D GT R5 R2 K0
0x78160014, // 000E JMPF R5 #0024
0x54160007, // 000F LDINT R5 8
0x04140A01, // 0010 SUB R5 R5 R1
0x24180A02, // 0011 GT R6 R5 R2
0x781A0000, // 0012 JMPF R6 #0014
0x5C140400, // 0013 MOVE R5 R2
0x381A0805, // 0014 SHL R6 K4 R5
0x04180D04, // 0015 SUB R6 R6 K4
0x541E00FE, // 0016 LDINT R7 255
0x38200C01, // 0017 SHL R8 R6 R1
0x041C0E08, // 0018 SUB R7 R7 R8
0x94200004, // 0019 GETIDX R8 R0 R4
0x2C201007, // 001A AND R8 R8 R7
0x2C240606, // 001B AND R9 R3 R6
0x38241201, // 001C SHL R9 R9 R1
0x30201009, // 001D OR R8 R8 R9
0x98000808, // 001E SETIDX R0 R4 R8
0x3C0C0605, // 001F SHR R3 R3 R5
0x04080405, // 0020 SUB R2 R2 R5
0x58040000, // 0021 LDCONST R1 K0
0x00100904, // 0022 ADD R4 R4 K4
0x7001FFE8, // 0023 JMP #000D
0x80040000, // 0024 RET 1 R0
})
)
);
#else
be_local_closure(setbits, /* name */
be_nested_proto(
10, /* nstack */
@@ -1926,6 +2060,7 @@ be_local_closure(setbits, /* name */
})
)
);
#endif
/*******************************************************************/
#if !BE_USE_PRECOMPILED_OBJECT
lib/libesp32/berry/src/be_code.c
+14
View File
@@ -261,11 +261,25 @@ static int newconst(bfuncinfo *finfo, bvalue *k)
{
int idx = be_vector_count(&finfo->kvec);
be_vector_push_c(finfo->lexer->vm, &finfo->kvec, k);
#if BE_USE_COMPACT_KTAB
/* During compilation the `bvalue` vector `kvec` is the source of truth, but
* we must keep it reachable by the GC (constants such as classes are only
* referenced from here). We expose it through proto->kval with ktype==NULL
* acting as a sentinel meaning "in-progress, kval is actually a bvalue[]".
* end_func later replaces this with the real compact kval/ktype. */
finfo->proto->kval = (union bvaldata*) be_vector_data(&finfo->kvec);
finfo->proto->ktype = NULL;
finfo->proto->nconst = be_vector_capacity(&finfo->kvec);
if (k == NULL) {
var_setnil((bvalue*)be_vector_at(&finfo->kvec, idx));
}
#else
finfo->proto->ktab = be_vector_data(&finfo->kvec);
finfo->proto->nconst = be_vector_capacity(&finfo->kvec);
if (k == NULL) {
var_setnil(&finfo->proto->ktab[idx]);
}
#endif
return idx;
}
lib/libesp32/berry/src/be_constobj.h
+131
View File
@@ -99,6 +99,21 @@ extern "C" {
.next = (uint32_t)(_next) & 0xFFFFFF \
}
/* Compact map node (BE_USE_COMPACT_MAP): the node is
* { key_v, key_type, next, val_type, val_v }. The be_ckey_* macros emit the
* first three elements (key payload, key type, chain link masked to 16 bits);
* the node template then appends the value type byte and the value payload.
* be_ckey / be_ckey_weak are recognized by coc (strong / weak) like
* be_const_key / be_const_key_weak. */
#define be_ckey(_str, _next) \
{ .c = &be_const_str_##_str }, BE_STRING, ((uint32_t)(_next) & 0xFFFF)
#define be_ckey_weak(_str, _next) \
{ .c = &be_const_str_##_str }, BE_STRING, ((uint32_t)(_next) & 0xFFFF)
#define be_ckey_int(_i, _next) \
{ .i = (bint)(_i) }, BE_INT, ((uint32_t)(_next) & 0xFFFF)
#define be_const_func(_func) { \
.v.nf = (_func), \
.type = BE_NTVFUNC \
@@ -184,8 +199,13 @@ extern "C" {
.type = BE_LIST \
}
#if BE_USE_COMPACT_MAP
#define be_define_const_map_slots(_name) \
const bmapnodec _name##_slots[] =
#else
#define be_define_const_map_slots(_name) \
const bmapnode _name##_slots[] =
#endif
#define be_define_const_map(_name, _size) \
const bmap _name = { \
@@ -352,6 +372,65 @@ const bntvmodule_t be_native_module(_module) = { \
(uint32_t)(_next) & 0xFFFFFF \
}
/* Compact-map (BE_USE_COMPACT_MAP) literal-string key: emits the first three
* elements of a bmapnodec initializer (key payload, key type, chain link masked
* to 16 bits). Compact sibling of be_nested_key, for hand-written const maps. */
#define be_ckey_nested(_str, _hash, _len, _next) \
{ .s=(be_nested_const_str(_str, _hash, _len )) }, \
BE_STRING, \
(uint32_t)(_next) & 0xFFFF
/* Compact constant table (BE_USE_COMPACT_KTAB): payload-word initializers.
* These produce a `union bvaldata` (no type field) used in the split
* constant arrays emitted by solidify. The type lives in the parallel
* `bbyte` array. Mirrors the readable be_const_* style.
* be_kv_str / be_kv_str_weak / be_kv_str_long are recognized by the `coc`
* tool (strong / weak / long-bclstring respectively) so the referenced
* string is registered in the string table — keep them in lockstep with
* tools/coc/coc_parser.py. */
#define be_kv_nil() { .i = 0 }
#define be_kv_int(_v) { .i = (bint)(_v) }
#define be_kv_bool(_v) { .b = (bbool)(_v) }
#define be_kv_real(_hex) { .p = (void*)(_hex) }
#define be_kv_str(_name) { .s = (bstring*)&be_const_str_##_name }
#define be_kv_str_weak(_name) { .s = (bstring*)&be_const_str_##_name }
#define be_kv_str_long(_name) { .s = (bstring*)&be_const_str_##_name }
#define be_kv_class(_class) { .c = &(_class) }
#define be_kv_closure(_closure) { .c = &(_closure) }
#define be_kv_comptr(_ptr) { .c = (const void*)(_ptr) }
#define be_kv_func(_func) { .nf = (_func) }
/* Compact map value payloads (BE_USE_COMPACT_MAP) for the types that do not
* occur in a function ktab: a generic pointer payload (nested map / list /
* simple instance), and a bytes instance. be_kv_bytes_instance is recognized
* by coc so the referenced bytes literal is registered. */
#define be_kv_ptr(_expr) { .c = (const void*)(_expr) }
#define be_kv_bytes_instance(_bytes) { .c = &be_const_instance_##_bytes }
/* be_ckv_*: a full compact-map-node value = "<type byte>, <payload>" (two
* positional elements of bmapnodec: val_type then val_v). These mirror the
* be_const_* family and are produced by the coc generator (block_builder.py)
* from `be_const_<x>(...)` -> `be_ckv_<x>(...)`. Keep in lockstep with the
* be_const_* macros above. */
#define be_ckv_nil() BE_NIL, { .i = 0 }
#define be_ckv_int(_v) BE_INT, { .i = (bint)(_v) }
#define be_ckv_var(_v) BE_INDEX, { .i = (bint)(_v) }
#define be_ckv_real(_v) BE_REAL, { .r = (breal)(_v) }
#define be_ckv_real_hex(_v) BE_REAL, { .p = (void*)(_v) }
#define be_ckv_bool(_v) BE_BOOL, { .b = (bbool)(_v) }
#define be_ckv_str(_s) BE_STRING, { .s = (bstring*)(_s) }
#define be_ckv_comptr(_v) BE_COMPTR, { .c = (const void*)(_v) }
#define be_ckv_class(_c) BE_CLASS, { .c = &(_c) }
#define be_ckv_closure(_c) BE_CLOSURE, { .c = &(_c) }
#define be_ckv_static_closure(_c) BE_CLOSURE | BE_STATIC, { .c = &(_c) }
#define be_ckv_func(_f) BE_NTVFUNC, { .nf = (_f) }
#define be_ckv_static_func(_f) BE_NTVFUNC | BE_STATIC, { .nf = (_f) }
#define be_ckv_module(_m) BE_MODULE, { .c = &(_m) }
#define be_ckv_simple_instance(_i) BE_INSTANCE, { .c = (_i) }
#define be_ckv_map(_m) BE_MAP, { .c = &(_m) }
#define be_ckv_list(_l) BE_LIST, { .c = &(_l) }
#define be_ckv_bytes_instance(_b) BE_INSTANCE, { .c = &be_const_instance_##_b }
#else
#define be_define_const_str_weak(_name, _s, _len) \
@@ -385,6 +464,16 @@ const bcstring be_const_str_##_name = { \
uint32_t((_next)&0xFFFFFF) \
}
/* Compact map node (BE_USE_COMPACT_MAP), C++ variant. */
#define be_ckey(_str, _next) \
bvaldata((const void*)&be_const_str_##_str), BE_STRING, uint32_t((_next)&0xFFFF)
#define be_ckey_weak(_str, _next) \
bvaldata((const void*)&be_const_str_##_str), BE_STRING, uint32_t((_next)&0xFFFF)
#define be_ckey_int(_i, _next) \
bvaldata(bint(_i)), BE_INT, uint32_t((_next)&0xFFFF)
#define be_const_func(_func) { \
bvaldata(_func), \
BE_NTVFUNC \
@@ -455,8 +544,13 @@ const bcstring be_const_str_##_name = { \
BE_MODULE \
}
#if BE_USE_COMPACT_MAP
#define be_define_const_map_slots(_name) \
const bmapnodec _name##_slots[] =
#else
#define be_define_const_map_slots(_name) \
const bmapnode _name##_slots[] =
#endif
#define be_define_const_map(_name, _size) \
const bmap _name( \
@@ -491,6 +585,43 @@ const bntvmodule_t be_native_module_##_module = { \
(bmodule*)&(m_lib##_module) \
}
/* Compact constant table (BE_USE_COMPACT_KTAB): payload-word initializers,
* C++ variant using the union bvaldata constructors. */
#define be_kv_nil() bvaldata(bint(0))
#define be_kv_int(_v) bvaldata(bint(_v))
#define be_kv_bool(_v) bvaldata(bbool(_v))
#define be_kv_real(_hex) bvaldata((void*)(_hex))
#define be_kv_str(_name) bvaldata((const void*)&be_const_str_##_name)
#define be_kv_str_weak(_name) bvaldata((const void*)&be_const_str_##_name)
#define be_kv_str_long(_name) bvaldata((const void*)&be_const_str_##_name)
#define be_kv_class(_class) bvaldata((const void*)&(_class))
#define be_kv_closure(_closure) bvaldata((const void*)&(_closure))
#define be_kv_comptr(_ptr) bvaldata((const void*)(_ptr))
#define be_kv_func(_func) bvaldata((bntvfunc)(_func))
#define be_kv_ptr(_expr) bvaldata((const void*)(_expr))
#define be_kv_bytes_instance(_bytes) bvaldata((const void*)&be_const_instance_##_bytes)
/* be_ckv_*: full compact-map-node value "<type byte>, <payload>", C++ variant.
* Mirrors the be_const_* C++ payloads. */
#define be_ckv_nil() BE_NIL, bvaldata(bint(0))
#define be_ckv_int(_v) BE_INT, bvaldata(bint(_v))
#define be_ckv_var(_v) BE_INDEX, bvaldata(bint(_v))
#define be_ckv_real(_v) BE_REAL, bvaldata(breal(_v))
#define be_ckv_real_hex(_v) BE_REAL, bvaldata((void*)(_v))
#define be_ckv_bool(_v) BE_BOOL, bvaldata(bbool(_v))
#define be_ckv_str(_s) BE_STRING, bvaldata(bstring(_s))
#define be_ckv_comptr(_v) BE_COMPTR, bvaldata((void*)(_v))
#define be_ckv_class(_c) BE_CLASS, bvaldata(&(_c))
#define be_ckv_closure(_c) BE_CLOSURE, bvaldata(&(_c))
#define be_ckv_static_closure(_c) BE_CLOSURE | BE_STATIC, bvaldata(&(_c))
#define be_ckv_func(_f) BE_NTVFUNC, bvaldata((bntvfunc)(_f))
#define be_ckv_static_func(_f) BE_NTVFUNC | BE_STATIC, bvaldata((bntvfunc)(_f))
#define be_ckv_module(_m) BE_MODULE, bvaldata(&(_m))
#define be_ckv_simple_instance(_i) BE_INSTANCE, bvaldata((const void*)(_i))
#define be_ckv_map(_m) BE_MAP, bvaldata((const void*)&(_m))
#define be_ckv_list(_l) BE_LIST, bvaldata((const void*)&(_l))
#define be_ckv_bytes_instance(_b) BE_INSTANCE, bvaldata((const void*)&be_const_instance_##_b)
#endif
/* provide pointers to map and list classes for solidified code */
lib/libesp32/berry/src/be_func.c
+37
View File
@@ -95,7 +95,12 @@ bproto* be_newproto(bvm *vm)
bproto *p = cast_proto(gco);
if (p) {
p->upvals = NULL;
#if BE_USE_COMPACT_KTAB
p->kval = NULL;
p->ktype = NULL;
#else
p->ktab = NULL;
#endif
p->ptab = NULL;
p->code = NULL;
p->name = NULL;
@@ -123,6 +128,38 @@ bproto* be_newproto(bvm *vm)
return p;
}
#if BE_USE_COMPACT_KTAB
void be_proto_set_ktab(bvm *vm, bproto *proto, const bvalue *src, int nconst)
{
if (nconst > 0) {
/* one allocation: nconst payload words followed by nconst type bytes.
* be_malloc may trigger GC, so we fill a fresh block and only publish
* it to the proto afterwards (the proto keeps its previous, still-valid
* kval/ktype/nconst until then) */
char *blk = be_malloc(vm, be_proto_ktab_size(nconst));
union bvaldata *kval = (union bvaldata*)blk;
bbyte *ktype = (bbyte*)(blk + (size_t)nconst * sizeof(union bvaldata));
int i;
for (i = 0; i < nconst; ++i) {
if (src) {
kval[i] = src[i].v;
ktype[i] = (bbyte)src[i].type;
} else {
kval[i].i = 0; /* nil-initialize: GC-safe placeholder */
ktype[i] = BE_NIL;
}
}
proto->kval = kval;
proto->ktype = ktype;
proto->nconst = (int16_t)nconst;
} else {
proto->kval = NULL;
proto->ktype = NULL;
proto->nconst = 0;
}
}
#endif
bclosure* be_newclosure(bvm *vm, int nupval)
{
bgcobject *gco = be_newgcobj(vm, BE_CLOSURE, clousersize(nupval));
lib/libesp32/berry/src/be_func.h
+10
View File
@@ -20,6 +20,16 @@ void be_initupvals(bvm *vm, bclosure *cl);
void be_upvals_close(bvm *vm, bvalue *level);
void be_release_upvalues(bvm *vm, bclosure *cl);
bproto* be_newproto(bvm *vm);
#if BE_USE_COMPACT_KTAB
/* Build a proto's compact constant table (kval+ktype, in a single
* allocation) from a temporary `bvalue` array of `nconst` entries.
* Sets proto->kval, proto->ktype and proto->nconst. Does not free `src`;
* the caller owns it. `src` may be NULL when nconst == 0. */
void be_proto_set_ktab(bvm *vm, bproto *proto, const bvalue *src, int nconst);
/* Size in bytes of a runtime-allocated compact ktab block of `nconst`. */
#define be_proto_ktab_size(_nconst) \
((size_t)(_nconst) * (sizeof(union bvaldata) + sizeof(bbyte)))
#endif
bclosure* be_newclosure(bvm *vm, int nupval);
bntvclos* be_newntvclosure(bvm *vm, bntvfunc cf, int nupvals);
bstring* be_func_varname(bproto *proto, int index, int pc);
lib/libesp32/berry/src/be_gc.c
+28 -1
View File
@@ -222,12 +222,28 @@ static void mark_proto(bvm *vm, bgcobject *obj)
bproto *p = cast_proto(obj);
gc_try (p != NULL) {
int count;
bvalue *k = p->ktab;
bproto **ptab = p->ptab;
vm->gc.gray = p->gray; /* remove object from gray list */
#if BE_USE_COMPACT_KTAB
if (p->ktype == NULL) {
/* in-progress (compile-time): kval actually points to a bvalue[] */
bvalue *k = (bvalue*)p->kval;
for (count = 0; count < p->nconst; ++count) {
mark_gray_var(vm, k + count);
}
} else {
for (count = 0; count < p->nconst; ++count) {
bvalue k;
proto_const_get(p, count, k); /* materialize constant to mark it */
mark_gray_var(vm, &k);
}
}
#else
bvalue *k = p->ktab;
for (count = p->nconst; count--; ++k) {
mark_gray_var(vm, k);
}
#endif
for (count = p->nproto; count--; ++ptab) {
mark_gray(vm, gc_object(*ptab));
}
@@ -324,7 +340,18 @@ static void free_proto(bvm *vm, bgcobject *obj)
if (!(proto->varg & BE_VA_SHARED_KTAB)) { /* do not free shared ktab */
/*caveat: the shared ktab is never GCed, in practice this is not a problem */
/* since shared ktab are primarily meant for solidification hence not gc-able */
#if BE_USE_COMPACT_KTAB
if (proto->ktype == NULL) {
/* in-progress / abandoned compile: kval is a bvalue[] */
be_free(vm, proto->kval, proto->nconst * sizeof(bvalue));
} else {
/* kval and ktype share a single allocation, kval at the front */
be_free(vm, proto->kval,
(size_t)proto->nconst * (sizeof(union bvaldata) + sizeof(bbyte)));
}
#else
be_free(vm, proto->ktab, proto->nconst * sizeof(bvalue));
#endif
}
be_free(vm, proto->ptab, proto->nproto * sizeof(bproto*));
be_free(vm, proto->code, proto->codesize * sizeof(binstruction));
lib/libesp32/berry/src/be_libs.c
+5
View File
@@ -15,6 +15,8 @@ extern void be_load_rangelib(bvm *vm);
extern void be_load_filelib(bvm *vm);
extern void be_load_byteslib(bvm *vm);
extern void be_load_relib(bvm *vm);
void be_loadlibs(bvm *vm)
{
be_load_baselib(vm);
@@ -25,5 +27,8 @@ void be_loadlibs(bvm *vm)
be_load_filelib(vm);
be_load_byteslib(vm);
be_load_baselib_next(vm);
#if BE_USE_RE_MODULE
be_load_relib(vm); /* class re_pattern */
#endif
#endif
}
lib/libesp32/berry/src/be_listlib.c
+6 -6
View File
@@ -294,15 +294,15 @@ static int iter_closure(bvm *vm)
* directly without using by the stack. */
bntvclos *func = var_toobj(vm->cf->func);
bvalue *uv0 = be_ntvclos_upval(func, 0)->value; /* list value */
bvalue *uv1 = be_ntvclos_upval(func, 1)->value; /* iter value */
bvalue *next = cast(bvalue*, var_toobj(uv1)) + 1;
bvalue *uv1 = be_ntvclos_upval(func, 1)->value; /* iter value (index) */
bint idx = var_toint(uv1) + 1;
blist *list = var_toobj(uv0);
if (next >= be_list_end(list)) {
if (idx >= be_list_count(list)) {
be_stop_iteration(vm);
}
var_toobj(uv1) = next; /* set upvale[1] (iter value) */
var_setint(uv1, idx); /* set upvalue[1] (iter index) */
/* push next value to top */
var_setval(vm->top, next);
var_setval(vm->top, be_list_at(list, idx));
be_incrtop(vm);
be_return(vm);
}
@@ -312,7 +312,7 @@ static int m_iter(bvm *vm)
be_pushntvclosure(vm, iter_closure, 2);
be_getmember(vm, 1, ".p");
be_setupval(vm, -2, 0);
be_pushiter(vm, -1);
be_pushint(vm, -1); /* start index before first element */
be_setupval(vm, -3, 1);
be_pop(vm, 2);
be_return(vm);
lib/libesp32/berry/src/be_map.c
+102
View File
@@ -123,6 +123,98 @@ static int eqnode(bvm *vm, bmapnode *node, bvalue *key, uint32_t hash)
return 0;
}
#if BE_USE_COMPACT_MAP
/* Compare a candidate key given as (type, payload) against `key`. Shared by
* the const-compact-map lookup path; uses the same equality rules as eqnode. */
static int keyeq(bvm *vm, int ktype, union bvaldata kv, bvalue *key, uint32_t hash)
{
if (var_isnil(key)) {
return 0;
}
#if BE_USE_OVERLOAD_HASH
if (var_isinstance(key)) {
bvalue tk;
tk.type = ktype;
tk.v = kv;
return be_vm_iseq(vm, key, &tk);
}
#endif
if ((signed char)ktype == key->type && _hashcode(vm, ktype, kv) == hash) {
bvalue tk;
tk.type = ktype;
tk.v = kv;
switch (key->type) {
case BE_BOOL: return var_tobool(key) == var_tobool(&tk);
case BE_INT: return var_toint(key) == var_toint(&tk);
case BE_REAL: return var_toreal(key) == var_toreal(&tk);
case BE_STRING: return be_eqstr(var_tostr(key), var_tostr(&tk));
default: return var_toobj(key) == var_toobj(&tk);
}
}
return 0;
}
/* Scratch values used to expose packed const-map nodes through the regular
* bvalue* / bmapnode* APIs. Safe because const maps are read-only and every
* caller consumes the returned value before the next map call (Berry is
* single-threaded). `find` and `next` use separate scratch so an iteration
* and a lookup do not clobber each other. */
static bvalue map_const_find_scratch;
static bmapnode map_const_next_scratch;
/* Lookup in a const (read-only) compact map. Returns a pointer to a scratch
* bvalue holding the decoded value, or NULL if not found. */
static bvalue* map_find_const(bvm *vm, bmap *map, bvalue *key)
{
if (map->size == 0) { /* solidified empty map */
return NULL;
}
uint32_t hash = hashcode(key);
bmapnodec *slots = (bmapnodec*)map->slots;
bmapnodec *slot = slots + (hash % (uint32_t)map->size);
if (slot->key_type == BE_NIL) {
return NULL;
}
for (;;) {
if (keyeq(vm, slot->key_type, slot->key_v, key, hash)) {
map_const_find_scratch.type = slot->val_type;
map_const_find_scratch.v = slot->val_v;
return &map_const_find_scratch;
}
uint32_t n = slot->next;
if (n == BE_MAP_LASTNODE_COMPACT) {
return NULL;
}
slot = slots + n;
}
}
/* Iterate a const (read-only) compact map. The cursor `*iter` walks the
* compact array (stored as a bmapnode* but really a bmapnodec*); the returned
* node is a scratch in the regular bmapnode layout so all existing iteration
* consumers (reading node->key / node->value) work unchanged. */
static bmapnode* map_next_const(bmap *map, bmapiter *iter)
{
bmapnodec *slots = (bmapnodec*)map->slots;
bmapnodec *end = slots + map->size;
bmapnodec *cur = (bmapnodec*)*iter;
cur = cur ? cur + 1 : slots;
while (cur < end && cur->key_type == BE_NIL) {
++cur;
}
*iter = (bmapnode*)cur; /* store the compact cursor for the next call */
if (cur < end) {
map_const_next_scratch.key.type = cur->key_type;
map_const_next_scratch.key.next = cur->next;
map_const_next_scratch.key.v = cur->key_v;
map_const_next_scratch.value.type = cur->val_type;
map_const_next_scratch.value.v = cur->val_v;
return &map_const_next_scratch;
}
return NULL;
}
#endif /* BE_USE_COMPACT_MAP */
static bmapnode* findprev(bmap *map, bmapnode *list, bmapnode *slot)
{
int n, pos = pos(map, slot);
@@ -251,6 +343,11 @@ void be_map_delete(bvm *vm, bmap *map)
bvalue* be_map_find(bvm *vm, bmap *map, bvalue *key)
{
#if BE_USE_COMPACT_MAP
if (gc_isconst(map)) {
return map_find_const(vm, map, key);
}
#endif
bmapnode *entry = find(vm, map, key, hashcode(key));
return entry ? value(entry) : NULL;
}
@@ -335,6 +432,11 @@ void be_map_removestr(bvm *vm, bmap *map, bstring *key)
bmapnode* be_map_next(bmap *map, bmapiter *iter)
{
#if BE_USE_COMPACT_MAP
if (gc_isconst(map)) {
return map_next_const(map, iter);
}
#endif
bmapnode *end = map->slots + map->size;
*iter = *iter ? *iter + 1 : map->slots;
while (*iter < end && isnil(*iter)) {
lib/libesp32/berry/src/be_map.h
+16
View File
@@ -21,6 +21,22 @@ typedef struct bmapnode {
bvalue value;
} bmapnode;
#if BE_USE_COMPACT_MAP
/* Packed read-only node for constant (solidified/flash) maps. The value's
* type byte is folded into the key word so the node is 12 bytes on 32-bit
* targets (vs 16 for bmapnode). Used ONLY for const maps; mutable runtime
* maps always use bmapnode. `bmap.slots` is reinterpreted as bmapnodec* when
* gc_isconst(map) is true. */
typedef struct bmapnodec {
union bvaldata key_v; /* key payload */
uint32_t key_type : 8; /* key type */
uint32_t next : 16; /* chain link (LASTNODE == 0xFFFF) */
uint32_t val_type : 8; /* value type (stolen from the old next:24) */
union bvaldata val_v; /* value payload */
} bmapnodec;
#define BE_MAP_LASTNODE_COMPACT 0xFFFF
#endif
struct bmap {
bcommon_header;
bgcobject *gray; /* for gc gray list */
lib/libesp32/berry/src/be_object.h
+29
View File
@@ -155,7 +155,19 @@ typedef struct bproto {
int16_t nproto; /* proto count */
bgcobject *gray; /* for gc gray list */
bupvaldesc *upvals;
#if BE_USE_COMPACT_KTAB
/* compact constant table: structure-of-arrays, the `type` byte is split
* out of the `bvalue` to avoid its 3 bytes of padding on 32-bit targets.
* `kval`/`ktype` are parallel arrays of `nconst` entries. For runtime
* protos both arrays live in a single allocation (kval first, ktype
* right after); for solidified protos they are two `const` flash arrays.
* Declared non-const (like the legacy `ktab`) so flash macros cast and
* the rare runtime mutations (bytecode loader, ktab sharing) compile. */
union bvaldata *kval; /* constants payload words */
bbyte *ktype; /* constants type bytes (parallel to kval) */
#else
bvalue *ktab; /* constants table */
#endif
struct bproto **ptab; /* proto table */
binstruction *code; /* instructions sequence */
bstring *name; /* function name */
@@ -260,6 +272,23 @@ typedef const char* (*breader)(struct blexer*, void*, size_t*);
#define var_tontvfunc(_v) ((_v)->v.nf)
#define var_toidx(_v) cast_int(var_toint(_v))
/* Generic read accessors for a proto's constant table. They work the same
* whether the build uses the compact (structure-of-arrays) representation or
* the legacy `bvalue[]`. `proto_const_get` materializes constant `_idx` of
* proto `_pr` into the `bvalue` lvalue `_dst`. `proto_const_type` returns the
* type byte. These are used on the cold paths (GC mark, bytecode save,
* solidify). The VM hot path inlines the equivalent logic directly. */
#if BE_USE_COMPACT_KTAB
#define proto_const_get(_pr, _idx, _dst) do { \
(_dst).v = (_pr)->kval[_idx]; \
(_dst).type = (_pr)->ktype[_idx]; \
} while (0)
#define proto_const_type(_pr, _idx) ((_pr)->ktype[_idx])
#else
#define proto_const_get(_pr, _idx, _dst) ((_dst) = (_pr)->ktab[_idx])
#define proto_const_type(_pr, _idx) ((_pr)->ktab[_idx].type)
#endif
const char* be_vtype2str(bvalue *v);
bstring* be_vtype2bstring(bvalue *v);
bvalue* be_indexof(bvm *vm, int idx);
lib/libesp32/berry/src/be_parser.c
+18
View File
@@ -259,8 +259,10 @@ static void begin_func(bparser *parser, bfuncinfo *finfo, bblockinfo *binfo)
proto->code = be_vector_data(&finfo->code);
proto->codesize = be_vector_capacity(&finfo->code);
be_vector_init(vm, &finfo->kvec, sizeof(bvalue)); /* vector for constants */
#if !BE_USE_COMPACT_KTAB
proto->ktab = be_vector_data(&finfo->kvec);
proto->nconst = be_vector_capacity(&finfo->kvec);
#endif
be_vector_init(vm, &finfo->pvec, sizeof(bproto*)); /* vector for subprotos */
proto->ptab = be_vector_data(&finfo->pvec);
proto->nproto = be_vector_capacity(&finfo->pvec);
@@ -331,13 +333,29 @@ static void end_func(bparser *parser)
setupvals(finfo); /* close upvals */
proto->code = be_vector_release(vm, &finfo->code); /* compact all vectors and return NULL if empty */
proto->codesize = finfo->pc;
#if BE_USE_COMPACT_KTAB
{ /* build the compact constant table from the scratch bvalue vector */
bvalue *kdata = be_vector_release(vm, &finfo->kvec);
int nconst = be_vector_count(&finfo->kvec);
/* keep the released bvalue[] visible to the GC (ktype==NULL sentinel)
* while be_proto_set_ktab allocates the compact block (which may GC) */
proto->kval = (union bvaldata*) kdata;
proto->ktype = NULL;
proto->nconst = (int16_t)nconst;
be_proto_set_ktab(vm, proto, kdata, nconst);
if (kdata) { be_free(vm, kdata, nconst * sizeof(bvalue)); }
}
#else
proto->ktab = be_vector_release(vm, &finfo->kvec);
proto->nconst = be_vector_count(&finfo->kvec);
#endif
proto->ptab = be_vector_release(vm, &finfo->pvec);
proto->nproto = be_vector_count(&finfo->pvec);
#if BE_USE_MEM_ALIGNED
proto->code = be_move_to_aligned(vm, proto->code, proto->codesize * sizeof(binstruction)); /* move `code` to 4-bytes aligned memory region */
#if !BE_USE_COMPACT_KTAB
proto->ktab = be_move_to_aligned(vm, proto->ktab, proto->nconst * sizeof(bvalue)); /* move `ktab` to 4-bytes aligned memory region */
#endif
#endif /* BE_USE_MEM_ALIGNED */
#if BE_DEBUG_RUNTIME_INFO
proto->lineinfo = be_vector_release(vm, &finfo->linevec); /* move `lineinfo` to 4-bytes aligned memory region */
lib/libesp32/berry/src/be_solidifylib.c
+343 -9
View File
@@ -117,6 +117,11 @@ static void toidentifier(char *to, const char *p)
static void m_solidify_bvalue(bvm *vm, bbool str_literal, const bvalue * value, const char *prefix_name, const char *key, void* fout);
#if BE_USE_COMPACT_MAP
static void m_solidify_map_value(bvm *vm, bbool str_literal, const bvalue * value, const char *prefix_name, const char *key, void* fout);
static void m_solidify_kval(bvm *vm, bbool str_literal, const bvalue * value, const char *prefix_name, const char *key, void* fout);
#endif
static void m_solidify_map(bvm *vm, bbool str_literal, bmap * map, const char *prefix_name, void* fout)
{
// compact first
@@ -124,6 +129,46 @@ static void m_solidify_map(bvm *vm, bbool str_literal, bmap * map, const char *p
logfmt(" be_nested_map(%i,\n", map->count);
#if BE_USE_COMPACT_MAP
logfmt(" ( (struct bmapnode*) &(const bmapnodec[]) {\n");
for (int i = 0; i < map->size; i++) {
bmapnode * node = &map->slots[i];
if (node->key.type == BE_NIL) {
continue; /* key not used */
}
int key_next = node->key.next;
if (0xFFFFFF == key_next) {
key_next = -1; /* more readable */
}
if (node->key.type == BE_STRING) {
/* convert the string literal to identifier */
const char * key = str(node->key.v.s);
size_t id_len = toidentifier_length(key);
char id_buf[id_len];
toidentifier(id_buf, key);
if (!str_literal) {
logfmt(" { be_ckey(%s, %i), ", id_buf, key_next);
} else {
logfmt(" { be_ckey_weak(%s, %i), ", id_buf, key_next);
}
m_solidify_map_value(vm, str_literal, &node->value, prefix_name, str(node->key.v.s), fout);
} else if (node->key.type == BE_INT) {
#if BE_INTGER_TYPE == 2
logfmt(" { be_ckey_int(%lli, %i), ", node->key.v.i, key_next);
#else
logfmt(" { be_ckey_int(%i, %i), ", node->key.v.i, key_next);
#endif
m_solidify_map_value(vm, str_literal, &node->value, prefix_name, NULL, fout);
} else {
char error[64];
snprintf(error, sizeof(error), "Unsupported type in key: %i", node->key.type);
be_raise(vm, "internal_error", error);
}
logfmt(" },\n");
}
logfmt(" }))");
#else
logfmt(" ( (struct bmapnode*) &(const bmapnode[]) {\n");
for (int i = 0; i < map->size; i++) {
bmapnode * node = &map->slots[i];
@@ -162,6 +207,7 @@ static void m_solidify_map(bvm *vm, bbool str_literal, bmap * map, const char *p
logfmt(" },\n");
}
logfmt(" }))"); // TODO need terminal comma?
#endif
}
@@ -178,6 +224,95 @@ static void m_solidify_list(bvm *vm, bbool str_literal, const blist * list, cons
logfmt(" }))"); // TODO need terminal comma?
}
#if BE_USE_COMPACT_MAP
/* Emit one compact map node's value: the value type byte, then the value
* payload (a `union bvaldata` initializer). Mirrors the legacy m_solidify_bvalue
* but splits type from payload to match the bmapnodec layout. */
static void m_solidify_map_value(bvm *vm, bbool str_literal, const bvalue * value, const char *prefix_name, const char *key, void* fout)
{
int type = var_primetype(value);
/* --- value type byte --- */
const char *tname;
switch (type) {
case BE_NIL: tname = "BE_NIL"; break;
case BE_BOOL: tname = "BE_BOOL"; break;
case BE_INT: tname = "BE_INT"; break;
case BE_INDEX: tname = "BE_INDEX"; break;
case BE_REAL: tname = "BE_REAL"; break;
case BE_STRING: tname = "BE_STRING"; break;
case BE_CLOSURE: tname = "BE_CLOSURE"; break;
case BE_CLASS: tname = "BE_CLASS"; break;
case BE_COMPTR: tname = "BE_COMPTR"; break;
case BE_NTVFUNC: tname = "BE_NTVFUNC"; break;
case BE_INSTANCE: tname = "BE_INSTANCE"; break;
case BE_MAP: tname = "BE_MAP"; break;
case BE_LIST: tname = "BE_LIST"; break;
default:
{
char error[80];
snprintf(error, sizeof(error), "Unsupported type in compact map value: %i", type);
be_raise(vm, "internal_error", error);
}
return;
}
if (var_isstatic(value)) {
logfmt("%s | BE_STATIC, ", tname);
} else {
logfmt("%s, ", tname);
}
/* --- value payload --- */
switch (type) {
case BE_INSTANCE:
{
binstance * ins = (binstance *) var_toobj(value);
bclass * cl = ins->_class;
if (cl == &be_class_bytes) {
const void * bufptr = var_toobj(&ins->members[0]);
int32_t len = var_toint(&ins->members[1]);
size_t hex_len = len * 2 + 1;
char * hex_out = be_pushbuffer(vm, hex_len);
be_bytes_tohex(hex_out, hex_len, bufptr, len);
lognofmt("be_kv_bytes_instance(");
lognofmt(hex_out);
lognofmt(")");
be_pop(vm, 1);
} else if (ins->super || ins->sub) {
be_raise(vm, "internal_error", "instance must not have a super/sub class");
} else {
const char * cl_ptr = "";
if (cl == &be_class_map) { cl_ptr = "map"; }
else if (cl == &be_class_list) { cl_ptr = "list"; }
else { be_raise(vm, "internal_error", "unsupported class"); }
/* payload is a pointer to a nested simple instance */
logfmt("be_kv_ptr(be_nested_simple_instance(&be_class_%s, {\n", cl_ptr);
if (cl == &be_class_map) {
logfmt(" be_const_map( * ");
} else {
logfmt(" be_const_list( * ");
}
m_solidify_bvalue(vm, str_literal, &ins->members[0], prefix_name, key, fout);
logfmt(" ) } ))");
}
}
break;
case BE_MAP:
logfmt("be_kv_ptr(");
m_solidify_map(vm, str_literal, (bmap *) var_toobj(value), prefix_name, fout);
logfmt(")");
break;
case BE_LIST:
logfmt("be_kv_ptr(");
m_solidify_list(vm, str_literal, (blist *) var_toobj(value), prefix_name, fout);
logfmt(")");
break;
default:
/* scalars, strings, closures, classes, comptr, ntvfunc */
m_solidify_kval(vm, str_literal, value, prefix_name, key, fout);
break;
}
}
#endif /* BE_USE_COMPACT_MAP */
// pass key name in case of class, or NULL if none
static void m_solidify_bvalue(bvm *vm, bbool str_literal, const bvalue * value, const char *prefix_name, const char *key, void* fout)
{
@@ -315,6 +450,156 @@ static void m_solidify_bvalue(bvm *vm, bbool str_literal, const bvalue * value,
static void m_solidify_subclass(bvm *vm, bbool str_literal, const bclass *cl, void* fout);
/* read constant `idx` of proto `pr` into a bvalue, regardless of layout */
static bvalue proto_kvalue(const bproto *pr, int idx)
{
bvalue v;
proto_const_get(pr, idx, v);
return v;
}
#if BE_USE_COMPACT_KTAB || BE_USE_COMPACT_MAP
/* emit the `union bvaldata` payload-word initializer for one constant */
static void m_solidify_kval(bvm *vm, bbool str_literal, const bvalue * value, const char *prefix_name, const char *key, void* fout)
{
int type = var_primetype(value);
switch (type) {
case BE_NIL:
logfmt("be_kv_nil()");
break;
case BE_BOOL:
logfmt("be_kv_bool(%i)", var_tobool(value));
break;
case BE_INT:
case BE_INDEX:
#if BE_INTGER_TYPE == 2
logfmt("be_kv_int(%lli)", var_toint(value));
#else
logfmt("be_kv_int(%i)", (int)var_toint(value));
#endif
break;
case BE_REAL:
#if BE_USE_SINGLE_FLOAT
logfmt("be_kv_real(0x%08" PRIX32 ")", (uint32_t)(uintptr_t)var_toobj(value));
#else
logfmt("be_kv_real(0x%016" PRIx64 ")", (uint64_t)var_toobj(value));
#endif
break;
case BE_STRING:
{
const char * cstr = str(var_tostr(value));
size_t len = strlen(cstr);
size_t id_len = toidentifier_length(cstr);
char id_buf_stack[64];
char *id_buf = id_buf_stack;
if (id_len >= 64) {
id_buf = be_os_malloc(id_len);
if (!id_buf) {
be_raise(vm, "memory_error", "could not allocated buffer");
}
}
toidentifier(id_buf, cstr);
/* be_kv_str / be_kv_str_weak / be_kv_str_long carry the &be_const_str_
* detail inside the macro and are recognized by coc (strong / weak /
* long-bclstring) so the referenced string is registered */
if (len >= 255) {
logfmt("be_kv_str_long(%s)", id_buf);
} else if (!str_literal) {
logfmt("be_kv_str(%s)", id_buf);
} else {
logfmt("be_kv_str_weak(%s)", id_buf);
}
if (id_buf != id_buf_stack) {
be_os_free(id_buf);
}
}
break;
case BE_CLOSURE:
{
bclosure *clo = (bclosure*) var_toobj(value);
const char * func_name = str(clo->proto->name);
size_t id_len = toidentifier_length(func_name);
char func_name_id[id_len];
toidentifier(func_name_id, func_name);
logfmt("be_kv_closure(%s%s%s_closure)",
prefix_name ? prefix_name : "", prefix_name ? "_" : "",
func_name_id);
}
break;
case BE_CLASS:
logfmt("be_kv_class(be_class_%s)", str(((bclass*) var_toobj(value))->name));
break;
case BE_COMPTR:
logfmt("be_kv_comptr(&be_ntv_%s_%s)", prefix_name ? prefix_name : "unknown", key ? key : "unknown");
break;
case BE_NTVFUNC:
logfmt("be_kv_func(be_ntv_%s_%s)", prefix_name ? prefix_name : "unknown", key ? key : "unknown");
break;
default:
{
char error[80];
snprintf(error, sizeof(error), "Unsupported type in compact ktab constants: %i", type);
be_raise(vm, "internal_error", error);
}
}
}
/* emit the type-byte symbolic name for one constant (with BE_STATIC if set) */
static void m_solidify_ktype(bvm *vm, const bvalue * value, void* fout)
{
int type = var_primetype(value);
const char *name;
switch (type) {
case BE_NIL: name = "BE_NIL"; break;
case BE_BOOL: name = "BE_BOOL"; break;
case BE_INT: name = "BE_INT"; break;
case BE_INDEX: name = "BE_INDEX"; break;
case BE_REAL: name = "BE_REAL"; break;
case BE_STRING: name = "BE_STRING"; break;
case BE_CLOSURE: name = "BE_CLOSURE"; break;
case BE_CLASS: name = "BE_CLASS"; break;
case BE_COMPTR: name = "BE_COMPTR"; break;
case BE_NTVFUNC: name = "BE_NTVFUNC"; break;
default:
{
char error[80];
snprintf(error, sizeof(error), "Unsupported type in compact ktab constants: %i", type);
be_raise(vm, "internal_error", error);
}
return;
}
if (var_isstatic(value)) {
logfmt("%s | BE_STATIC", name);
} else {
logfmt("%s", name);
}
}
#endif /* BE_USE_COMPACT_KTAB || BE_USE_COMPACT_MAP */
/* emit a proto's constant table as two inline arrays (payload words + types) */
#if BE_USE_COMPACT_KTAB
static void m_solidify_proto_ktab_compact(bvm *vm, bbool str_literal, const bproto *pr, int indent, void* fout)
{
logfmt("%*s( &(const union bvaldata[%2d]) { /* constants */\n", indent, "", pr->nconst);
for (int k = 0; k < pr->nconst; k++) {
bvalue kv = proto_kvalue(pr, k);
logfmt("%*s/* K%-3d */ ", indent, "", k);
m_solidify_kval(vm, str_literal, &kv, NULL, NULL, fout);
logfmt(",\n");
}
logfmt("%*s}),\n", indent, "");
logfmt("%*s( &(const bbyte[%2d]) { /* constant types */\n", indent, "", pr->nconst);
for (int k = 0; k < pr->nconst; k++) {
bvalue kv = proto_kvalue(pr, k);
logfmt("%*s/* K%-3d */ ", indent, "", k);
m_solidify_ktype(vm, &kv, fout);
logfmt(",\n");
}
logfmt("%*s}),\n", indent, "");
}
#endif /* BE_USE_COMPACT_KTAB */
/* solidify any inner class */
static void m_solidify_closure_inner_class(bvm *vm, bbool str_literal, const bclosure *clo, void* fout)
{
@@ -322,12 +607,13 @@ static void m_solidify_closure_inner_class(bvm *vm, bbool str_literal, const bcl
bproto *pr = clo->proto;
if ((!gc_isconst(clo)) && (pr->nconst > 0) && (!(pr->varg & BE_VA_SHARED_KTAB)) && (!(pr->varg & BE_VA_NOCOMPACT))) { /* if shared ktab or nocompact, skip */
for (int k = 0; k < pr->nconst; k++) {
if (var_type(&pr->ktab[k]) == BE_CLASS) {
if (proto_const_type(pr, k) == BE_CLASS) {
if ((k == 0) && (pr->varg & BE_VA_STATICMETHOD)) {
// it is the implicit '_class' variable from a static method, don't dump the class
} else {
// output the class
m_solidify_subclass(vm, str_literal, (bclass*) var_toobj(&pr->ktab[k]), fout);
bvalue kv = proto_kvalue(pr, k);
m_solidify_subclass(vm, str_literal, (bclass*) var_toobj(&kv), fout);
}
}
}
@@ -372,19 +658,32 @@ static void m_solidify_proto(bvm *vm, bbool str_literal, const bproto *pr, const
logfmt("%*s%d, /* has constants */\n", indent, "", (pr->nconst > 0) ? 1 : 0);
if (pr->nconst > 0) {
// we output the full table unless it's a shared ktab
#if BE_USE_COMPACT_KTAB
if (pr->varg & BE_VA_SHARED_KTAB) {
logfmt("%*s&be_kval_%s, &be_ktype_%s, /* shared constants */\n", indent, "", prefix_name, prefix_name);
} else {
m_solidify_proto_ktab_compact(vm, str_literal, pr, indent, fout);
}
#else
if (pr->varg & BE_VA_SHARED_KTAB) {
logfmt("%*s&be_ktab_%s, /* shared constants */\n", indent, "", prefix_name);
} else {
logfmt("%*s( &(const bvalue[%2d]) { /* constants */\n", indent, "", pr->nconst);
for (int k = 0; k < pr->nconst; k++) {
bvalue kv = proto_kvalue(pr, k);
logfmt("%*s/* K%-3d */ ", indent, "", k);
m_solidify_bvalue(vm, str_literal, &pr->ktab[k], NULL, NULL, fout);
m_solidify_bvalue(vm, str_literal, &kv, NULL, NULL, fout);
logfmt(",\n");
}
logfmt("%*s}),\n", indent, "");
}
#endif
} else {
#if BE_USE_COMPACT_KTAB
logfmt("%*sNULL, NULL, /* no const */\n", indent, "");
#else
logfmt("%*sNULL, /* no const */\n", indent, "");
#endif
}
/* convert the string literal to identifier */
@@ -631,20 +930,19 @@ static void m_compact_class(bvm *vm, bbool str_literal, const bclass *cla, void*
// iterate on each bvalue in ktab
for (int i = 0; i < pr->nconst; i++) {
bvalue ki = proto_kvalue(pr, i);
// look if the bvalue pair is already in ktab
int found = 0;
for (int j = 0; j < ktab_size; j++) {
// to avoid any size issue, we compare all bytes
// berry_log_C("// p1=%p p2=%p sz=%i", &pr->ktab[i], &ktab[j], sizeof(bvalue));
if ((pr->ktab[i].type == ktab[j].type) && (pr->ktab[i].v.i == ktab[j].v.i) && (pr->ktab[i].v.c == ktab[j].v.c)) {
// if (memcmp(&pr->ktab[i], &ktab[j], sizeof(bvalue)) == 0) {
if ((ki.type == ktab[j].type) && (ki.v.i == ktab[j].v.i) && (ki.v.c == ktab[j].v.c)) {
found = 1;
break;
}
}
// if not already there, add it
if (!found) {
ktab[ktab_size++] = pr->ktab[i];
ktab[ktab_size++] = ki;
}
if (ktab_size >= MAX_KTAB_SIZE) {
logfmt("// ktab too big for class '%s' - skipping\n", classname);
@@ -661,6 +959,18 @@ static void m_compact_class(bvm *vm, bbool str_literal, const bclass *cla, void*
/* allocate a proper ktab */
bvalue *new_ktab = be_malloc(vm, sizeof(bvalue) * ktab_size);
memmove(new_ktab, ktab, sizeof(bvalue) * ktab_size);
#if BE_USE_COMPACT_KTAB
/* build a single shared compact ktab block (kval words + type bytes)
* that every method of this class will point to at runtime */
char *shared_blk = be_malloc(vm, (size_t)ktab_size * (sizeof(union bvaldata) + sizeof(bbyte)));
union bvaldata *shared_kval = (union bvaldata*)shared_blk;
bbyte *shared_ktype = (bbyte*)(shared_blk + (size_t)ktab_size * sizeof(union bvaldata));
for (int i = 0; i < ktab_size; i++) {
shared_kval[i] = new_ktab[i].v;
shared_ktype[i] = (bbyte)new_ktab[i].type;
}
be_free(vm, new_ktab, sizeof(bvalue) * ktab_size); /* intermediate, no longer needed */
#endif
/* second iteration to replace ktab and patch code */
iter = be_map_iter();
@@ -674,10 +984,10 @@ static void m_compact_class(bvm *vm, bbool str_literal, const bclass *cla, void*
uint8_t mapping_array[MAX_KTAB_SIZE];
// iterate in proto ktab to get the index in the global ktab
for (int i = 0; i < pr->nconst; i++) {
bvalue ki = proto_kvalue(pr, i);
for (int j = 0; j < ktab_size; j++) {
// compare all bytes
if ((pr->ktab[i].type == ktab[j].type) && (pr->ktab[i].v.i == ktab[j].v.i) && (pr->ktab[i].v.c == ktab[j].v.c)) {
// if (memcmp(&pr->ktab[i], &ktab[j], sizeof(bvalue)) == 0) {
if ((ki.type == ktab[j].type) && (ki.v.i == ktab[j].v.i) && (ki.v.c == ktab[j].v.c)) {
mapping_array[i] = j;
break;
}
@@ -685,7 +995,12 @@ static void m_compact_class(bvm *vm, bbool str_literal, const bclass *cla, void*
}
// replace ktab
#if BE_USE_COMPACT_KTAB
pr->kval = shared_kval;
pr->ktype = shared_ktype;
#else
pr->ktab = new_ktab;
#endif
pr->nconst = ktab_size;
// flag as shared ktab
pr->varg |= BE_VA_SHARED_KTAB;
@@ -804,6 +1119,24 @@ static void m_compact_class(bvm *vm, bbool str_literal, const bclass *cla, void*
// output shared ktab
int indent = 0;
#if BE_USE_COMPACT_KTAB
logfmt("// compact class '%s' ktab size: %d, total: %d (saved %i bvalues)\n", classname, ktab_size, ktab_total, (ktab_total - ktab_size));
logfmt("static const union bvaldata be_kval_class_%s[%i] = {\n", classname, ktab_size);
for (int k = 0; k < ktab_size; k++) {
logfmt("%*s/* K%-3d */ ", indent + 2, "", k);
m_solidify_kval(vm, str_literal, &ktab[k], NULL, NULL, fout);
logfmt(",\n");
}
logfmt("%*s};\n", indent, "");
logfmt("static const bbyte be_ktype_class_%s[%i] = {\n", classname, ktab_size);
for (int k = 0; k < ktab_size; k++) {
logfmt("%*s/* K%-3d */ ", indent + 2, "", k);
m_solidify_ktype(vm, &ktab[k], fout);
logfmt(",\n");
}
logfmt("%*s};\n", indent, "");
logfmt("\n");
#else
logfmt("// compact class '%s' ktab size: %d, total: %d (saved %i bytes)\n", classname, ktab_size, ktab_total, (ktab_total - ktab_size) * 8);
logfmt("static const bvalue be_ktab_class_%s[%i] = {\n", classname, ktab_size);
for (int k = 0; k < ktab_size; k++) {
@@ -813,6 +1146,7 @@ static void m_compact_class(bvm *vm, bbool str_literal, const bclass *cla, void*
}
logfmt("%*s};\n", indent, "");
logfmt("\n");
#endif
}
// takes a class or a module
lib/libesp32/berry/src/be_var.c
+3 -2
View File
@@ -162,8 +162,9 @@ int be_builtin_find(bvm *vm, bstring *name)
bstring* be_builtin_name(bvm *vm, int index)
{
bmap *map = builtin(vm).vtab;
bmapnode *end, *node = map->slots;
for (end = node + map->size; node < end; ++node) {
bmapnode *node;
bmapiter iter = be_map_iter();
while ((node = be_map_next(map, &iter)) != NULL) {
if (var_isstr(&node->key) && node->value.v.i == index) {
return node->key.v.s;
}
lib/libesp32/berry/src/be_vm.c
+36 -1
View File
@@ -30,8 +30,21 @@
be_raise(vm, except, be_pushfstring(vm, __VA_ARGS__))
#define RA() (reg + IGET_RA(ins)) /* Get value of register A */
#if BE_USE_COMPACT_KTAB
/* With a compact constant table there is no `bvalue[]` to point into, so a
* constant operand is materialized into a per-operand scratch `bvalue`
* (`_krkb` for B, `_krkc` for C) and its address is returned. Register
* operands still return a direct pointer into the register file. */
#define RK_CONST(_idx, _scr) \
((_scr).v = kval[_idx], (_scr).type = ktype[_idx], &(_scr))
#define RKB() (isKB(ins) ? RK_CONST(KR2idx(IGET_RKB(ins)), _krkb) \
: reg + KR2idx(IGET_RKB(ins)))
#define RKC() (isKC(ins) ? RK_CONST(KR2idx(IGET_RKC(ins)), _krkc) \
: reg + KR2idx(IGET_RKC(ins)))
#else
#define RKB() ((isKB(ins) ? ktab : reg) + KR2idx(IGET_RKB(ins))) /* Get value of register or constant B */
#define RKC() ((isKC(ins) ? ktab : reg) + KR2idx(IGET_RKC(ins))) /* Get value of register or constant C */
#endif
#define var2cl(_v) cast(bclosure*, var_toobj(_v)) /* cast var to closure */
#define var2real(_v) (var_isreal(_v) ? (_v)->v.r : (breal)(_v)->v.i) /* get var as real or convert to real if integer */
@@ -594,13 +607,25 @@ BERRY_API void be_vm_delete(bvm *vm)
static void vm_exec(bvm *vm)
{
bclosure *clos;
bvalue *ktab, *reg;
bvalue *reg;
#if BE_USE_COMPACT_KTAB
const union bvaldata *kval; /* current constant payload words */
const bbyte *ktype; /* current constant type bytes */
bvalue _krkb, _krkc; /* scratch for materialized constant operands B/C */
#else
bvalue *ktab;
#endif
binstruction ins;
vm->cf->status |= BASE_FRAME;
newframe: /* a new call frame */
be_assert(var_isclosure(vm->cf->func));
clos = var_toobj(vm->cf->func); /* `clos` is the current function/closure */
#if BE_USE_COMPACT_KTAB
kval = clos->proto->kval; /* current constant payload words */
ktype = clos->proto->ktype; /* current constant type bytes */
#else
ktab = clos->proto->ktab; /* `ktab` is the current constant table */
#endif
reg = vm->reg; /* `reg` is the current stack base for the callframe */
#if BE_USE_PERF_COUNTERS
vm->counter_enter++;
@@ -625,7 +650,13 @@ newframe: /* a new call frame */
}
opcase(LDCONST): {
bvalue *dst = RA();
#if BE_USE_COMPACT_KTAB
int _kidx = IGET_Bx(ins);
dst->v = kval[_kidx];
dst->type = ktype[_kidx];
#else
*dst = ktab[IGET_Bx(ins)];
#endif
dispatch();
}
opcase(GETGBL): {
@@ -979,7 +1010,11 @@ newframe: /* a new call frame */
dispatch();
}
opcase(CLASS): {
#if BE_USE_COMPACT_KTAB
bclass *c = (bclass*)kval[IGET_Bx(ins)].p;
#else
bclass *c = var_toobj(ktab + IGET_Bx(ins));
#endif
be_class_upvalue_init(vm, c);
dispatch();
}
lib/libesp32/berry/src/berry.h
+52
View File
@@ -523,6 +523,31 @@ typedef bclass_ptr bclass_array[];
* @brief define bproto
*
*/
#if BE_USE_COMPACT_KTAB
#define be_define_local_proto(_name, _nstack, _argc, _is_const, _is_subproto, _is_upval) \
static const bproto _name##_proto = { \
NULL, /**< bgcobject *next */ \
BE_PROTO, /**< type BE_PROTO */ \
0x08, /**< marked outside of GC */ \
(_nstack), /**< nstack */ \
BE_IIF(_is_upval)(sizeof(_name##_upvals)/sizeof(bupvaldesc),0), /**< nupvals */ \
(_argc), /**< argc */ \
0, /**< varg */ \
sizeof(_name##_code)/sizeof(uint32_t), /**< codesize */ \
BE_IIF(_is_const)(sizeof(_name##_ktype)/sizeof(bbyte),0), /**< nconst */ \
BE_IIF(_is_subproto)(sizeof(_name##_subproto)/sizeof(bproto*),0), /**< proto */ \
NULL, /**< bgcobject *gray */ \
BE_IIF(_is_upval)((bupvaldesc*)&_name##_upvals,NULL), /**< bupvaldesc *upvals */ \
BE_IIF(_is_const)((union bvaldata*)&_name##_kval,NULL), /**< kval */ \
BE_IIF(_is_const)((bbyte*)&_name##_ktype,NULL), /**< ktype */ \
BE_IIF(_is_subproto)((struct bproto**)&_name##_subproto,NULL), /**< bproto **ptab */ \
(binstruction*) &_name##_code, /**< code */ \
be_local_const_str(_name##_str_name), /**< name */ \
PROTO_SOURCE_FILE_STR(_name) /**< source */ \
PROTO_RUNTIME_BLOCK /**< */ \
PROTO_VAR_INFO_BLOCK /**< */ \
}
#else
#define be_define_local_proto(_name, _nstack, _argc, _is_const, _is_subproto, _is_upval) \
static const bproto _name##_proto = { \
NULL, /**< bgcobject *next */ \
@@ -545,12 +570,38 @@ typedef bclass_ptr bclass_array[];
PROTO_RUNTIME_BLOCK /**< */ \
PROTO_VAR_INFO_BLOCK /**< */ \
}
#endif
/**
* @def be_nested_proto
* @brief new version for more compact literals
*
*/
#if BE_USE_COMPACT_KTAB
#define be_nested_proto(_nstack, _argc, _varg, _has_upval, _upvals, _has_subproto, _protos, _has_const, _kval, _ktype, _fname, _source, _code) \
& (const bproto) { \
NULL, /**< bgcobject *next */ \
BE_PROTO, /**< type BE_PROTO */ \
0x08, /**< marked outside of GC */ \
(_nstack), /**< nstack */ \
BE_IIF(_has_upval)(sizeof(*_upvals)/sizeof(bupvaldesc),0), /**< nupvals */ \
(_argc), /**< argc */ \
(_varg), /**< varg */ \
sizeof(*_code)/sizeof(binstruction), /**< codesize */ \
BE_IIF(_has_const)(sizeof(*_ktype)/sizeof(bbyte),0), /**< nconst (from type array) */ \
BE_IIF(_has_subproto)(sizeof(*_protos)/sizeof(bproto*),0), /**< proto */ \
NULL, /**< bgcobject *gray */ \
(bupvaldesc*) _upvals, /**< bupvaldesc *upvals */ \
(union bvaldata*) _kval, /**< kval */ \
(bbyte*) _ktype, /**< ktype */ \
(struct bproto**) _protos, /**< bproto **ptab */ \
(binstruction*) _code, /**< code */ \
((bstring*) _fname), /**< name */ \
PROTO_SOURCE_FILE(_source) /**< source */ \
PROTO_RUNTIME_BLOCK /**< */ \
PROTO_VAR_INFO_BLOCK /**< */ \
}
#else
#define be_nested_proto(_nstack, _argc, _varg, _has_upval, _upvals, _has_subproto, _protos, _has_const, _ktab, _fname, _source, _code) \
& (const bproto) { \
NULL, /**< bgcobject *next */ \
@@ -573,6 +624,7 @@ typedef bclass_ptr bclass_array[];
PROTO_RUNTIME_BLOCK /**< */ \
PROTO_VAR_INFO_BLOCK /**< */ \
}
#endif
/**
* @def be_define_local_closure
lib/libesp32/berry/tests/list_mutate_iter.be
+33
View File
@@ -0,0 +1,33 @@
# Test: mutating a list during iteration should not segfault
# This reproduces the bug where pushing to a list during a for loop
# caused a stale pointer dereference due to list reallocation.
var entities = []
class Entity
var x, y
def init(x, y)
self.x = x
self.y = y
end
def update()
entities.push(Entity(10, 10))
end
end
entities.push(Entity(1, 2))
var count = 0
for e : entities
assert(type(e) == 'instance')
assert(classname(e) == 'Entity')
e.update()
count += 1
if count > 10
break
end
end
assert(count > 1)
# The list should have grown from the pushes during iteration
assert(entities.size() > 1)
lib/libesp32/berry/tests/property/test_solidification_output.py
+106 -3
View File
@@ -383,6 +383,7 @@ def test_constants_in_output(source):
"""Integer and string constants in the output should match the proto."""
from berry_port.be_object import BE_INT, BE_STRING, var_toint, var_tostr
from berry_port.be_string import be_str2cstr
from berry_port.berry_conf import BE_USE_COMPACT_KTAB
vm, clo, proto = _compile_to_inner_closure(source)
@@ -396,7 +397,11 @@ def test_constants_in_output(source):
continue
if var_type(k) == BE_INT:
val = var_toint(k)
assert f"be_const_int({val})" in output
# compact mode emits be_kv_int(...), legacy emits be_const_int(...)
if BE_USE_COMPACT_KTAB:
assert f"be_kv_int({val})" in output
else:
assert f"be_const_int({val})" in output
elif var_type(k) == BE_STRING:
s = be_str2cstr(var_tostr(k))
from berry_port.be_solidifylib import toidentifier
@@ -408,6 +413,10 @@ def test_constants_in_output(source):
# Property 8f: Cross-validation with C binary — closure solidification
# ============================================================================
# A string literal of >=255 chars triggers the "long string" path: legacy
# solidify emits be_nested_str_long(...), compact emits be_kv_str_long(...).
_LONG_STR = "Z" * 260
_CLOSURE_TEST_VECTORS = [
("simple return int", "def f()\n return 42\nend", True),
("simple return nil", "def f()\n return nil\nend", True),
@@ -424,6 +433,10 @@ _CLOSURE_TEST_VECTORS = [
("string concat", "def f()\n return 'foo' + 'bar'\nend", True),
("simple return int (no weak)", "def f()\n return 42\nend", False),
("unary add (no weak)", "def f(x)\n return x + 1\nend", False),
("long string weak",
'def f()\n return "' + _LONG_STR + '"\nend', True),
("long string non-weak",
'def f()\n return "' + _LONG_STR + '"\nend', False),
]
@@ -530,7 +543,13 @@ def test_class_compaction_shared_ktab():
except Exception as e:
import pytest
pytest.skip(f"Class solidification failed: {e}")
assert "be_ktab_class_C" in out
from berry_port.berry_conf import BE_USE_COMPACT_KTAB
if BE_USE_COMPACT_KTAB:
# compact mode emits split value/type arrays
assert "be_kval_class_C" in out
assert "be_ktype_class_C" in out
else:
assert "be_ktab_class_C" in out
assert "/* shared constants */" in out
assert "compact class 'C'" in out
@@ -565,6 +584,85 @@ def test_class_compaction_deduplicates_long_strings():
assert "ktab size: 1, total: 2" in out
# ============================================================================
# Property 8o: Long strings (>=255) in a function ktab
# ============================================================================
def test_long_string_closure_macro():
"""A string constant >=255 chars must use the 'long string' macro:
be_kv_str_long(...) in compact mode, be_nested_str_long(...) in legacy.
In both cases the encoded string must appear and the ktab entry's type
must remain BE_STRING."""
from berry_port.berry_conf import BE_USE_COMPACT_KTAB
source = 'def f()\n return "' + _LONG_STR + '"\nend'
out = _compile_and_solidify_python(source, str_literal=True)
# the long string content appears as an identifier
assert _LONG_STR in out
if BE_USE_COMPACT_KTAB:
# long strings use be_kv_str_long(...) so coc registers a bclstring
assert "be_kv_str_long(%s)" % _LONG_STR in out
# must NOT be emitted as a short strong/weak reference
assert "be_kv_str(%s)" % _LONG_STR not in out
assert "be_kv_str_weak(%s)" % _LONG_STR not in out
# the parallel type array still tags it as a string
assert "BE_STRING" in out
else:
assert "be_nested_str_long(%s)" % _LONG_STR in out
def test_long_string_closure_matches_c():
"""Cross-validate long-string closure solidification against the C binary.
Proves the chosen long-string macro (be_kv_str_long / be_nested_str_long)
matches whatever the C build emits in the same configuration."""
import pytest
if not _c_binary_available():
pytest.skip("C binary not available")
source = 'def f()\n return "' + _LONG_STR + '"\nend'
for str_literal in (True, False):
py_out = _compile_and_solidify_python(source, str_literal=str_literal)
c_out = _solidify_with_c_binary(source, str_literal=str_literal)
if c_out is None:
pytest.skip("C binary solidification failed")
assert (_strip_trailing_whitespace(py_out) ==
_strip_trailing_whitespace(c_out)), (
"Long-string mismatch (str_literal=%s):\n--- Python ---\n%s\n"
"--- C ---\n%s" % (str_literal, py_out, c_out))
def test_long_string_shared_ktab():
"""A long string (>=255) shared across class methods must be deduplicated
into a single shared-ktab entry and use the long-string macro in compact
mode (be_kv_str_long inside be_kval_class_C)."""
import pytest
from berry_port.berry_conf import BE_USE_COMPACT_KTAB
source = (
"class C\n"
f' def f1() return "{_LONG_STR}" end\n'
f' def f2() return "{_LONG_STR}" end\n'
"end"
)
try:
out = _compile_class_and_solidify_python(source, str_literal=True)
except Exception as e:
pytest.skip(f"Class solidification failed: {e}")
assert "compact class 'C'" in out
# deduplicated to a single entry
assert "ktab size: 1, total: 2" in out
# the long string appears exactly once
assert out.count(_LONG_STR) == 1
if BE_USE_COMPACT_KTAB:
assert "be_kval_class_C" in out
assert "be_ktype_class_C" in out
assert "be_kv_str_long(%s)" % _LONG_STR in out
else:
assert "be_ktab_class_C" in out
assert "be_nested_str_long(%s)" % _LONG_STR in out
# ============================================================================
# Properties 8j-8l: Marker comments for empty sections
# ============================================================================
@@ -609,6 +707,7 @@ def test_known_simple_function_output():
"""Verify exact output for a trivial function against known-good output."""
source = "def f()\n return 42\nend"
out = _compile_and_solidify_python(source, str_literal=True)
from berry_port.berry_conf import BE_USE_COMPACT_KTAB
assert "be_local_closure(f," in out
assert "be_nested_proto(" in out
assert "1, /* nstack */" in out
@@ -619,7 +718,11 @@ def test_known_simple_function_output():
assert "0, /* has sup protos */" in out
assert "NULL, /* no sub protos */" in out
assert "0, /* has constants */" in out
assert "NULL, /* no const */" in out
# compact mode emits two NULLs (kval + ktype), legacy emits one (ktab)
if BE_USE_COMPACT_KTAB:
assert "NULL, NULL, /* no const */" in out
else:
assert "NULL, /* no const */" in out
assert "be_str_weak(f)," in out
assert "&be_const_str_solidified," in out
assert "( &(const binstruction[" in out
lib/libesp32/berry/tools/coc/block_builder.py
+14 -4
View File
@@ -24,6 +24,7 @@ class block_builder:
self.strtab = []
self.strtab_weak = []
self.strtab_long = []
self.macro = macro
self.block.name = obj.name
if depend(obj, macro):
@@ -77,13 +78,22 @@ class block_builder:
entlist = hmap.entry_list()
ostr = ""
compact = self.macro.query("BE_USE_COMPACT_MAP")
ostr += "static be_define_const_map_slots(" + name + ") {\n"
for ent in entlist:
if literal:
ostr += " { be_const_key_weak(" + ent.key + ", "
if compact:
# compact map node: { key, value_type, value_payload }
keymac = "be_ckey_weak" if literal else "be_ckey"
# be_const_xxx(...) -> be_ckv_xxx(...) (type byte + payload)
value = ent.value.replace("be_const_", "be_ckv_", 1)
ostr += " { " + keymac + "(" + ent.key + ", "
ostr += str(ent.next) + "), " + value + " },\n"
else:
ostr += " { be_const_key(" + ent.key + ", "
ostr += str(ent.next) + "), " + ent.value + " },\n"
if literal:
ostr += " { be_const_key_weak(" + ent.key + ", "
else:
ostr += " { be_const_key(" + ent.key + ", "
ostr += str(ent.next) + "), " + ent.value + " },\n"
ostr += "};\n\n"
if local:
lib/libesp32/berry/tools/coc/coc
+1 -1
View File
@@ -25,7 +25,7 @@ class builder:
self.macro = macro_table()
for path in self.config:
self.macro.scan_file(path)
self.macro.scan_file(path)
for d in self.input:
self.scandir(d)
lib/libesp32/berry/tools/coc/coc_parser.py
+6
View File
@@ -32,9 +32,15 @@ class coc_parser:
"be_const_key(": self.parse_string,
"be_nested_str(": self.parse_string,
"be_const_key_weak(": self.parse_string_weak,
"be_ckey(": self.parse_string,
"be_ckey_weak(": self.parse_string_weak,
"be_kv_bytes_instance(": self.parse_bin,
"be_nested_str_weak(": self.parse_string_weak,
"be_nested_str_long(": self.parse_string_long,
"be_str_weak(": self.parse_string_weak,
"be_kv_str(": self.parse_string,
"be_kv_str_weak(": self.parse_string_weak,
"be_kv_str_long(": self.parse_string_long,
}
while len(self.text) > 0:
lib/libesp32/berry_mapping/src/be_mapping.h
+10
View File
@@ -47,6 +47,11 @@
bvaldata((const void*) &ctype_func_def##_f), \
BE_CTYPE_FUNC | BE_STATIC \
}
// compact map node value (BE_USE_COMPACT_MAP): emits "<type byte>, <payload>"
#define be_ckv_ctype_func(_f) \
BE_CTYPE_FUNC, bvaldata((const void*) &ctype_func_def##_f)
#define be_ckv_static_ctype_func(_f) \
BE_CTYPE_FUNC | BE_STATIC, bvaldata((const void*) &ctype_func_def##_f)
#else // __cplusplus
typedef const void* be_constptr;
#define be_const_ctype_func(_f) { \
@@ -57,6 +62,11 @@ typedef const void* be_constptr;
.v.nf = (const void*) &ctype_func_def##_f, \
.type = BE_CTYPE_FUNC | BE_STATIC \
}
// compact map node value (BE_USE_COMPACT_MAP): emits "<type byte>, <payload>"
#define be_ckv_ctype_func(_f) \
BE_CTYPE_FUNC, { .nf = (const void*) &ctype_func_def##_f }
#define be_ckv_static_ctype_func(_f) \
BE_CTYPE_FUNC | BE_STATIC, { .nf = (const void*) &ctype_func_def##_f }
#endif // __cplusplus
#define BE_FUNC_CTYPE_DECLARE(_name, _ret_arg, _in_arg) \
lib/libesp32/berry_tasmota/include/be_ctypes.h
+14
View File
@@ -71,6 +71,19 @@ static void ctypes_register_class(bvm *vm, const bclass * ctypes_class) {
}
// Define a sub-class of ctypes with only one member which points to the ctypes defintion
#if BE_USE_COMPACT_MAP
// Compact map node layout (BE_USE_COMPACT_MAP): { key, value_type, value_payload }
#define be_define_ctypes_class(_c_name, _def, _super, _name) \
be_local_class(_c_name, \
0, \
_super, \
be_nested_map(1, \
( (struct bmapnode*) &(const bmapnodec[]) { \
{ be_ckey_nested("_def", 1985022181, 4, -1), be_ckv_comptr(_def) },\
})), \
(be_nested_const_str(_name, 0, sizeof(_name)-1)) \
)
#else
#define be_define_ctypes_class(_c_name, _def, _super, _name) \
be_local_class(_c_name, \
0, \
@@ -81,6 +94,7 @@ static void ctypes_register_class(bvm *vm, const bclass * ctypes_class) {
})), \
(be_nested_const_str(_name, 0, sizeof(_name)-1)) \
)
#endif
// list of simple classes, sorted
typedef struct be_ctypes_class_by_name_t {