WIP: Use ref tables for Lua callbacks, disable JIT

JIT needs a memory allocator that can guarantee allocations for mcode
that fall within ARM branchable range.

Disable until global system allocator is improved.

include/mt32pi.h

@@ -112,6 +112,7 @@ private:
 	virtual void OnUDPMIDIDataReceived(const u8* pData, size_t nSize) override { ParseMIDIBytes(pData, nSize); };
 
 	// Initialization
+	bool InitLua();
 	bool InitNetwork();
 	bool InitMT32Synth();
 	bool InitSoundFontSynth();
@@ -225,15 +226,20 @@ private:
 
 	// Scripting engine
 	lua_State* m_pLuaState;
+	int m_nLuaMIDIShortMessageFilterTable;
+	int m_nLuaMIDISysExMessageFilterTable;
 
 	static int LuaPanicHandler(lua_State* pLuaState);
 	static int LuaPrint(lua_State* pLuaState);
 	static int LuaLCDLog(lua_State* pLuaState);
 	static int LuaRegisterMIDIShortMessageFilter(lua_State* pLuaState);
+	static int LuaRegisterMIDISysExMessageFilter(lua_State* pLuaState);
 	static int LuaSetMasterVolume(lua_State* pLuaState);
 	static int LuaSendMIDIShortMessage(lua_State* pLuaState);
 	static int LuaSendMIDISysExMessage(lua_State* pLuaState);
 
+	static void RegisterLuaCallback(lua_State* pLuaState, int nTableRef);
+
 	static void EventHandler(const TEvent& Event);
 	static void USBMIDIDeviceRemovedHandler(CDevice* pDevice, void* pContext);
 	static void USBMIDIPacketHandler(unsigned nCable, u8* pPacket, unsigned nLength);

patches/luajit-2.1-circle.patch

@@ -1,27 +1,37 @@
 diff --git a/src/lj_mcode.c b/src/lj_mcode.c
-index 163aada4..c456da00 100644
+index 163aada4..b2a01e95 100644
 --- a/src/lj_mcode.c
 +++ b/src/lj_mcode.c
-@@ -124,6 +124,24 @@ static int mcode_setprot(void *p, size_t sz, int prot)
+@@ -124,6 +124,34 @@ static int mcode_setprot(void *p, size_t sz, int prot)
    return mprotect(p, sz, prot);
  }
  
 +#elif LJ_TARGET_BAREMETAL
-+#define MCPROT_RW	(1 << 0)
-+#define MCPROT_RX	(1 << 1)
-+#define MCPROT_RWX	(1 << 2)
++extern int luadebugprint(const char *fmt, ...);
++
++/* Fallback allocator. This will fail if memory is not executable by default. */
++#define MCPROT_RW	0
++#define MCPROT_RX	0
++#define MCPROT_RWX	0
 +
 +static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot)
 +{
-+	return NULL;
++  UNUSED(hint); UNUSED(prot);
++  //void* r = lj_mem_new(J->L, sz);
++  //luadebugprint("Alloc'd %d bytes, hint 0x%08X, got 0x%08X", sz, hint, (uintptr_t)r);
++  //return r;
++  return lj_mem_new(J->L, sz);
 +}
 +
 +static void mcode_free(jit_State *J, void *p, size_t sz)
 +{
++  //luadebugprint("Free'd %d bytes", sz);
++  lj_mem_free(J2G(J), p, sz);
 +}
 +
 +static int mcode_setprot(void *p, size_t sz, int prot)
 +{
++  UNUSED(p); UNUSED(sz); UNUSED(prot);
 +  return 0;
 +}
  #else

sdcard/lua/main.lua

@@ -8,7 +8,7 @@ local function SC55Compat(status, data1, data2)
 		-- CC #5 - Portamento Time MSB
 		if data1 == 0x05 then
 			-- Convert to LSB message
-			-- mt32pi.LCDLog(3, string.format("%02X %02X %02X -> 0x25 %02X %02X", status, data1, data2,  data1, data2))
+			mt32pi.LCDLog(3, string.format("%02X %02X %02X -> 0x25 %02X %02X", status, data1, data2,  data1, data2))
 			data1 = 0x25
 			return true, status, data1, data2
 

src/mt32pi.cpp

@@ -40,6 +40,8 @@ const char MT32PiFullName[] = MT32_PI_NAME " " MT32_PI_VERSION;
 const char WLANFirmwarePath[] = "SD:firmware/";
 const char WLANConfigFile[]   = "SD:wpa_supplicant.conf";
 
+const char LuaMainScript[] = "SD:lua/main.lua";
+
 constexpr u32 LCDUpdatePeriodMillis                = 16;
 constexpr u32 MisterUpdatePeriodMillis             = 50;
 constexpr u32 LEDTimeoutMillis                     = 50;
@@ -62,15 +64,13 @@ const struct luaL_Reg CMT32Pi::s_LuaLib[] =
 {
 	{ "LCDLog",				LuaLCDLog				},
 	{ "RegisterMIDIShortMessageFilter",	LuaRegisterMIDIShortMessageFilter 	},
+	{ "RegisterMIDISysExMessageFilter",	LuaRegisterMIDISysExMessageFilter 	},
 	{ "SetMasterVolume",			LuaSetMasterVolume			},
 	{ "SendMIDIShortMessage",		LuaSendMIDIShortMessage			},
 	{ "SendMIDISysExMessage",		LuaSendMIDISysExMessage			},
 	{ nullptr,				nullptr					},
 };
 
-int LuaFilters[10];
-size_t nLuaFilters = 0;
-
 CMT32Pi::CMT32Pi(CI2CMaster* pI2CMaster, CSPIMaster* pSPIMaster, CInterruptSystem* pInterrupt, CGPIOManager* pGPIOManager, CSerialDevice* pSerialDevice, CUSBHCIDevice* pUSBHCI)
 	: CMultiCoreSupport(CMemorySystem::Get()),
 	  CMIDIParser(),
@@ -135,7 +135,9 @@ CMT32Pi::CMT32Pi(CI2CMaster* pI2CMaster, CSPIMaster* pSPIMaster, CInterruptSyste
 	  m_pMT32Synth(nullptr),
 	  m_pSoundFontSynth(nullptr),
 
-	  m_pLuaState(nullptr)
+	  m_pLuaState(nullptr),
+	  m_nLuaMIDIShortMessageFilterTable(0),
+	  m_nLuaMIDISysExMessageFilterTable(0)
 {
 	s_pThis = this;
 }
@@ -322,25 +324,8 @@ bool CMT32Pi::Initialize(bool bSerialMIDIAvailable)
 	SetPowerSaveTimeout(m_pConfig->SystemPowerSaveTimeout);
 
 	// Init scripting engine
-	m_pLuaState = luaL_newstate();
-	luaL_openlibs(m_pLuaState);
-
-	// Override print() and panic handler
-	lua_atpanic(m_pLuaState, LuaPanicHandler);
-	lua_register(m_pLuaState, "print", LuaPrint);
-
-	// Register mt32-pi API
-	luaL_newlib(m_pLuaState, s_LuaLib);
-	lua_setglobal(m_pLuaState, "mt32pi");
-
-	if (luaL_dofile(m_pLuaState, "SD:mt32-pi.lua") == LUA_OK)
-		LOGNOTE("Loaded Lua script");
-	else
-	{
-		LOGNOTE("Lua script not loaded: %s", lua_tostring(m_pLuaState, -1));
-		lua_close(m_pLuaState);
-		m_pLuaState = nullptr;
-	}
+	LCDLog(TLCDLogType::Startup, "Init Lua");
+	InitLua();
 
 	// Clear LCD
 	if (m_pLCD)
@@ -356,6 +341,41 @@ bool CMT32Pi::Initialize(bool bSerialMIDIAvailable)
 	return true;
 }
 
+bool CMT32Pi::InitLua()
+{
+	assert(m_pLuaState == nullptr);
+
+	m_pLuaState = luaL_newstate();
+	luaL_openlibs(m_pLuaState);
+
+	// Override panic handler and print()
+	lua_atpanic(m_pLuaState, LuaPanicHandler);
+	lua_register(m_pLuaState, "print", LuaPrint);
+
+	// Register mt32-pi API
+	luaL_newlib(m_pLuaState, s_LuaLib);
+	lua_setglobal(m_pLuaState, "mt32pi");
+
+	// Create tables for storing callbacks
+	lua_newtable(m_pLuaState);
+	m_nLuaMIDIShortMessageFilterTable = luaL_ref(m_pLuaState, LUA_REGISTRYINDEX);
+	lua_newtable(m_pLuaState);
+	m_nLuaMIDISysExMessageFilterTable = luaL_ref(m_pLuaState, LUA_REGISTRYINDEX);
+	lua_pop(m_pLuaState, 2);
+
+	// Run init.lua
+	if (luaL_dofile(m_pLuaState, LuaMainScript) == LUA_OK)
+		LOGNOTE("Loaded main Lua script");
+	else
+	{
+		LOGNOTE("Lua script not loaded: %s", lua_tostring(m_pLuaState, -1));
+		lua_close(m_pLuaState);
+		m_pLuaState = nullptr;
+	}
+
+	return m_pLuaState != nullptr;
+}
+
 bool CMT32Pi::InitNetwork()
 {
 	assert(m_pNet == nullptr);
@@ -701,50 +721,86 @@ void CMT32Pi::OnUnderVoltageDetected()
 	LCDLog(TLCDLogType::Warning, "Low voltage! Chk PSU");
 }
 
+// extern "C" {
+// int luadebugprint(const char* pFormat, ...)
+// {
+// 	char Buffer[256];
+
+// 	va_list Args;
+// 	va_start(Args, pFormat);
+// 	vsnprintf(Buffer, sizeof(Buffer), pFormat, Args);
+// 	va_end(Args);
+
+// 	CLogger::Get()->Write("luadebugprint", LogDebug, Buffer);
+
+// 	return 0;
+// }
+// }
+
 void CMT32Pi::OnShortMessage(u32 nMessage)
 {
 	// Filter using scripting engine
 	if (m_pLuaState)
 	{
-		for (size_t i = 0; i < nLuaFilters; ++i)
+		// Push callback table
+		lua_rawgeti(m_pLuaState, LUA_REGISTRYINDEX, m_nLuaMIDIShortMessageFilterTable);
+		assert(lua_istable(m_pLuaState, -1));
+
+		// Push placeholder key
+		lua_pushnil(m_pLuaState);
+
+		while (lua_next(m_pLuaState, -2))
 		{
-			lua_rawgeti(m_pLuaState, LUA_REGISTRYINDEX, LuaFilters[i]);
+			assert(lua_isfunction(m_pLuaState, -1));
 
-			// Ensure function exists
-			if (!lua_isfunction(m_pLuaState, 1))
-				lua_pop(m_pLuaState, 1);
-			else
+			const int nStackSize = lua_gettop(m_pLuaState);
+
+			lua_pushinteger(m_pLuaState, nMessage & 0xFF);
+			lua_pushinteger(m_pLuaState, (nMessage >> 8) & 0xFF);
+			lua_pushinteger(m_pLuaState, (nMessage >> 16) & 0xFF);
+
+			if (lua_pcall(m_pLuaState, 3, LUA_MULTRET, 0) != LUA_OK)
 			{
-				const int nStackSize = lua_gettop(m_pLuaState);
+				const char* const pFormat =  "Lua error: %s";
+				const char* const pMessage = lua_tostring(m_pLuaState, -1);
 
-				lua_pushinteger(m_pLuaState, nMessage & 0xFF);
-				lua_pushinteger(m_pLuaState, (nMessage >> 8) & 0xFF);
-				lua_pushinteger(m_pLuaState, (nMessage >> 16) & 0xFF);
+				// Pop error object and table value
+				lua_pop(m_pLuaState, 2);
 
-				if (lua_pcall(m_pLuaState, 3, LUA_MULTRET, 0) != LUA_OK)
-				{
-					const char* const pFormat =  "Lua error: %s";
-					const char* const pMessage = lua_tostring(m_pLuaState, -1);
-					lua_pop(m_pLuaState, 1);
+				LOGERR(pFormat, pMessage);
+				LCDLog(TLCDLogType::Error, pFormat, pMessage);
+				continue;
+			}
 
-					LOGERR(pFormat, pMessage);
-					LCDLog(TLCDLogType::Error, pFormat, pMessage);
-					return;
-				}
+			const int nReturns = lua_gettop(m_pLuaState) - nStackSize;
+			bool nConsumed = false;
 
-				const int nReturns = lua_gettop(m_pLuaState) - nStackSize;
+			if (nReturns)
+			{
+				nConsumed = lua_toboolean(m_pLuaState, -1);
 
 				// If function returned a MIDI message, update it
-				if (nReturns == 3)
+				if (nReturns == 4)
 				{
-					nMessage = (luaL_checkinteger(m_pLuaState, -3) & 0xFF) |
-						((luaL_checkinteger(m_pLuaState, -2) & 0xFF) << 8) |
-						((luaL_checkinteger(m_pLuaState, -1) & 0xFF) << 16);
+					nMessage = (luaL_checkinteger(m_pLuaState, -4) & 0xFF) |
+						((luaL_checkinteger(m_pLuaState, -3) & 0xFF) << 8) |
+						((luaL_checkinteger(m_pLuaState, -2) & 0xFF) << 16);
 				}
+			}
 
-				lua_pop(m_pLuaState, nReturns);
+			// Pop callback return values and table value
+			lua_pop(m_pLuaState, nReturns + 1);
+
+			// Filter consumed the message, pop table key and break
+			if (nConsumed)
+			{
+				lua_pop(m_pLuaState, 1);
+				break;
 			}
 		}
+
+		// Pop table
+		lua_pop(m_pLuaState, 1);
 	}
 
 	// Active sensing
@@ -769,30 +825,40 @@ void CMT32Pi::OnSysExMessage(const u8* pData, size_t nSize)
 	// Filter using scripting engine
 	if (m_pLuaState)
 	{
-		// Ensure function exists
-		lua_getglobal(m_pLuaState, "OnMIDISysExMessage");
+		// Push callback table
+		lua_rawgeti(m_pLuaState, LUA_REGISTRYINDEX, m_nLuaMIDISysExMessageFilterTable);
+		assert(lua_istable(m_pLuaState, -1));
 
-		if (!lua_isfunction(m_pLuaState, 1))
-			lua_pop(m_pLuaState, 1);
-		else
+		// Push placeholder key
+		lua_pushnil(m_pLuaState);
+
+		while (lua_next(m_pLuaState, -2))
 		{
+			assert(lua_isfunction(m_pLuaState, -1));
 			lua_pushlstring(m_pLuaState, reinterpret_cast<const char*>(pData), nSize);
 
 			if (lua_pcall(m_pLuaState, 1, 1, 0) != LUA_OK)
 			{
 				const char* const pFormat =  "Lua error: %s";
 				const char* const pMessage = lua_tostring(m_pLuaState, -1);
-				lua_pop(m_pLuaState, 1);
+
+				// Pop error object and table value
+				lua_pop(m_pLuaState, 2);
 
 				LOGERR(pFormat, pMessage);
 				LCDLog(TLCDLogType::Error, pFormat, pMessage);
-				return;
+				continue;
 			}
 
 			// If function returned a SysEx message, update it
 			pData = reinterpret_cast<const u8*>(lua_tolstring(m_pLuaState, -1, &nSize));
-			lua_pop(m_pLuaState, 1);
+
+			// Pop table value and callback return value
+			lua_pop(m_pLuaState, 2);
 		}
+
+		// Pop table
+		lua_pop(m_pLuaState, 1);
 	}
 
 	if (!pData || !nSize)
@@ -1432,7 +1498,8 @@ void CMT32Pi::IRQMIDIReceiveHandler(const u8* pData, size_t nSize)
 
 int CMT32Pi::LuaPanicHandler(lua_State* pLuaState)
 {
-	LOGPANIC("Lua error: %s", lua_tostring(pLuaState, -1));
+	LOGERR("Lua panic: %s", lua_tostring(pLuaState, -1));
+	assert(false);
 	return 0;
 }
 
@@ -1486,17 +1553,33 @@ int CMT32Pi::LuaSetMasterVolume(lua_State* pLuaState)
 
 int CMT32Pi::LuaRegisterMIDIShortMessageFilter(lua_State* pLuaState)
 {
-	if (!lua_isfunction(pLuaState, 1))
+	RegisterLuaCallback(pLuaState, s_pThis->m_nLuaMIDIShortMessageFilterTable);
+	return 0;
+}
+
+int CMT32Pi::LuaRegisterMIDISysExMessageFilter(lua_State* pLuaState)
+{
+	RegisterLuaCallback(pLuaState, s_pThis->m_nLuaMIDISysExMessageFilterTable);
+	return 0;
+}
+
+void CMT32Pi::RegisterLuaCallback(lua_State* pLuaState, int nTableRef)
+{
+	if (!lua_isfunction(pLuaState, -1))
 	{
-		LOGERR("RegisterMIDIShortMessageFilter(): argument is not a function");
-		return 0;
+		LOGERR("RegisterLuaCallback(): argument is not a function");
+		return;
 	}
 
-	int nRef = luaL_ref(pLuaState, LUA_REGISTRYINDEX);
-	LuaFilters[nLuaFilters++] = nRef;
+	// Get callback table from registry
+	lua_rawgeti(pLuaState, LUA_REGISTRYINDEX, nTableRef);
 
-	LOGNOTE("Registered Lua function %d", nRef);
-	return 0;
+	// Copy callback function to top of stack and store a reference in the table
+	lua_pushvalue(pLuaState, -2);
+	int nRef = luaL_ref(pLuaState, -2);
+	lua_pop(pLuaState, 1);
+
+	LOGNOTE("Registered Lua callback %d", nRef);
 }
 
 int CMT32Pi::LuaSendMIDIShortMessage(lua_State* pLuaState)