Add support for MPY ABI v6.

Pybricks firmware v3.2.0b2 has updated to MicroPython 1.19 which
includes breaking changes to the MPY binary file format.

package.json

@@ -15,6 +15,7 @@
         "@pybricks/firmware": "4.15.0",
         "@pybricks/ide-docs": "2.2.0",
         "@pybricks/mpy-cross-v5": "^2.0.0",
+        "@pybricks/mpy-cross-v6": "^1.0.0",
         "@reduxjs/toolkit": "^1.8.3",
         "@shopify/react-i18n": "^7.1.3",
         "@svgr/webpack": "^6.2.1",

src/editor/Editor.tsx

@@ -392,10 +392,9 @@ const Editor: React.VFC = () => {
         () => ({
             id: 'pybricks.action.check',
             label: i18n.translate(I18nId.Check),
-            // REVISIT: the compile options here might need to be changed - hopefully there is
-            // one setting that works for all hub types for cases where we aren't connected.
             run: (e) => {
-                dispatch(compile(e.getValue(), []));
+                // for checking, use the most recent compiler
+                dispatch(compile(e.getValue(), 6, []));
             },
             keybindings: [monaco.KeyCode.F2],
         }),

src/firmware/sagas.test.ts

@@ -107,6 +107,7 @@ describe('flashFirmware', () => {
             action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -622,6 +623,7 @@ describe('flashFirmware', () => {
             action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -767,6 +769,7 @@ describe('flashFirmware', () => {
             action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -861,6 +864,7 @@ describe('flashFirmware', () => {
             action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -964,6 +968,7 @@ describe('flashFirmware', () => {
             action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -1107,6 +1112,7 @@ describe('flashFirmware', () => {
             action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -1236,6 +1242,7 @@ describe('flashFirmware', () => {
             let action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -1477,6 +1484,7 @@ describe('flashFirmware', () => {
             let action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -1536,6 +1544,7 @@ describe('flashFirmware', () => {
             let action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -1596,6 +1605,7 @@ describe('flashFirmware', () => {
             let action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -1659,6 +1669,7 @@ describe('flashFirmware', () => {
             let action = await saga.take();
             expect(action).toMatchInlineSnapshot(`
                 Object {
+                  "abiVersion": 5,
                   "options": Array [
                     "-mno-unicode",
                   ],
@@ -1759,6 +1770,7 @@ describe('flashFirmware', () => {
         action = await saga.take();
         expect(action).toMatchInlineSnapshot(`
             Object {
+              "abiVersion": 5,
               "options": Array [
                 "-mno-unicode",
               ],

src/firmware/sagas.ts

@@ -207,7 +207,9 @@ function* loadFirmware(
         yield* disconnectAndCancel();
     }
 
-    yield* put(compile(program, metadata['mpy-cross-options']));
+    yield* put(
+        compile(program, metadata['mpy-abi-version'], metadata['mpy-cross-options']),
+    );
     const { mpy, mpyFail } = yield* race({
         mpy: take(didCompile),
         mpyFail: take(didFailToCompile),

src/hub/actions.ts

@@ -13,8 +13,9 @@ export const checksum = createAction((checksum: number) => ({
 
 // High-level hub actions.
 
-export const downloadAndRun = createAction(() => ({
+export const downloadAndRun = createAction((abiVersion: number) => ({
     type: 'hub.action.downloadAndRun',
+    abiVersion,
 }));
 
 export const didStartDownload = createAction(() => ({

src/hub/reducers.test.ts

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2021 The Pybricks Authors
+// Copyright (c) 2021-2022 The Pybricks Authors
 
 import { AnyAction } from 'redux';
 import { didReceiveStatusReport } from '../ble-pybricks-service/actions';
@@ -19,6 +19,7 @@ test('initial state', () => {
     expect(reducers(undefined, {} as AnyAction)).toMatchInlineSnapshot(`
         Object {
           "downloadProgress": null,
+          "mpyAbiVersion": 6,
           "runtime": "hub.runtime.disconnected",
         }
     `);

src/hub/reducers.ts

@@ -2,9 +2,12 @@
 // Copyright (c) 2020-2022 The Pybricks Authors
 
 import { Reducer, combineReducers } from 'redux';
+import * as semver from 'semver';
+import { bleDIServiceDidReceiveFirmwareRevision } from '../ble-device-info-service/actions';
 import { didReceiveStatusReport } from '../ble-pybricks-service/actions';
 import { Status, statusToFlag } from '../ble-pybricks-service/protocol';
 import { didConnect, didDisconnect } from '../ble/actions';
+import { pythonVersionToSemver } from '../utils/version';
 import {
     didFailToFinishDownload,
     didFinishDownload,
@@ -118,4 +121,18 @@ const downloadProgress: Reducer<number | null> = (state = null, action) => {
     return state;
 };
 
-export default combineReducers({ runtime, downloadProgress });
+const mpyAbiVersion: Reducer<number> = (state = 6, action) => {
+    if (bleDIServiceDidReceiveFirmwareRevision.matches(action)) {
+        // HACK: there is not a good way to get the supported MPY ABI version
+        // from a running hub, so we use heuristics on the firmware version.
+        if (semver.satisfies(pythonVersionToSemver(action.version), '>=3.2.0-beta.2')) {
+            return 6;
+        }
+
+        return 5;
+    }
+
+    return state;
+};
+
+export default combineReducers({ runtime, downloadProgress, mpyAbiVersion });

src/hub/sagas.test.ts

@@ -31,7 +31,7 @@ describe('downloadAndRun', () => {
 
         saga.updateState({ editor: { isReady: true } });
 
-        saga.put(downloadAndRun());
+        saga.put(downloadAndRun(5));
 
         // first, it gets the value from the current editor
         const editorValueAction = await saga.take();
@@ -41,7 +41,7 @@ describe('downloadAndRun', () => {
 
         // then it tries to compile the program in the current editor
         const compileAction = await saga.take();
-        expect(compileAction).toEqual(compile('', ['-mno-unicode']));
+        expect(compileAction).toEqual(compile('', 5, ['-mno-unicode']));
         saga.put(didCompile(new Uint8Array(30)));
 
         // then it notifies that loading has begun

src/hub/sagas.ts

@@ -46,10 +46,17 @@ function* waitForWrite(id: number): SagaGenerator<{
     });
 }
 
-function* handleDownloadAndRun(): Generator {
+function* handleDownloadAndRun(action: ReturnType<typeof downloadAndRun>): Generator {
     const script = yield* editorGetValue();
 
-    yield* put(compile(script, ['-mno-unicode']));
+    yield* put(
+        compile(
+            script,
+            action.abiVersion,
+            // no-unicode option was removed in MPY ABI v6
+            action.abiVersion < 6 ? ['-mno-unicode'] : [],
+        ),
+    );
 
     const { mpy, mpyFail } = yield* race({
         mpy: take(didCompile),

src/mpy/actions.ts

@@ -1,14 +1,17 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2020,2022 The Pybricks Authors
+// Copyright (c) 2020-2022 The Pybricks Authors
 
 import { createAction } from '../actions';
 
 /** Action that requests that a script is compiled. */
-export const compile = createAction((script: string, options: string[]) => ({
-    type: 'mpy.action.compile',
-    script,
-    options,
-}));
+export const compile = createAction(
+    (script: string, abiVersion: number, options: string[]) => ({
+        type: 'mpy.action.compile',
+        script,
+        abiVersion,
+        options,
+    }),
+);
 
 export const didCompile = createAction((data: Uint8Array) => ({
     type: 'mpy.action.didCompile',

src/mpy/sagas.test.ts

@@ -1,33 +1,28 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2020,2022 The Pybricks Authors
+// Copyright (c) 2020-2022 The Pybricks Authors
 
 import { AsyncSaga } from '../../test';
 import { compile, didCompile, didFailToCompile } from './actions';
 import mpy from './sagas';
 
-enum MpyFeatureFlags {
-    MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE = 1 << 0,
-    MICROPY_PY_BUILTINS_STR_UNICODE = 1 << 1,
-}
-
 test('compiler works', async () => {
     const saga = new AsyncSaga(mpy);
 
-    saga.put(compile('print("hello!")', []));
+    saga.put(compile('print("hello!")', 6, []));
 
     const action = await saga.take();
     expect(didCompile.matches(action)).toBeTruthy();
     const { data } = action as ReturnType<typeof didCompile>;
     expect(data[0]).toBe('M'.charCodeAt(0));
-    expect(data[1]).toBe(5); // ABI version
-    expect(data[2]).toBe(MpyFeatureFlags.MICROPY_PY_BUILTINS_STR_UNICODE);
+    expect(data[1]).toBe(6); // ABI version
+    expect(data[2]).toBe(0); // flags
     expect(data[3]).toBe(31); // small int bits
 });
 
 test('compiler error works', async () => {
     const saga = new AsyncSaga(mpy);
 
-    saga.put(compile('syntax error!', []));
+    saga.put(compile('syntax error!', 6, []));
 
     const action = await saga.take();
     expect(didFailToCompile.matches(action)).toBeTruthy();

src/mpy/sagas.ts

@@ -1,8 +1,10 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2020,2022 The Pybricks Authors
+// Copyright (c) 2020-2022 The Pybricks Authors
 
-import { compile as mpyCrossCompile } from '@pybricks/mpy-cross-v5';
-import wasm from '@pybricks/mpy-cross-v5/build/mpy-cross.wasm';
+import { compile as mpyCrossCompileV5 } from '@pybricks/mpy-cross-v5';
+import wasmV5 from '@pybricks/mpy-cross-v5/build/mpy-cross.wasm';
+import { compile as mpyCrossCompileV6 } from '@pybricks/mpy-cross-v6';
+import wasmV6 from '@pybricks/mpy-cross-v6/build/mpy-cross.wasm';
 import { call, put, takeEvery } from 'typed-redux-saga/macro';
 import { compile, didCompile, didFailToCompile } from './actions';
 
@@ -12,19 +14,54 @@ import { compile, didCompile, didFailToCompile } from './actions';
  * @param action A mpy compile action.
  */
 function* handleCompile(action: ReturnType<typeof compile>): Generator {
-    const result = yield* call(() =>
-        mpyCrossCompile(
-            'main.py',
-            action.script,
-            action.options,
-            // HACK: testing user agent for jsdom is needed only for getting unit tests to work
-            navigator.userAgent.includes('jsdom') ? undefined : wasm,
-        ),
-    );
-    if (result.status === 0 && result.mpy) {
-        yield* put(didCompile(result.mpy));
-    } else {
-        yield* put(didFailToCompile(result.err));
+    switch (action.abiVersion) {
+        case 5:
+            {
+                const result = yield* call(() =>
+                    mpyCrossCompileV5(
+                        'main.py',
+                        action.script,
+                        action.options,
+                        // HACK: testing user agent for jsdom is needed only for getting unit tests to work
+                        navigator.userAgent.includes('jsdom') ? undefined : wasmV5,
+                    ),
+                );
+                if (result.status === 0 && result.mpy) {
+                    yield* put(didCompile(result.mpy));
+                } else {
+                    yield* put(didFailToCompile(result.err));
+                }
+            }
+            break;
+
+        case 6:
+            {
+                const result = yield* call(() =>
+                    mpyCrossCompileV6(
+                        'main.py',
+                        action.script,
+                        action.options,
+                        // HACK: testing user agent for jsdom is needed only for getting unit tests to work
+                        navigator.userAgent.includes('jsdom') ? undefined : wasmV6,
+                    ),
+                );
+                if (result.status === 0 && result.mpy) {
+                    yield* put(didCompile(result.mpy));
+                } else {
+                    yield* put(didFailToCompile(result.err));
+                }
+            }
+            break;
+
+        default:
+            {
+                yield* put(
+                    didFailToCompile([
+                        `unsupported MPY ABI version: ${action.abiVersion}`,
+                    ]),
+                );
+            }
+            break;
     }
 }
 

src/toolbar/buttons/run/RunButton.test.tsx

@@ -20,5 +20,5 @@ it('should dispatch action when clicked', async () => {
 
     await user.click(button.getByRole('button', { name: 'Run' }));
 
-    expect(dispatch).toHaveBeenCalledWith(downloadAndRun());
+    expect(dispatch).toHaveBeenCalledWith(downloadAndRun(6));
 });

src/toolbar/buttons/run/RunButton.tsx

@@ -14,6 +14,7 @@ type RunButtonProps = Pick<ActionButtonProps, 'id'>;
 
 const RunButton: React.VoidFunctionComponent<RunButtonProps> = ({ id }) => {
     const downloadProgress = useSelector((s) => s.hub.downloadProgress);
+    const mpyAbiVersion = useSelector((s) => s.hub.mpyAbiVersion);
     const runtime = useSelector((s) => s.hub.runtime);
     const isEditorReady = useSelector((s) => s.editor.isReady);
     const keyboardShortcut = 'F5';
@@ -37,7 +38,7 @@ const RunButton: React.VoidFunctionComponent<RunButtonProps> = ({ id }) => {
             enabled={isEditorReady && runtime === HubRuntimeState.Idle}
             showProgress={runtime === HubRuntimeState.Loading}
             progress={downloadProgress === null ? undefined : downloadProgress}
-            onAction={() => dispatch(downloadAndRun())}
+            onAction={() => dispatch(downloadAndRun(mpyAbiVersion))}
         />
     );
 };

yarn.lock

@@ -2283,6 +2283,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@pybricks/mpy-cross-v6@npm:^1.0.0":
+  version: 1.0.0
+  resolution: "@pybricks/mpy-cross-v6@npm:1.0.0"
+  checksum: ff09564d5923665ad836185405e8a1f830890913d82e466440bc7cb90e6b9754aad537d7d55360dbad44cfa6612cce5b593b28b981a4d82f8ed6ec640d42bf5f
+  languageName: node
+  linkType: hard
+
 "@pybricks/pybricks-code@workspace:.":
   version: 0.0.0-use.local
   resolution: "@pybricks/pybricks-code@workspace:."
@@ -2294,6 +2301,7 @@ __metadata:
     "@pybricks/firmware": 4.15.0
     "@pybricks/ide-docs": 2.2.0
     "@pybricks/mpy-cross-v5": ^2.0.0
+    "@pybricks/mpy-cross-v6": ^1.0.0
     "@reduxjs/toolkit": ^1.8.3
     "@shopify/react-i18n": ^7.1.3
     "@svgr/webpack": ^6.2.1