diff --git a/package.json b/package.json index 89a587ef..05f45bfa 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "@blueprintjs/popover2": "^1.6.4", "@blueprintjs/select": "^4.6.4", "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7", - "@pybricks/firmware": "5.0.0", + "@pybricks/firmware": "6.0.1", "@pybricks/ide-docs": "2.2.0", "@pybricks/jedi": "^1.0.1", "@pybricks/mpy-cross-v5": "^2.0.0", diff --git a/src/editor/sagas.ts b/src/editor/sagas.ts index 519bc7e4..ef72fdcb 100644 --- a/src/editor/sagas.ts +++ b/src/editor/sagas.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2022 The Pybricks Authors +import type { DatabaseChangeType, IDatabaseChange } from 'dexie-observable/api'; import { monaco } from 'react-monaco-editor'; import { EventChannel, buffers, eventChannel } from 'redux-saga'; import { @@ -16,7 +17,7 @@ import { take, takeEvery, } from 'typed-redux-saga/macro'; -import { UUID } from '../fileStorage'; +import { FileStorageDb, UUID } from '../fileStorage'; import { fileStorageDidFailToLoadTextFile, fileStorageDidFailToStoreTextFileViewState, @@ -29,15 +30,18 @@ import { } from '../fileStorage/actions'; import { pythonMessageComplete, + pythonMessageDeleteUserFile, pythonMessageDidComplete, pythonMessageDidFailToComplete, pythonMessageDidFailToGetSignature, pythonMessageDidFailToInit, pythonMessageDidGetSignature, pythonMessageDidInit, + pythonMessageDidMountUserFileSystem, pythonMessageGetSignature, pythonMessageInit, pythonMessageSetInterruptBuffer, + pythonMessageWriteUserFile, } from '../pybricksMicropython/python-message'; import { RootState } from '../reducers'; import { acquireLock, defined, ensureError } from '../utils'; @@ -390,6 +394,97 @@ function* monitorEditors(): Generator { } } +// HACK: dexie-observable exports const enum, so we have to redefine values +const DatabaseChangeTypeCreate: DatabaseChangeType.Create = 1; +const DatabaseChangeTypeUpdate: DatabaseChangeType.Update = 2; +const DatabaseChangeTypeDelete: DatabaseChangeType.Delete = 3; + +/** + * Mirrors the Dexie-based file system to the Emscripten file system in the + * Python Web Worker. + * + * @param worker The web worker. + */ +function* mirrorFileSystem(worker: Worker): Generator { + // wait for file storage to become ready if it isn't already + if (!(yield* select((s: RootState) => s.fileStorage.isInitialized))) { + yield take(fileStorageDidInitialize); + } + + const db = yield* getContext('fileStorage'); + + // subscribe to future changes + const dbChangedChan = eventChannel((emit) => { + db.on('changes').subscribe(emit); + return () => db.on('changes').unsubscribe(emit); + }); + + // copy all existing files + yield* call(() => + db.transaction('r', db._contents, () => + db._contents.each((f) => + worker.postMessage(pythonMessageWriteUserFile(f.path, f.contents)), + ), + ), + ); + + // handle future changes + try { + for (;;) { + const changes = yield* take(dbChangedChan); + + for (const c of changes) { + // only interested in metadata table changes + if (c.table !== db.metadata.name) { + continue; + } + + switch (c.type) { + case DatabaseChangeTypeCreate: + case DatabaseChangeTypeUpdate: + // only send message if file was created or contents + // changed - ignore other metadata changes + if ( + c.type === DatabaseChangeTypeUpdate && + c.obj.sha256 === c.oldObj.sha256 + ) { + break; + } + + yield* call(() => + db.transaction('r', db._contents, async () => { + const file = await db._contents.get(c.obj.path); + + // istanbul ignore if: programmer error if we hit this + if (!file) { + console.error( + `could not find file '${c.obj.path}'`, + ); + return; + } + + worker.postMessage( + pythonMessageWriteUserFile( + file.path, + file.contents, + ), + ); + }), + ); + + break; + + case DatabaseChangeTypeDelete: + worker.postMessage(pythonMessageDeleteUserFile(c.oldObj.path)); + break; + } + } + } + } finally { + dbChangedChan.close(); + } +} + /** * Runs a web worker with Pyodide so that we can use Jedi for intellisense. */ @@ -446,6 +541,11 @@ function* runJedi(): Generator { defined(messageEvent); + if (pythonMessageDidMountUserFileSystem.matches(messageEvent.data)) { + yield* fork(mirrorFileSystem, worker); + continue; + } + if (pythonMessageDidFailToInit.matches(messageEvent.data)) { yield* put(editorCompletionDidFailToInit()); throw messageEvent.data.error; diff --git a/src/firmware/actions.ts b/src/firmware/actions.ts index 9d058d08..7a8549bc 100644 --- a/src/firmware/actions.ts +++ b/src/firmware/actions.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2020-2022 The Pybricks Authors -import { FirmwareMetadata, FirmwareReaderError } from '@pybricks/firmware'; +import { FirmwareReaderError } from '@pybricks/firmware'; import { createAction } from '../actions'; export enum MetadataProblem { @@ -86,7 +86,7 @@ export type FailToFinishReasonZipError = Reason export type FailToFinishReasonBadMetadata = Reason & { - property: keyof FirmwareMetadata; + property: string; problem: MetadataProblem; }; @@ -120,15 +120,12 @@ export type FailToFinishReason = /** * Creates a new action to flash firmware to a hub. * @param data The firmware zip file data or `null` to get firmware later. - * @param customProgram If defined, flash the path of a program from file storage, - * otherwise use the main.py program from firmware.zip. * @param hubName A custom hub name or an empty string to use the default name. */ export const flashFirmware = createAction( - (data: ArrayBuffer | null, customProgram: string | undefined, hubName: string) => ({ + (data: ArrayBuffer | null, hubName: string) => ({ type: 'flashFirmware.action.flashFirmware', data, - customProgram, hubName, }), ); @@ -155,6 +152,18 @@ export const didFinish = createAction(() => ({ type: 'flashFirmware.action.didFinish', })); +function isError(err: unknown): err is Error { + const maybeError = err as Error; + + return ( + maybeError !== undefined && + typeof maybeError.name === 'string' && + typeof maybeError.message === 'string' + ); +} + +// FIXME: get rid of this monstrosity + const didFailToFinishType = 'flashFirmware.action.didFailToFinish'; function didFailToFinishCreator(reason: FailToFinishReasonType.FailedToConnect): { @@ -216,7 +225,7 @@ function didFailToFinishCreator( function didFailToFinishCreator( reason: FailToFinishReasonType.BadMetadata, - property: keyof FirmwareMetadata, + property: string, problem: MetadataProblem, ): { type: typeof didFailToFinishType; @@ -260,7 +269,7 @@ function didFailToFinishCreator( } { if (reason === FailToFinishReasonType.BleError) { // istanbul ignore if: programmer error give wrong arg - if (!(arg1 instanceof Error)) { + if (!isError(arg1)) { throw new Error('missing or invalid err'); } return { @@ -312,7 +321,9 @@ function didFailToFinishCreator( arg1 !== 'mpy-abi-version' && arg1 !== 'mpy-cross-options' && arg1 !== 'user-mpy-offset' && - arg1 !== 'max-firmware-size' + arg1 !== 'max-firmware-size' && + arg1 !== 'checksum-size' && + arg1 !== 'hub-name-size' ) { throw new Error('missing or invalid property'); } @@ -328,7 +339,7 @@ function didFailToFinishCreator( if (reason === FailToFinishReasonType.Unknown) { // istanbul ignore if: programmer error give wrong arg - if (!(arg1 instanceof Error)) { + if (!isError(arg1)) { throw new Error('missing or invalid err'); } return { diff --git a/src/firmware/installPybricksDialog/InstallPybricksDialog.tsx b/src/firmware/installPybricksDialog/InstallPybricksDialog.tsx index e2f24aac..e9964f56 100644 --- a/src/firmware/installPybricksDialog/InstallPybricksDialog.tsx +++ b/src/firmware/installPybricksDialog/InstallPybricksDialog.tsx @@ -7,7 +7,6 @@ import { Callout, Checkbox, Classes, - Code, Collapse, ControlGroup, DialogStep, @@ -15,15 +14,12 @@ import { Icon, InputGroup, Intent, - MenuItem, MultistepDialog, NonIdealState, Pre, Spinner, - Switch, } from '@blueprintjs/core'; import { Classes as Classes2, Popover2 } from '@blueprintjs/popover2'; -import { Select2 } from '@blueprintjs/select'; import { FirmwareMetadata, HubType } from '@pybricks/firmware'; import { fileOpen } from 'browser-fs-access'; import classNames from 'classnames'; @@ -33,7 +29,6 @@ import { useDispatch } from 'react-redux'; import { useLocalStorage } from 'usehooks-ts'; import { alertsShowAlert } from '../../alerts/actions'; import { - appName, pybricksUsbDfuWindowsDriverInstallUrl, pybricksUsbLinuxUdevRulesUrl, } from '../../app/constants'; @@ -42,13 +37,10 @@ import { Hub, hubBootloaderType, hubHasBluetoothButton, - hubHasExternalFlash, hubHasUSB, } from '../../components/hubPicker'; import { HubPicker } from '../../components/hubPicker/HubPicker'; import { useHubPickerSelectedHub } from '../../components/hubPicker/hooks'; -import { FileMetadata } from '../../fileStorage'; -import { useFileStorageMetadata } from '../../fileStorage/hooks'; import { useSelector } from '../../reducers'; import { ensureError } from '../../utils'; import ExternalLinkIcon from '../../utils/ExternalLinkIcon'; @@ -363,27 +355,16 @@ const AcceptLicensePanel: React.VoidFunctionComponent = }; type SelectOptionsPanelProps = { - hubType: Hub; hubName: string; - includeProgram: boolean; - selectedIncludeFile: FileMetadata | undefined; onChangeHubName(hubName: string): void; - onChangeIncludeProgram(includeProgram: boolean): void; - onChangeSelectedIncludeFile(selectedIncludeFile: FileMetadata | undefined): void; }; const ConfigureOptionsPanel: React.VoidFunctionComponent = ({ - hubType, hubName, - includeProgram, - selectedIncludeFile, onChangeHubName, - onChangeIncludeProgram, - onChangeSelectedIncludeFile, }) => { const i18n = useI18n(); const isHubNameValid = validateHubName(hubName); - const files = useFileStorageMetadata(); return (
@@ -415,86 +396,6 @@ const ConfigureOptionsPanel: React.VoidFunctionComponent - - {(hubHasExternalFlash(hubType) && ( -

- {i18n.translate( - 'optionsPanel.customMain.notApplicable.message', - )} -

- )) || ( - - main.py }, - )} - checked={includeProgram} - onChange={(e) => - onChangeIncludeProgram( - (e.target as HTMLInputElement).checked, - ) - } - /> - ( - - )} - noResults={ - - } - filterable={false} - popoverProps={{ minimal: true }} - disabled={!includeProgram} - onItemSelect={onChangeSelectedIncludeFile} - > -
); }; @@ -617,8 +518,6 @@ export const InstallPybricksDialog: React.VoidFunctionComponent = () => { const { isOpen } = useSelector((s) => s.firmware.installPybricksDialog); const dispatch = useDispatch(); const [hubName, setHubName] = useState(''); - const [includeProgram, setIncludeProgram] = useState(false); - const [selectedIncludeFile, setSelectedIncludeFile] = useState(); const [licenseAccepted, setLicenseAccepted] = useState(false); const [hubType] = useHubPickerSelectedHub(); const { firmwareData } = useFirmware(hubType); @@ -646,7 +545,6 @@ export const InstallPybricksDialog: React.VoidFunctionComponent = () => { firmwareInstallPybricksDialogAccept( hubBootloaderType(selectedHubType), selectedFirmwareData?.firmwareZip ?? new ArrayBuffer(0), - selectedIncludeFile?.path, hubName, ), ), @@ -685,13 +583,8 @@ export const InstallPybricksDialog: React.VoidFunctionComponent = () => { title={i18n.translate('optionsPanel.title')} panel={ } backButtonProps={{ text: i18n.translate('backButton.label') }} diff --git a/src/firmware/installPybricksDialog/actions.ts b/src/firmware/installPybricksDialog/actions.ts index 6653de68..00449b5d 100644 --- a/src/firmware/installPybricksDialog/actions.ts +++ b/src/firmware/installPybricksDialog/actions.ts @@ -14,20 +14,13 @@ type FlashMethod = 'ble-lwp3-bootloader' | 'usb-lego-dfu'; * Action that indicates the user accepted the install Pybricks firmware dialog. * @param flashMethod The connection method and protocol used for flashing. * @param firmwareZip The firmware.zip raw data. - * @param customProgram Optional path of custom program to include when flashing firmware. * @param hubName The hub name to use when flashing firmware. */ export const firmwareInstallPybricksDialogAccept = createAction( - ( - flashMethod: FlashMethod, - firmwareZip: ArrayBuffer, - customProgram: string | undefined, - hubName: string, - ) => ({ + (flashMethod: FlashMethod, firmwareZip: ArrayBuffer, hubName: string) => ({ type: 'firmware.installPybricksDialog.action.accept', flashMethod, firmwareZip, - customProgram, hubName, }), ); diff --git a/src/firmware/installPybricksDialog/translations/en.json b/src/firmware/installPybricksDialog/translations/en.json index 12eaeed0..3f254b24 100644 --- a/src/firmware/installPybricksDialog/translations/en.json +++ b/src/firmware/installPybricksDialog/translations/en.json @@ -50,19 +50,6 @@ "labelInfo": "(optional)", "help": "Enter a name here to customize the hub name when flashing the firmware. This name will be used in the Bluetooth advertising data and can be used to identify the hub when connecting.", "error": "The name is too long." - }, - "customMain": { - "label": "Include custom program", - "labelInfo": "(optional)", - "notApplicable": { - "message": "This hub has external flash memory so including a custom program when flashing firmware is not needed." - }, - "include": { - "label": "Include selected program as {main}", - "noSelection": "(no selection)", - "noFiles": "(no files)", - "help": "Enable to include your program when flashing the firmware or disable to use the default program. Flashing your program along with the firmware will allow you to run your program without being connected to {appName}" - } } }, "bootloaderPanel": { diff --git a/src/firmware/sagas.test.ts b/src/firmware/sagas.test.ts index 8b7ef4c1..e66823cc 100644 --- a/src/firmware/sagas.test.ts +++ b/src/firmware/sagas.test.ts @@ -4,6 +4,8 @@ import { ToasterInstance } from '@blueprintjs/core'; import { FirmwareMetadata, + FirmwareMetadataV110, + FirmwareMetadataV200, FirmwareReaderError, FirmwareReaderErrorCode, } from '@pybricks/firmware'; @@ -55,9 +57,9 @@ afterEach(() => { describe('flashFirmware', () => { describe('normal flow using app supplied firmware', () => { - test('success', async () => { - const metadata: FirmwareMetadata = { - 'metadata-version': '1.0.0', + test('metadata v1.x works', async () => { + const metadata: FirmwareMetadataV110 = { + 'metadata-version': '1.1.0', 'device-id': HubType.MoveHub, 'checksum-type': 'sum', 'firmware-version': '1.2.3', @@ -65,7 +67,7 @@ describe('flashFirmware', () => { 'mpy-abi-version': 5, 'mpy-cross-options': ['-mno-unicode'], 'user-mpy-offset': 100, - 'hub-name-offset': 90, + 'hub-name-offset': 54, 'max-hub-name-size': 10, }; @@ -86,7 +88,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, 'test name')); + saga.put(flashFirmwareAction(null, 'test name')); // first step is to connect to the hub bootloader @@ -207,6 +209,134 @@ describe('flashFirmware', () => { await saga.end(); }); + test('metadata v2.x works', async () => { + const metadata: FirmwareMetadataV200 = { + 'metadata-version': '2.0.0', + 'device-id': HubType.MoveHub, + 'firmware-version': '1.2.3', + 'checksum-type': 'sum', + 'checksum-size': 1024, + 'hub-name-offset': 54, + 'hub-name-size': 10, + }; + + const zip = new JSZip(); + zip.file('firmware-base.bin', new Uint8Array(64)); + zip.file('firmware.metadata.json', JSON.stringify(metadata)); + zip.file('ReadMe_OSS.txt', 'test'); + + jest.spyOn(window, 'fetch').mockResolvedValueOnce( + new Response(await zip.generateAsync({ type: 'blob' })), + ); + + const saga = new AsyncSaga(flashFirmware, { + nextMessageId: createCountFunc(), + toaster: mock(), + }); + + // saga is triggered by this action + + saga.put(flashFirmwareAction(null, 'test name')); + + // first step is to connect to the hub bootloader + + let action = await saga.take(); + expect(action).toEqual(connect()); + + saga.updateState({ + bootloader: { connection: BootloaderConnectionState.Connected }, + }); + saga.put(didConnect()); + + // then find out what kind of hub it is + + action = await saga.take(); + expect(action).toEqual(infoRequest(0)); + + saga.put(didRequest(0)); + saga.put(infoResponse(0x01000000, 0x08005000, 0x081f800, HubType.MoveHub)); + + // then start flashing the firmware + + // should get didStart action just before starting to erase + action = await saga.take(); + expect(action).toEqual(didStart()); + + // erase first + + action = await saga.take(); + expect(action).toEqual(alertsShowAlert('firmware', 'releaseButton')); + + action = await saga.take(); + expect(action).toEqual(eraseRequest(1, /* isCityHub */ false)); + + saga.put(didRequest(1)); + saga.put(eraseResponse(Result.OK)); + + // then write the new firmware + + const totalFirmwareSize = 68; + action = await saga.take(); + expect(action).toEqual(initRequest(2, totalFirmwareSize)); + + saga.put(didRequest(2)); + saga.put(initResponse(Result.OK)); + + const dummyPayload = new ArrayBuffer(0); + let id = 2; + for (let count = 1, offset = 0; ; count++, offset += 14) { + action = await saga.take(); + expect(action).toEqual( + programRequest(++id, 0x08005000 + offset, dummyPayload), + ); + expect( + (action as ReturnType).payload.byteLength, + ).toBe(Math.min(14, totalFirmwareSize - offset)); + + saga.put(didRequest(id)); + + action = await saga.take(); + expect(action).toEqual(didProgress(offset / totalFirmwareSize)); + + // Have to be careful that a checksum request is not sent after + // last payload is sent, otherwise the hub gets confused. + + if (offset + 14 >= totalFirmwareSize) { + expect(count).toBe(5); + break; + } + + if (count % 10 === 0) { + action = await saga.take(); + expect(action).toEqual(checksumRequest(++id)); + + saga.put(didRequest(id)); + saga.put(checksumResponse(0)); + } + } + + // hub indicates success + + saga.put(programResponse(0xe0, totalFirmwareSize)); + + action = await saga.take(); + expect(action).toEqual(didProgress(1)); + + // and finally reboot the hub + + action = await saga.take(); + expect(action).toEqual(rebootRequest(++id)); + + saga.put(didRequest(id)); + + // then we are done + + action = await saga.take(); + expect(action).toEqual(didFinish()); + + await saga.end(); + }); + test('fail to connect', async () => { const metadata: FirmwareMetadata = { 'metadata-version': '1.0.0', @@ -236,7 +366,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -286,7 +416,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -354,7 +484,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -418,7 +548,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -485,7 +615,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -545,7 +675,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -611,7 +741,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -694,7 +824,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -760,7 +890,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -859,7 +989,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -967,7 +1097,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -1115,7 +1245,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader @@ -1264,7 +1394,6 @@ describe('flashFirmware', () => { saga.put( flashFirmwareAction( await zip.generateAsync({ type: 'arraybuffer' }), - undefined, '', ), ); @@ -1417,7 +1546,6 @@ describe('flashFirmware', () => { saga.put( flashFirmwareAction( await zip.generateAsync({ type: 'arraybuffer' }), - undefined, '', ), ); @@ -1465,7 +1593,6 @@ describe('flashFirmware', () => { saga.put( flashFirmwareAction( await zip.generateAsync({ type: 'arraybuffer' }), - undefined, '', ), ); @@ -1512,7 +1639,6 @@ describe('flashFirmware', () => { saga.put( flashFirmwareAction( await zip.generateAsync({ type: 'arraybuffer' }), - undefined, '', ), ); @@ -1573,7 +1699,6 @@ describe('flashFirmware', () => { saga.put( flashFirmwareAction( await zip.generateAsync({ type: 'arraybuffer' }), - undefined, '', ), ); @@ -1635,7 +1760,6 @@ describe('flashFirmware', () => { saga.put( flashFirmwareAction( await zip.generateAsync({ type: 'arraybuffer' }), - undefined, '', ), ); @@ -1700,7 +1824,6 @@ describe('flashFirmware', () => { saga.put( flashFirmwareAction( await zip.generateAsync({ type: 'arraybuffer' }), - undefined, '', ), ); @@ -1787,7 +1910,7 @@ describe('flashFirmware', () => { // saga is triggered by this action - saga.put(flashFirmwareAction(null, undefined, '')); + saga.put(flashFirmwareAction(null, '')); // first step is to connect to the hub bootloader diff --git a/src/firmware/sagas.ts b/src/firmware/sagas.ts index 9c9bdb80..3045716b 100644 --- a/src/firmware/sagas.ts +++ b/src/firmware/sagas.ts @@ -7,6 +7,8 @@ import { FirmwareReaderError, HubType, encodeHubName, + metadataIsV100, + metadataIsV110, } from '@pybricks/firmware'; import cityHubZip from '@pybricks/firmware/build/cityhub.zip'; import moveHubZip from '@pybricks/firmware/build/movehub.zip'; @@ -29,11 +31,6 @@ import { takeEvery, } from 'typed-redux-saga/macro'; import { alertsDidShowAlert, alertsShowAlert } from '../alerts/actions'; -import { - fileStorageDidFailToReadFile, - fileStorageDidReadFile, - fileStorageReadFile, -} from '../fileStorage/actions'; import { checksumRequest, checksumResponse, @@ -190,11 +187,10 @@ function* firmwareIterator(data: DataView, maxSize: number): Generator { * Loads Pybricks firmware from a .zip file. * * @param data The zip file raw data - * @param program User program or `undefined` to use main.py from firmware.zip + * @param hubName Optional custom name for the hub. */ function* loadFirmware( data: ArrayBuffer, - program: string | undefined, hubName: string, ): SagaGenerator<{ firmware: Uint8Array; deviceId: HubType }> { const [reader, readerErr] = yield* call(() => maybe(FirmwareReader.load(data))); @@ -219,90 +215,136 @@ function* loadFirmware( const firmwareBase = yield* call(() => reader.readFirmwareBase()); const metadata = yield* call(() => reader.readMetadata()); - // if a user program was not given, then use main.py from the firmware.zip - if (program === undefined) { - program = yield* call(() => reader.readMainPy()); - } + // v1.x allows appending main.py to firmware, later versions do not + if (metadataIsV100(metadata) || metadataIsV110(metadata)) { + const program = (yield* call(() => reader.readMainPy())) ?? ''; - // REVISIT: the firmware may eventually be changed to allow no main.py - // for now, ensure there is a program even if it does nothing - if (!program) { - program = ''; - } + if (![5, 6].includes(metadata['mpy-abi-version'])) { + yield* put( + didFailToFinish( + FailToFinishReasonType.BadMetadata, + 'mpy-abi-version', + MetadataProblem.NotSupported, + ), + ); + + // FIXME: we should return error/throw instead + yield* disconnectAndCancel(); + + // istanbul ignore next: needed for typescript flow + throw new Error('unreachable'); + } - if (![5, 6].includes(metadata['mpy-abi-version'])) { yield* put( - didFailToFinish( - FailToFinishReasonType.BadMetadata, - 'mpy-abi-version', - MetadataProblem.NotSupported, + compile( + program, + metadata['mpy-abi-version'], + metadata['mpy-cross-options'], ), ); + const { mpy, mpyFail } = yield* race({ + mpy: take(didCompile), + mpyFail: take(didFailToCompile), + }); - // FIXME: we should return error/throw instead - yield* disconnectAndCancel(); + if (mpyFail) { + // FIXME: we should return error/throw instead + yield* put(didFailToFinish(FailToFinishReasonType.FailedToCompile)); + yield* disconnectAndCancel(); - // istanbul ignore next: needed for typescript flow - throw new Error('unreachable'); + // istanbul ignore next: needed for typescript flow + throw new Error('unreachable'); + } + + defined(mpy); + + // compute offset for checksum - must be aligned to 4-byte boundary + const checksumOffset = + metadata['user-mpy-offset'] + + 4 + + mpy.data.length + + fmod(-mpy.data.length, 4); + + const firmware = new Uint8Array(checksumOffset + 4); + const firmwareView = new DataView(firmware.buffer); + + if (firmware.length > metadata['max-firmware-size']) { + // FIXME: we should return error/throw instead + yield* put(didFailToFinish(FailToFinishReasonType.FirmwareSize)); + yield* disconnectAndCancel(); + + // istanbul ignore next: needed for typescript flow + throw new Error('unreachable'); + } + + firmware.set(firmwareBase); + firmwareView.setUint32(metadata['user-mpy-offset'], mpy.data.length, true); + firmware.set(mpy.data, metadata['user-mpy-offset'] + 4); + + // if the firmware supports it, we can set a custom hub name + if (!metadataIsV100(metadata)) { + // empty string means use default name (don't write over firmware) + if (hubName) { + firmware.set( + encodeHubName(hubName, metadata), + metadata['hub-name-offset'], + ); + } + } + + const checksum = (function () { + switch (metadata['checksum-type']) { + case 'sum': + return sumComplement32( + firmwareIterator(firmwareView, metadata['max-firmware-size']), + ); + case 'crc32': + return crc32( + firmwareIterator(firmwareView, metadata['max-firmware-size']), + ); + default: + return undefined; + } + })(); + + if (!checksum) { + // FIXME: we should return error/throw instead + yield* put( + didFailToFinish( + FailToFinishReasonType.BadMetadata, + 'checksum-type', + MetadataProblem.NotSupported, + ), + ); + yield* disconnectAndCancel(); + + // istanbul ignore next: needed for typescript flow + throw new Error('unreachable'); + } + + firmwareView.setUint32(checksumOffset, checksum, true); + + return { firmware, deviceId: metadata['device-id'] }; } - yield* put( - compile(program, metadata['mpy-abi-version'], metadata['mpy-cross-options']), - ); - const { mpy, mpyFail } = yield* race({ - mpy: take(didCompile), - mpyFail: take(didFailToCompile), - }); - - if (mpyFail) { - // FIXME: we should return error/throw instead - yield* put(didFailToFinish(FailToFinishReasonType.FailedToCompile)); - yield* disconnectAndCancel(); - - // istanbul ignore next: needed for typescript flow - throw new Error('unreachable'); - } - - defined(mpy); - - // compute offset for checksum - must be aligned to 4-byte boundary - const checksumOffset = - metadata['user-mpy-offset'] + 4 + mpy.data.length + fmod(-mpy.data.length, 4); - - const firmware = new Uint8Array(checksumOffset + 4); + const firmware = new Uint8Array(firmwareBase.length + 4); const firmwareView = new DataView(firmware.buffer); - if (firmware.length > metadata['max-firmware-size']) { - // FIXME: we should return error/throw instead - yield* put(didFailToFinish(FailToFinishReasonType.FirmwareSize)); - yield* disconnectAndCancel(); - - // istanbul ignore next: needed for typescript flow - throw new Error('unreachable'); - } - firmware.set(firmwareBase); - firmwareView.setUint32(metadata['user-mpy-offset'], mpy.data.length, true); - firmware.set(mpy.data, metadata['user-mpy-offset'] + 4); - // if the firmware supports it, we can set a custom hub name - if (metadata['max-hub-name-size']) { - // empty string means use default name (don't write over firmware) - if (hubName) { - firmware.set(encodeHubName(hubName, metadata), metadata['hub-name-offset']); - } + // empty string means use default name (don't write over firmware) + if (hubName) { + firmware.set(encodeHubName(hubName, metadata), metadata['hub-name-offset']); } const checksum = (function () { switch (metadata['checksum-type']) { case 'sum': return sumComplement32( - firmwareIterator(firmwareView, metadata['max-firmware-size']), + firmwareIterator(firmwareView, metadata['checksum-size']), ); case 'crc32': - return crc32( - firmwareIterator(firmwareView, metadata['max-firmware-size']), - ); + return crc32(firmwareIterator(firmwareView, metadata['checksum-size'])); default: return undefined; } @@ -323,7 +365,7 @@ function* loadFirmware( throw new Error('unreachable'); } - firmwareView.setUint32(checksumOffset, checksum, true); + firmwareView.setUint32(firmwareBase.length, checksum, true); return { firmware, deviceId: metadata['device-id'] }; } @@ -339,37 +381,8 @@ function* handleFlashFirmware(action: ReturnType): Generat let firmware: Uint8Array | undefined = undefined; let deviceId: HubType | undefined = undefined; - let program: string | undefined = undefined; - - if (action.customProgram) { - yield* put(fileStorageReadFile(action.customProgram)); - - const { didRead, didFailToRead } = yield* race({ - didRead: take( - fileStorageDidReadFile.when((a) => a.path === action.customProgram), - ), - didFailToRead: take( - fileStorageDidFailToReadFile.when( - (a) => a.path === action.customProgram, - ), - ), - }); - - if (didFailToRead) { - throw didFailToRead.error; - } - - defined(didRead); - - program = didRead.contents; - } - if (action.data !== null) { - ({ firmware, deviceId } = yield* loadFirmware( - action.data, - program, - action.hubName, - )); + ({ firmware, deviceId } = yield* loadFirmware(action.data, action.hubName)); } yield* put(connect()); @@ -411,11 +424,7 @@ function* handleFlashFirmware(action: ReturnType): Generat } const data = yield* call(() => response.arrayBuffer()); - ({ firmware, deviceId } = yield* loadFirmware( - data, - program, - action.hubName, - )); + ({ firmware, deviceId } = yield* loadFirmware(data, action.hubName)); if (deviceId !== undefined && info.hubType !== deviceId) { yield* put(didFailToFinish(FailToFinishReasonType.DeviceMismatch)); @@ -689,11 +698,7 @@ function* handleFlashUsbDfu(action: ReturnType): Gen }), ); - const { firmware, deviceId } = yield* loadFirmware( - action.data, - undefined, - action.hubName, - ); + const { firmware, deviceId } = yield* loadFirmware(action.data, action.hubName); if (deviceId !== productIdMap.get(device.productId)) { yield* put(alertsShowAlert('firmware', 'firmwareMismatch')); @@ -819,13 +824,7 @@ function* handleInstallPybricks(): Generator { switch (accepted.flashMethod) { case 'ble-lwp3-bootloader': - yield* put( - flashFirmware( - accepted.firmwareZip, - accepted.customProgram, - accepted.hubName, - ), - ); + yield* put(flashFirmware(accepted.firmwareZip, accepted.hubName)); break; case 'usb-lego-dfu': yield* put(firmwareFlashUsbDfu(accepted.firmwareZip, accepted.hubName)); diff --git a/src/pybricksMicropython/python-message.ts b/src/pybricksMicropython/python-message.ts index 728b3f65..60be66fd 100644 --- a/src/pybricksMicropython/python-message.ts +++ b/src/pybricksMicropython/python-message.ts @@ -99,3 +99,20 @@ export const pythonMessageDidFailToGetSignature = createAction((error: Error) => type: 'python.message.didFailToGetSignature', error, })); + +export const pythonMessageWriteUserFile = createAction( + (path: string, contents: string) => ({ + type: 'python.message.writeUserFile', + path, + contents, + }), +); + +export const pythonMessageDeleteUserFile = createAction((path: string) => ({ + type: 'python.message.deleteUserFile', + path, +})); + +export const pythonMessageDidMountUserFileSystem = createAction(() => ({ + type: 'python.message.didMountUserFileSystem', +})); diff --git a/src/pybricksMicropython/python-worker.ts b/src/pybricksMicropython/python-worker.ts index 48820f9d..ae6546f9 100644 --- a/src/pybricksMicropython/python-worker.ts +++ b/src/pybricksMicropython/python-worker.ts @@ -11,15 +11,18 @@ import pyodidePackage from 'pyodide/package.json'; import { ensureError } from '../utils'; import { pythonMessageComplete, + pythonMessageDeleteUserFile, pythonMessageDidComplete, pythonMessageDidFailToComplete, pythonMessageDidFailToGetSignature, pythonMessageDidFailToInit, pythonMessageDidGetSignature, pythonMessageDidInit, + pythonMessageDidMountUserFileSystem, pythonMessageGetSignature, pythonMessageInit, pythonMessageSetInterruptBuffer, + pythonMessageWriteUserFile, } from './python-message'; /** @@ -61,6 +64,34 @@ async function init(): Promise { lockFileURL: new URL('pyodide/repodata.json', import.meta.url).toString(), }); + // REVISIT: it would be nice if we could make a custom driver to mount + // the custom Pybricks Code Dexie-based file system directly instead of + // mirroring it + const mountDir = '/user'; + pyodide.FS.mkdir(mountDir); + pyodide.FS.mount(pyodide.FS.filesystems.MEMFS, { root: '.' }, mountDir); + + self.addEventListener('message', async (e) => { + if (pythonMessageWriteUserFile.matches(e.data)) { + pyodide.FS.writeFile(`${mountDir}/${e.data.path}`, e.data.contents); + console.debug('copied', e.data.path, 'to emscripten fs'); + return; + } + + if (pythonMessageDeleteUserFile.matches(e.data)) { + pyodide.FS.unlink(`${mountDir}/${e.data.path}`); + console.debug('removed', e.data.path, ' from emscripten fs'); + return; + } + }); + + // separate message for file system ready since it takes a long time for + // the rest of the init + self.postMessage(pythonMessageDidMountUserFileSystem()); + + // add user directory to sys.path for code completion + await pyodide.runPythonAsync(`import sys; sys.path.append("${mountDir}")`); + // NB: using URL+import.meta.url for webpack magic - don't try to optimize it await pyodide.loadPackage( new URL('@pybricks/jedi/docstring-parser.whl', import.meta.url).toString(), diff --git a/yarn.lock b/yarn.lock index 5bb0bd26..52dfeae1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2369,12 +2369,12 @@ __metadata: languageName: node linkType: hard -"@pybricks/firmware@npm:5.0.0": - version: 5.0.0 - resolution: "@pybricks/firmware@npm:5.0.0" +"@pybricks/firmware@npm:6.0.1": + version: 6.0.1 + resolution: "@pybricks/firmware@npm:6.0.1" dependencies: jszip: ^3.7.1 - checksum: 049dd90e988aa574cfa0ead1e62bcb74e6fdfc9b709bc1c40874ddf3abb63cd35555d22806c91184bc2e982912a001e0ef94ef72ef66217eb0319bcaf45a7cb3 + checksum: c0d6e9bef7ac8b1009f90f64cedb8872bae02c6ef6c02fcaf73238407b5164c9c27312dc7afaf051663f7f086bca7b8ccbeec641ef3105ec03308e3d9221175a languageName: node linkType: hard @@ -2415,7 +2415,7 @@ __metadata: "@blueprintjs/popover2": ^1.6.4 "@blueprintjs/select": ^4.6.4 "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7 - "@pybricks/firmware": 5.0.0 + "@pybricks/firmware": 6.0.1 "@pybricks/ide-docs": 2.2.0 "@pybricks/jedi": ^1.0.1 "@pybricks/mpy-cross-v5": ^2.0.0