diff --git a/src/editor/sagas.ts b/src/editor/sagas.ts index 519bc7e4..ef72fdcb 100644 --- a/src/editor/sagas.ts +++ b/src/editor/sagas.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2022 The Pybricks Authors +import type { DatabaseChangeType, IDatabaseChange } from 'dexie-observable/api'; import { monaco } from 'react-monaco-editor'; import { EventChannel, buffers, eventChannel } from 'redux-saga'; import { @@ -16,7 +17,7 @@ import { take, takeEvery, } from 'typed-redux-saga/macro'; -import { UUID } from '../fileStorage'; +import { FileStorageDb, UUID } from '../fileStorage'; import { fileStorageDidFailToLoadTextFile, fileStorageDidFailToStoreTextFileViewState, @@ -29,15 +30,18 @@ import { } from '../fileStorage/actions'; import { pythonMessageComplete, + pythonMessageDeleteUserFile, pythonMessageDidComplete, pythonMessageDidFailToComplete, pythonMessageDidFailToGetSignature, pythonMessageDidFailToInit, pythonMessageDidGetSignature, pythonMessageDidInit, + pythonMessageDidMountUserFileSystem, pythonMessageGetSignature, pythonMessageInit, pythonMessageSetInterruptBuffer, + pythonMessageWriteUserFile, } from '../pybricksMicropython/python-message'; import { RootState } from '../reducers'; import { acquireLock, defined, ensureError } from '../utils'; @@ -390,6 +394,97 @@ function* monitorEditors(): Generator { } } +// HACK: dexie-observable exports const enum, so we have to redefine values +const DatabaseChangeTypeCreate: DatabaseChangeType.Create = 1; +const DatabaseChangeTypeUpdate: DatabaseChangeType.Update = 2; +const DatabaseChangeTypeDelete: DatabaseChangeType.Delete = 3; + +/** + * Mirrors the Dexie-based file system to the Emscripten file system in the + * Python Web Worker. + * + * @param worker The web worker. + */ +function* mirrorFileSystem(worker: Worker): Generator { + // wait for file storage to become ready if it isn't already + if (!(yield* select((s: RootState) => s.fileStorage.isInitialized))) { + yield take(fileStorageDidInitialize); + } + + const db = yield* getContext('fileStorage'); + + // subscribe to future changes + const dbChangedChan = eventChannel((emit) => { + db.on('changes').subscribe(emit); + return () => db.on('changes').unsubscribe(emit); + }); + + // copy all existing files + yield* call(() => + db.transaction('r', db._contents, () => + db._contents.each((f) => + worker.postMessage(pythonMessageWriteUserFile(f.path, f.contents)), + ), + ), + ); + + // handle future changes + try { + for (;;) { + const changes = yield* take(dbChangedChan); + + for (const c of changes) { + // only interested in metadata table changes + if (c.table !== db.metadata.name) { + continue; + } + + switch (c.type) { + case DatabaseChangeTypeCreate: + case DatabaseChangeTypeUpdate: + // only send message if file was created or contents + // changed - ignore other metadata changes + if ( + c.type === DatabaseChangeTypeUpdate && + c.obj.sha256 === c.oldObj.sha256 + ) { + break; + } + + yield* call(() => + db.transaction('r', db._contents, async () => { + const file = await db._contents.get(c.obj.path); + + // istanbul ignore if: programmer error if we hit this + if (!file) { + console.error( + `could not find file '${c.obj.path}'`, + ); + return; + } + + worker.postMessage( + pythonMessageWriteUserFile( + file.path, + file.contents, + ), + ); + }), + ); + + break; + + case DatabaseChangeTypeDelete: + worker.postMessage(pythonMessageDeleteUserFile(c.oldObj.path)); + break; + } + } + } + } finally { + dbChangedChan.close(); + } +} + /** * Runs a web worker with Pyodide so that we can use Jedi for intellisense. */ @@ -446,6 +541,11 @@ function* runJedi(): Generator { defined(messageEvent); + if (pythonMessageDidMountUserFileSystem.matches(messageEvent.data)) { + yield* fork(mirrorFileSystem, worker); + continue; + } + if (pythonMessageDidFailToInit.matches(messageEvent.data)) { yield* put(editorCompletionDidFailToInit()); throw messageEvent.data.error; diff --git a/src/pybricksMicropython/python-message.ts b/src/pybricksMicropython/python-message.ts index 728b3f65..60be66fd 100644 --- a/src/pybricksMicropython/python-message.ts +++ b/src/pybricksMicropython/python-message.ts @@ -99,3 +99,20 @@ export const pythonMessageDidFailToGetSignature = createAction((error: Error) => type: 'python.message.didFailToGetSignature', error, })); + +export const pythonMessageWriteUserFile = createAction( + (path: string, contents: string) => ({ + type: 'python.message.writeUserFile', + path, + contents, + }), +); + +export const pythonMessageDeleteUserFile = createAction((path: string) => ({ + type: 'python.message.deleteUserFile', + path, +})); + +export const pythonMessageDidMountUserFileSystem = createAction(() => ({ + type: 'python.message.didMountUserFileSystem', +})); diff --git a/src/pybricksMicropython/python-worker.ts b/src/pybricksMicropython/python-worker.ts index 48820f9d..ae6546f9 100644 --- a/src/pybricksMicropython/python-worker.ts +++ b/src/pybricksMicropython/python-worker.ts @@ -11,15 +11,18 @@ import pyodidePackage from 'pyodide/package.json'; import { ensureError } from '../utils'; import { pythonMessageComplete, + pythonMessageDeleteUserFile, pythonMessageDidComplete, pythonMessageDidFailToComplete, pythonMessageDidFailToGetSignature, pythonMessageDidFailToInit, pythonMessageDidGetSignature, pythonMessageDidInit, + pythonMessageDidMountUserFileSystem, pythonMessageGetSignature, pythonMessageInit, pythonMessageSetInterruptBuffer, + pythonMessageWriteUserFile, } from './python-message'; /** @@ -61,6 +64,34 @@ async function init(): Promise { lockFileURL: new URL('pyodide/repodata.json', import.meta.url).toString(), }); + // REVISIT: it would be nice if we could make a custom driver to mount + // the custom Pybricks Code Dexie-based file system directly instead of + // mirroring it + const mountDir = '/user'; + pyodide.FS.mkdir(mountDir); + pyodide.FS.mount(pyodide.FS.filesystems.MEMFS, { root: '.' }, mountDir); + + self.addEventListener('message', async (e) => { + if (pythonMessageWriteUserFile.matches(e.data)) { + pyodide.FS.writeFile(`${mountDir}/${e.data.path}`, e.data.contents); + console.debug('copied', e.data.path, 'to emscripten fs'); + return; + } + + if (pythonMessageDeleteUserFile.matches(e.data)) { + pyodide.FS.unlink(`${mountDir}/${e.data.path}`); + console.debug('removed', e.data.path, ' from emscripten fs'); + return; + } + }); + + // separate message for file system ready since it takes a long time for + // the rest of the init + self.postMessage(pythonMessageDidMountUserFileSystem()); + + // add user directory to sys.path for code completion + await pyodide.runPythonAsync(`import sys; sys.path.append("${mountDir}")`); + // NB: using URL+import.meta.url for webpack magic - don't try to optimize it await pyodide.loadPackage( new URL('@pybricks/jedi/docstring-parser.whl', import.meta.url).toString(),