// SPDX-License-Identifier: MIT // Copyright (c) 2022 The Pybricks Authors import type { DatabaseChangeType, IDatabaseChange } from 'dexie-observable/api'; import * as monaco from 'monaco-editor'; import { EventChannel, buffers, eventChannel } from 'redux-saga'; import { call, cancelled, delay, fork, getContext, put, race, select, take, takeEvery, } from 'typed-redux-saga/macro'; import { FileStorageDb, UUID } from '../fileStorage'; import { fileStorageDidFailToLoadTextFile, fileStorageDidFailToStoreTextFileViewState, fileStorageDidInitialize, fileStorageDidLoadTextFile, fileStorageDidStoreTextFileViewState, fileStorageLoadTextFile, fileStorageStoreTextFileValue, fileStorageStoreTextFileViewState, } from '../fileStorage/actions'; import { pythonMessageComplete, pythonMessageDeleteUserFile, pythonMessageDidComplete, pythonMessageDidFailToComplete, pythonMessageDidFailToGetSignature, pythonMessageDidFailToInit, pythonMessageDidGetSignature, pythonMessageDidInit, pythonMessageDidMountUserFileSystem, pythonMessageGetSignature, pythonMessageInit, pythonMessageSetInterruptBuffer, pythonMessageWriteUserFile, } from '../pybricksMicropython/python-message'; import { RootState } from '../reducers'; import { acquireLock, defined, ensureError } from '../utils'; import { createCountFunc } from '../utils/iter'; import { editorActivateFile, editorCloseFile, editorCompletionDidFailToInit, editorCompletionDidInit, editorCompletionInit, editorDidActivateFile, editorDidCloseFile, editorDidCreate, editorDidFailToActivateFile, editorDidFailToOpenFile, editorDidOpenFile, editorGetValueRequest, editorGetValueResponse, editorGoto, editorOpenFile, } from './actions'; import { EditorError } from './error'; import { ActiveFileHistoryManager, OpenFileManager } from './lib'; import { pybricksMicroPythonId } from './pybricksMicroPython'; function* handleEditorGetValueRequest( editor: monaco.editor.ICodeEditor, action: ReturnType, ): Generator { yield* put(editorGetValueResponse(action.id, editor.getValue())); } /** Handle changes to the model. */ function* handleModelDidChange( ms: number, chan: EventChannel, model: monaco.editor.ITextModel, ): Generator { for (;;) { yield* take(chan); const value = model.getValue(); // when the model changes, save it to storage. yield* put(fileStorageStoreTextFileValue(model.uri.path as UUID, value)); // failures are ignored // throttle the writes so we don't do it too often while user is typing quickly yield* delay(ms); } } function* handleEditorOpenFile( editor: monaco.editor.ICodeEditor, openFiles: OpenFileManager, action: ReturnType, ): Generator { let closeRequested = false; try { const defer: Array<() => void | Promise> = []; try { const modelUri = monaco.Uri.from({ scheme: 'pybricksCode', path: action.uuid, }); const releaseLock = yield* call(() => acquireLock(`pybricks.editor+${modelUri}`), ); if (!releaseLock) { throw new EditorError( 'FileInUse', 'the file is already open in another editor', ); } defer.push(releaseLock); yield* put(fileStorageLoadTextFile(action.uuid)); const { didLoad, didFailToLoad } = yield* race({ didLoad: take( fileStorageDidLoadTextFile.when((a) => a.uuid === action.uuid), ), didFailToLoad: take( fileStorageDidFailToLoadTextFile.when( (a) => a.uuid === action.uuid, ), ), }); if (didFailToLoad) { throw didFailToLoad.error; } defined(didLoad); const model = monaco.editor.createModel( didLoad.value, pybricksMicroPythonId, modelUri, ); defer.push(() => model.dispose()); // NB: the throttle effect doesn't work with event channels, so we // emulate the effect by using a buffer with size of one here... const didChangeModelChan = eventChannel((emit) => { const subscription = model.onDidChangeContent((e) => emit(e)); return () => subscription.dispose(); }, buffers.sliding(1)); defer.push(() => didChangeModelChan.close()); // ... and then fork to function that looks like // https://github.com/redux-saga/redux-saga/issues/620#issuecomment-259161095 yield* fork(handleModelDidChange, 1000, didChangeModelChan, model); openFiles.add(action.uuid, model, didLoad.viewState); defer.push(() => openFiles.remove(action.uuid)); yield* put(editorDidOpenFile(action.uuid)); yield* take(editorCloseFile.when((a) => a.uuid === action.uuid)); closeRequested = true; // if the file is the currently active file, the view state will // be out of sync, so we need to update it here before saving to // storage if (editor.getModel() === model) { openFiles.updateViewState(action.uuid, editor.saveViewState()); } // save the file contents on close since the change watcher is // throttled and the latest changes may not have been saved yet yield* put(fileStorageStoreTextFileValue(action.uuid, model.getValue())); } finally { for (const callback of defer.reverse()) { callback(); } // only send the did close action if the corresponding action requested it if (closeRequested) { yield* put(editorDidCloseFile(action.uuid)); } } } catch (err) { yield* put(editorDidFailToOpenFile(action.uuid, ensureError(err))); } } function* handleEditorActivateFile( editor: monaco.editor.ICodeEditor, openFiles: OpenFileManager, activeFileHistory: ActiveFileHistoryManager, action: ReturnType, ): Generator { try { if (!openFiles.has(action.uuid)) { yield* put(editorOpenFile(action.uuid)); const { didFailToOpen } = yield* race({ didOpen: take(editorDidOpenFile.when((a) => a.uuid === action.uuid)), didFailToOpen: take( editorDidFailToOpenFile.when((a) => a.uuid === action.uuid), ), }); if (didFailToOpen) { throw didFailToOpen.error; } } const file = openFiles.get(action.uuid); // istanbul ignore if: this should always be available after editorDidOpenFile if (file === undefined) { throw new Error('bug: could not get file from openFiles'); } // save the current view state for later activation const oldModel = editor.getModel(); if (oldModel) { openFiles.updateViewState( oldModel.uri.path as UUID, editor.saveViewState(), ); } editor.setModel(file.model); editor.restoreViewState(file.viewState); activeFileHistory.push(action.uuid); yield* put(editorDidActivateFile(action.uuid)); } catch (err) { yield* put(editorDidFailToActivateFile(action.uuid, ensureError(err))); } } function* handleEditorGoto( editor: monaco.editor.ICodeEditor, action: ReturnType, ): Generator { yield* put(editorActivateFile(action.uuid)); const { didActivate, didFailToActivate } = yield* race({ didActivate: take(editorDidActivateFile.when((a) => a.uuid === action.uuid)), didFailToActivate: take( editorDidFailToActivateFile.when((a) => a.uuid === action.uuid), ), }); if (didFailToActivate) { return; } defined(didActivate); editor.revealLineInCenterIfOutsideViewport(action.line); editor.setSelection({ startColumn: 1, startLineNumber: action.line, endColumn: Infinity, endLineNumber: action.line, }); } function* handleEditorDidCloseFile( activeFileHistory: ActiveFileHistoryManager, action: ReturnType, ): Generator { // handleEditorOpenFile handles most of the closing of files. // Here we only need to handle removing the closed file from the active // file history. const newActiveFile = activeFileHistory.pop(action.uuid); // if the closed file was the active file, we need to activate a new file // otherwise there will be no active file if (newActiveFile) { yield* put(editorActivateFile(newActiveFile)); } } /** * Monitors the editor for any possible view state change and stores the state * with the associated file when the state changes. */ function* monitorViewState(editor: monaco.editor.ICodeEditor): Generator { const ch = eventChannel< | monaco.editor.ICursorPositionChangedEvent | monaco.editor.ICursorSelectionChangedEvent | monaco.IScrollEvent >((emit) => { const subscriptions = new Array(); // there isn't a single view state change event, so this should be // all of the events that can trigger a state change subscriptions.push(editor.onDidChangeCursorPosition(emit)); subscriptions.push(editor.onDidChangeCursorSelection(emit)); subscriptions.push(editor.onDidScrollChange(emit)); return () => subscriptions.forEach((s) => s.dispose()); }, buffers.sliding(1)); try { for (;;) { yield* take(ch); const model = editor.getModel(); if (!model) { continue; } const uuid = model.uri.path as UUID; yield* put(fileStorageStoreTextFileViewState(uuid, editor.saveViewState())); yield* race({ didStore: take( fileStorageDidStoreTextFileViewState.when((a) => a.uuid === uuid), ), didFailToStore: take( fileStorageDidFailToStoreTextFileViewState.when( (a) => a.uuid === uuid, ), ), }); } } finally { ch.close(); } } function* handleDidCreateEditor(editor: monaco.editor.ICodeEditor): Generator { // first, we need to be sure that file storage is ready const isFileStorageInitialized = yield* select( (s: RootState) => s.fileStorage.isInitialized, ); if (!isFileStorageInitialized) { yield* take(fileStorageDidInitialize); } const openFiles = new OpenFileManager(); const activeFileHistory = new ActiveFileHistoryManager(editor.getId()); yield* takeEvery(editorGetValueRequest, handleEditorGetValueRequest, editor); yield* takeEvery(editorOpenFile, handleEditorOpenFile, editor, openFiles); yield* takeEvery( editorActivateFile, handleEditorActivateFile, editor, openFiles, activeFileHistory, ); yield* takeEvery(editorGoto, handleEditorGoto, editor); yield* takeEvery(editorDidCloseFile, handleEditorDidCloseFile, activeFileHistory); yield* fork(monitorViewState, editor); yield* put(editorDidCreate()); // this should restore all previously open files in the same order // the were last used (which may be different from the order in which // they were originally opened) for (const item of activeFileHistory.getFromStorage()) { yield* put(editorActivateFile(item)); yield* race({ didActivate: take(editorDidActivateFile.when((a) => a.uuid === item)), didFailToActivate: take( editorDidFailToActivateFile.when((a) => a.uuid === item), ), }); // Errors are ignored here since we don't want to pester the user with // error messages. } } function* monitorEditors(): Generator { const ch = eventChannel((emit) => { const subscription = monaco.editor.onDidCreateEditor(emit); return () => subscription.dispose(); }); try { yield* takeEvery(ch, handleDidCreateEditor); yield* take('__never__'); } finally { ch.close(); } } // HACK: dexie-observable exports const enum, so we have to redefine values const DatabaseChangeTypeCreate: DatabaseChangeType.Create = 1; const DatabaseChangeTypeUpdate: DatabaseChangeType.Update = 2; const DatabaseChangeTypeDelete: DatabaseChangeType.Delete = 3; /** * Mirrors the Dexie-based file system to the Emscripten file system in the * Python Web Worker. * * @param worker The web worker. */ function* mirrorFileSystem(worker: Worker): Generator { // wait for file storage to become ready if it isn't already if (!(yield* select((s: RootState) => s.fileStorage.isInitialized))) { yield take(fileStorageDidInitialize); } const db = yield* getContext('fileStorage'); // subscribe to future changes const dbChangedChan = eventChannel((emit) => { db.on('changes').subscribe(emit); return () => db.on('changes').unsubscribe(emit); }); // copy all existing files yield* call(() => db.transaction('r', db._contents, () => db._contents.each((f) => worker.postMessage(pythonMessageWriteUserFile(f.path, f.contents)), ), ), ); // handle future changes try { for (;;) { const changes = yield* take(dbChangedChan); for (const c of changes) { // only interested in metadata table changes if (c.table !== db.metadata.name) { continue; } switch (c.type) { case DatabaseChangeTypeCreate: case DatabaseChangeTypeUpdate: // only send message if file was created or contents // changed - ignore other metadata changes if ( c.type === DatabaseChangeTypeUpdate && c.obj.sha256 === c.oldObj.sha256 ) { break; } yield* call(() => db.transaction('r', db._contents, async () => { const file = await db._contents.get(c.obj.path); // istanbul ignore if: programmer error if we hit this if (!file) { console.error( `could not find file '${c.obj.path}'`, ); return; } worker.postMessage( pythonMessageWriteUserFile( file.path, file.contents, ), ); }), ); break; case DatabaseChangeTypeDelete: worker.postMessage(pythonMessageDeleteUserFile(c.oldObj.path)); break; } } } } finally { dbChangedChan.close(); } } /** * Runs a web worker with Pyodide so that we can use Jedi for intellisense. */ function* runJedi(): Generator { // TODO: web workers are not implemented in test environment if (process.env.NODE_ENV === 'test') { return; } const defer = new Array<() => void>(); try { console.debug('creating code completion worker'); // start the web worker and set up communication channels const worker = new Worker( new URL('../pybricksMicropython/python-worker.ts', import.meta.url), ); defer.push(() => worker.terminate()); const messageChannel = eventChannel((emit) => { worker.addEventListener('message', emit); return () => worker.removeEventListener('message', emit); }, buffers.expanding()); defer.push(() => messageChannel.close()); const errorChannel = eventChannel((emit) => { worker.addEventListener('error', emit); return () => worker.removeEventListener('error', emit); }, buffers.expanding()); defer.push(() => errorChannel.close()); // wait for the Python runtime to start and get in a ready state worker.postMessage(pythonMessageInit()); yield* put(editorCompletionInit()); for (;;) { const { messageEvent, errorEvent } = yield* race({ messageEvent: take(messageChannel), errorEvent: take(errorChannel), }); if (errorEvent) { yield* put(editorCompletionDidFailToInit()); throw errorEvent.error; } defined(messageEvent); if (pythonMessageDidMountUserFileSystem.matches(messageEvent.data)) { yield* fork(mirrorFileSystem, worker); continue; } if (pythonMessageDidFailToInit.matches(messageEvent.data)) { yield* put(editorCompletionDidFailToInit()); throw messageEvent.data.error; } if (pythonMessageDidInit.matches(messageEvent.data)) { break; } } console.debug('code completion engine is ready'); yield* put(editorCompletionDidInit()); // configure interrupts // https://pyodide.org/en/stable/usage/keyboard-interrupts.html // HACK: Using WebAssembly.Memory instead of SharedArrayBuffer to avoid exception. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer/Planned_changes#api_changes const interrupt = new Uint8Array( new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }).buffer, ); const setInterrupt = () => { interrupt[0] = 2; //2 === SIGINT }; const clearInterrupt = () => { interrupt[0] = 0; }; // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements if (crossOriginIsolated) { worker.postMessage(pythonMessageSetInterruptBuffer(interrupt)); } else { console.warn( 'required headers missing for SharedArrayBuffer, cancellation will not work', ); } // register intellisense hooks with editor const nextId = createCountFunc(); const completionItemProviderChan = eventChannel<{ model: monaco.editor.ITextModel; position: monaco.Position; context: monaco.languages.CompletionContext; token: monaco.CancellationToken; resolve: (value: monaco.languages.CompletionList | null) => void; }>((emit) => { const subscription = monaco.languages.registerCompletionItemProvider( pybricksMicroPythonId, { triggerCharacters: ['.'], provideCompletionItems( model, position, context, token, ): Promise { return new Promise((resolve) => { emit({ model, position, context, token, resolve }); }); }, }, ); return () => subscription.dispose(); }, buffers.expanding()); defer.push(() => completionItemProviderChan.close()); const signatureProviderChan = eventChannel<{ model: monaco.editor.ITextModel; position: monaco.Position; token: monaco.CancellationToken; context: monaco.languages.SignatureHelpContext; resolve: (value: monaco.languages.SignatureHelpResult | null) => void; }>((emit) => { const subscription = monaco.languages.registerSignatureHelpProvider( pybricksMicroPythonId, { signatureHelpTriggerCharacters: ['(', ','], signatureHelpRetriggerCharacters: [')'], provideSignatureHelp(model, position, token, context) { return new Promise((resolve) => { emit({ model, position, token, context, resolve }); }); }, }, ); return () => subscription.dispose(); }, buffers.expanding()); defer.push(() => signatureProviderChan.close()); // Serialize requests from editor. Due to the way cancellation works, we // can only have one pending message from the web worker at a time. for (;;) { const { complete, getSignature } = yield* race({ complete: take(completionItemProviderChan), getSignature: take(signatureProviderChan), }); if (complete) { // for debugging const id = nextId(); console.debug(`${id}: requested completion item`); const subscription = complete.token.onCancellationRequested(() => { console.debug(`${id}: requested cancelation`); setInterrupt(); }); try { clearInterrupt(); worker.postMessage( pythonMessageComplete( complete.model.getValue(), complete.position.lineNumber, complete.position.column, ), ); for (;;) { const msg = yield* take(messageChannel); if (pythonMessageDidFailToComplete.matches(msg.data)) { if ( msg.data.error instanceof DOMException && msg.data.error.name === 'AbortError' ) { console.log(`${id} canceled`); } else { console.error(msg.data.error); } complete.resolve(null); break; } if (pythonMessageDidComplete.matches(msg.data)) { const list: monaco.languages.CompletionItem[] = JSON.parse( msg.data.completionListJson, ); // maintain sort order from jedi for (const [i, item] of list.entries()) { item.sortText = String(i).padStart(5, '0'); } console.debug(list); complete.resolve({ suggestions: list }); console.debug(`${id}: resolved: ${msg.data.type}`); break; } } } finally { subscription.dispose(); } } else if (getSignature) { // for debugging const id = nextId(); console.debug(`${id}: requested signatures`); const subscription = getSignature.token.onCancellationRequested(() => { console.debug(`${id}: requested cancelation`); setInterrupt(); }); try { clearInterrupt(); worker.postMessage( pythonMessageGetSignature( getSignature.model.getValue(), getSignature.position.lineNumber, getSignature.position.column, ), ); for (;;) { const msg = yield* take(messageChannel); if (pythonMessageDidFailToGetSignature.matches(msg.data)) { if ( msg.data.error instanceof DOMException && msg.data.error.name === 'AbortError' ) { console.log(`${id} canceled`); } else { console.error(msg.data.error); } getSignature.resolve(null); break; } if (pythonMessageDidGetSignature.matches(msg.data)) { const signatures = JSON.parse(msg.data.signatureHelpJson); console.debug(signatures); getSignature.resolve({ value: signatures, dispose: () => undefined, }); console.debug(`${id}: resolved: ${msg.data.type}`); break; } } } finally { subscription.dispose(); } } } } catch (err) { const isCancelled = yield* cancelled(); if (!isCancelled) { console.error(err); } } finally { defer.forEach((item) => item()); } } export default function* (): Generator { yield* fork(monitorEditors); yield* fork(runJedi); }