diff --git a/package.json b/package.json index 1dbe6e3a..326f734a 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@types/web-locks-api": "^0.0.2", "@types/wicg-file-system-access": "^2020.9.5", "@types/zen-push": "^0.1.1", + "assert": "^2.0.0", "babel-jest": "^29.2.1", "babel-loader": "^8.2.3", "babel-plugin-macros": "^3.0.1", @@ -85,6 +86,7 @@ "prompts": "^2.4.2", "prop-types": "^15.8.1", "pyodide": "0.21.3", + "python-ast": "^0.1.0", "react": "^16.13.1", "react-app-polyfill": "^3.0.0", "react-aria": "^3.20.0", @@ -116,6 +118,7 @@ "typescript": "~4.8.4", "usehooks-ts": "^2.9.1", "user-agent-data-types": "^0.3.0", + "util": "^0.12.5", "web-vitals": "^3.0.4", "webpack": "^5.74.0", "webpack-dev-server": "^4.11.1", @@ -166,6 +169,7 @@ }, "packageManager": "yarn@3.2.0", "resolutions": { + "antlr4ts@^0.5.0-alpha.3": "0.5.0-alpha.4", "mq-polyfill@1.1.8": "patch:mq-polyfill@npm:1.1.8#.yarn/patches/mq-polyfill-npm-1.1.8-62fe162439.patch", "react-error-overlay": "6.0.9", "react-dev-utils@^12.0.1": "patch:react-dev-utils@npm:12.0.1#.yarn/patches/react-dev-utils-npm-12.0.1-83ba06e3ee.patch", diff --git a/src/fileStorage/index.ts b/src/fileStorage/index.ts index f066cead..51e25b11 100644 --- a/src/fileStorage/index.ts +++ b/src/fileStorage/index.ts @@ -29,7 +29,7 @@ export type FileMetadata = Readonly<{ * * IMPORTANT: if this type is changed, we need to modify the database schema to match */ -type FileContents = { +export type FileContents = { /** The path of the file in storage. */ path: string; /** The contents of the file. */ diff --git a/src/mpy/sagas.ts b/src/mpy/sagas.ts index 08b9817a..498cfa7f 100644 --- a/src/mpy/sagas.ts +++ b/src/mpy/sagas.ts @@ -7,7 +7,8 @@ import { compile as mpyCrossCompileV6 } from '@pybricks/mpy-cross-v6'; import wasmV6 from '@pybricks/mpy-cross-v6/build/mpy-cross-v6.wasm'; import { call, getContext, put, select, takeEvery } from 'typed-redux-saga/macro'; import { editorGetValue } from '../editor/sagas'; -import { FileStorageDb } from '../fileStorage'; +import { FileContents, FileStorageDb } from '../fileStorage'; +import { findImportedModules, resolveModule } from '../pybricksMicropython/lib'; import { RootState } from '../reducers'; import { compile, @@ -20,6 +21,27 @@ import { const encoder = new TextEncoder(); +/** + * Converts JavaScript string to C string. + * @param str A string. + * @returns Zero-terminated, UTF-8 encoded byte array. + */ +function cString(str: string): Uint8Array { + return encoder.encode(str + '\x00'); +} + +/** + * Encodes *value* as a 32-bit unsigned integer in little endian order. + * @param value An integer between 0 and 2^32. + * @returns A 4-byte array containing the encoded valued. + */ +function encodeUInt32LE(value: number): ArrayBuffer { + const buf = new ArrayBuffer(4); + const view = new DataView(buf); + view.setUint32(0, value, true); + return buf; +} + /** * Compiles a script to .mpy and dispatches either didCompile on success or * didFailToCompile on error. @@ -77,6 +99,12 @@ function* handleCompile(action: ReturnType): Generator { } } +/** + * Compiles code into the Pybricks multi-mpy6 file format. + * + * This includes a __main__ module which is the file currently open in the + * editor and any imported modules that can be found in the user file system. + */ function* handleCompileMulti6(): Generator { // REVISIT: should we be getting the active file here or have it as an // action parameter? @@ -98,31 +126,79 @@ function* handleCompileMulti6(): Generator { return; } - const script = yield* editorGetValue(); + const mainPy = yield* editorGetValue(); - const result = yield* call(() => - mpyCrossCompileV6( - metadata.path ?? '__main__.py', - script, - undefined, - // HACK: testing user agent for jsdom is needed only for getting unit tests to work - navigator.userAgent.includes('jsdom') ? undefined : wasmV6, - ), - ); + const pyFiles = new Map([ + ['__main__', { path: metadata.path ?? '__main__.py', contents: mainPy }], + ]); - // TODO: add support for imports and append to blob + const checkedModules = new Set(['__main__']); + const uncheckedScripts = new Array(mainPy); - if (result.status === 0 && result.mpy) { - const sizeBuf = new ArrayBuffer(4); - const sizeView = new DataView(sizeBuf); - sizeView.setUint32(0, result.mpy.length, true); + for (;;) { + // parse all unchecked scripts to find imported modules that haven't + // been checked yet - const blob = new Blob([sizeBuf, encoder.encode('__main__\x00'), result.mpy]); + const uncheckedModules = new Set(); - yield* put(mpyDidCompileMulti6(blob)); - } else { - yield* put(mpyDidFailToCompileMulti6(result.err)); + for (const uncheckedScript of uncheckedScripts) { + const importedModules = findImportedModules(uncheckedScript); + + for (const m of importedModules) { + if (!checkedModules.has(m)) { + uncheckedModules.add(m); + } + } + } + + // all of the scripts have been checked now, so clear the unchecked list + uncheckedScripts.length = 0; + + // when no more new modules are found, we are done + if (uncheckedModules.size === 0) { + break; + } + + // try to resolve unchecked modules in the file system + for (const m of uncheckedModules) { + const file = yield* call(() => resolveModule(db, m)); + + // if found, queue the module to be compiled and to be parsed + // for additional imports + if (file) { + pyFiles.set(m, file); + uncheckedScripts.push(file.contents); + } + + checkedModules.add(m); + } } + + const blobParts: BlobPart[] = []; + + for (const [m, py] of pyFiles) { + const result = yield* call(() => + mpyCrossCompileV6( + py.path, + py.contents, + undefined, + // HACK: testing user agent for jsdom is needed only for getting unit tests to work + navigator.userAgent.includes('jsdom') ? undefined : wasmV6, + ), + ); + + if (result.status !== 0 || !result.mpy) { + yield* put(mpyDidFailToCompileMulti6(result.err)); + return; + } + + // each file is encoded as the size, module name, and mpy binary + blobParts.push(encodeUInt32LE(result.mpy.length)); + blobParts.push(cString(m)); + blobParts.push(result.mpy); + } + + yield* put(mpyDidCompileMulti6(new Blob(blobParts))); } export default function* (): Generator { diff --git a/src/pybricksMicropython/lib.test.ts b/src/pybricksMicropython/lib.test.ts index f480d095..abed405c 100644 --- a/src/pybricksMicropython/lib.test.ts +++ b/src/pybricksMicropython/lib.test.ts @@ -3,6 +3,7 @@ import { FileNameValidationResult, + findImportedModules, pythonFileExtension, pythonFileExtensionRegex, validateFileName, @@ -76,3 +77,46 @@ describe('validateFileName', () => { ); }); }); + +test('findImportedModules', async () => { + const script = ` +import a +import b, c +import d.d +import e.e as e +import f.f as f, g +from h import x +from h import x as y +from i import (x, y) +from i import (x as y, z) +from j import * +from . import x +from . import x as y +from .r import x +from ..r import x +from ...r import x +from ....r import x +`; + + const modules = findImportedModules(script); + + expect(modules).toEqual( + new Set([ + 'a', + 'b', + 'c', + 'd.d', + 'e.e', + 'f.f', + 'g', + 'h', + 'i', + 'j', + '.', + '.r', + '..r', + '...r', + '....r', + ]), + ); +}); diff --git a/src/pybricksMicropython/lib.ts b/src/pybricksMicropython/lib.ts index 1360fb6d..ab0d6330 100644 --- a/src/pybricksMicropython/lib.ts +++ b/src/pybricksMicropython/lib.ts @@ -1,6 +1,9 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2022 The Pybricks Authors +import { parse, walk } from 'python-ast'; +import type { FileContents, FileStorageDb } from '../fileStorage'; + /** The Python file extension ('.py') */ export const pythonFileExtension = '.py'; @@ -67,3 +70,78 @@ export function validateFileName( return FileNameValidationResult.IsOk; } + +/** + * Finds modules imported by a Python script. + * + * @param py A Python Script. + * @returns A list of the names of modules imported by this file. + */ +export function findImportedModules(py: string): ReadonlySet { + const modules = new Set(); + const tree = parse(py); + + // find all import statements in the syntax tree and collect imported modules + walk( + { + enterImport_stmt: (ctx) => { + // import statements have two forms: + // import_stmt: import_name | import_from; + + // import_name: 'import' dotted_as_names + const name = ctx.import_name(); + + if (name) { + // dotted_as_names: dotted_as_name (',' dotted_as_name)*; + // dotted_as_name: dotted_name ('as' NAME)?; + for (const dottedAsName of name + .dotted_as_names() + .dotted_as_name()) { + // dotted_name: NAME ('.' NAME)*; + modules.add(dottedAsName.dotted_name().text); + } + } + + // import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 'import' ('*' | '(' import_as_names ')' | import_as_names )); + const from = ctx.import_from(); + + if (from) { + // the leading dots aren't included in dotted_name, so + // we need to collect them separately + const leadingDots = from + .DOT() + .concat(from.ELLIPSIS()) + .map((n) => n.symbol.text) + .join(''); + + // dotted_name: NAME ('.' NAME)*; + const dottedName = from.dotted_name()?.text ?? ''; + + modules.add(leadingDots + dottedName); + } + }, + }, + tree, + ); + + return modules; +} + +export async function resolveModule( + db: FileStorageDb, + module: string, +): Promise { + const modulePath = module.replace(/\./g, '/') + '.py'; + + return await db.transaction('r', db.metadata, db._contents, async () => { + const match = await db.metadata.where('path').equals(modulePath).first(); + + if (!match) { + return undefined; + } + + const file = await db._contents.get(match.path); + + return file; + }); +} diff --git a/yarn.lock b/yarn.lock index 2b498230..ff9f7442 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2434,6 +2434,7 @@ __metadata: "@types/zen-push": ^0.1.1 "@typescript-eslint/eslint-plugin": ^5.40.1 "@typescript-eslint/parser": ^5.40.1 + assert: ^2.0.0 babel-jest: ^29.2.1 babel-loader: ^8.2.3 babel-plugin-macros: ^3.0.1 @@ -2492,6 +2493,7 @@ __metadata: prompts: ^2.4.2 prop-types: ^15.8.1 pyodide: 0.21.3 + python-ast: ^0.1.0 react: ^16.13.1 react-app-polyfill: ^3.0.0 react-aria: ^3.20.0 @@ -2523,6 +2525,7 @@ __metadata: typescript: ~4.8.4 usehooks-ts: ^2.9.1 user-agent-data-types: ^0.3.0 + util: ^0.12.5 web-vitals: ^3.0.4 webpack: ^5.74.0 webpack-dev-server: ^4.11.1 @@ -5483,6 +5486,13 @@ __metadata: languageName: node linkType: hard +"antlr4ts@npm:0.5.0-alpha.4": + version: 0.5.0-alpha.4 + resolution: "antlr4ts@npm:0.5.0-alpha.4" + checksum: 37948499d59477f5b5a8ea71dfb8b5330e71d5a7cee60f57351dd744219b8619fa6aac1a5b6ec1a9991846e8ddc9ca47680eb166c59b44333369b3115e7aa358 + languageName: node + linkType: hard + "anymatch@npm:^3.0.3, anymatch@npm:~3.1.2": version: 3.1.2 resolution: "anymatch@npm:3.1.2" @@ -5631,6 +5641,18 @@ __metadata: languageName: node linkType: hard +"assert@npm:^2.0.0": + version: 2.0.0 + resolution: "assert@npm:2.0.0" + dependencies: + es6-object-assign: ^1.1.0 + is-nan: ^1.2.1 + object-is: ^1.0.1 + util: ^0.12.0 + checksum: bb91f181a86d10588ee16c5e09c280f9811373974c29974cbe401987ea34e966699d7989a812b0e19377b511ea0bc627f5905647ce569311824848ede382cae8 + languageName: node + linkType: hard + "ast-types-flow@npm:^0.0.7": version: 0.0.7 resolution: "ast-types-flow@npm:0.0.7" @@ -5684,6 +5706,13 @@ __metadata: languageName: node linkType: hard +"available-typed-arrays@npm:^1.0.5": + version: 1.0.5 + resolution: "available-typed-arrays@npm:1.0.5" + checksum: 20eb47b3cefd7db027b9bbb993c658abd36d4edd3fe1060e83699a03ee275b0c9b216cc076ff3f2db29073225fb70e7613987af14269ac1fe2a19803ccc97f1a + languageName: node + linkType: hard + "axe-core@npm:^4.3.5": version: 4.4.1 resolution: "axe-core@npm:4.4.1" @@ -7563,6 +7592,38 @@ __metadata: languageName: node linkType: hard +"es-abstract@npm:^1.20.0": + version: 1.20.4 + resolution: "es-abstract@npm:1.20.4" + dependencies: + call-bind: ^1.0.2 + es-to-primitive: ^1.2.1 + function-bind: ^1.1.1 + function.prototype.name: ^1.1.5 + get-intrinsic: ^1.1.3 + get-symbol-description: ^1.0.0 + has: ^1.0.3 + has-property-descriptors: ^1.0.0 + has-symbols: ^1.0.3 + internal-slot: ^1.0.3 + is-callable: ^1.2.7 + is-negative-zero: ^2.0.2 + is-regex: ^1.1.4 + is-shared-array-buffer: ^1.0.2 + is-string: ^1.0.7 + is-weakref: ^1.0.2 + object-inspect: ^1.12.2 + object-keys: ^1.1.1 + object.assign: ^4.1.4 + regexp.prototype.flags: ^1.4.3 + safe-regex-test: ^1.0.0 + string.prototype.trimend: ^1.0.5 + string.prototype.trimstart: ^1.0.5 + unbox-primitive: ^1.0.2 + checksum: 89297cc785c31aedf961a603d5a07ed16471e435d3a1b6d070b54f157cf48454b95cda2ac55e4b86ff4fe3276e835fcffd2771578e6fa634337da49b26826141 + languageName: node + linkType: hard + "es-module-lexer@npm:^0.9.0": version: 0.9.3 resolution: "es-module-lexer@npm:0.9.3" @@ -7590,6 +7651,13 @@ __metadata: languageName: node linkType: hard +"es6-object-assign@npm:^1.1.0": + version: 1.1.0 + resolution: "es6-object-assign@npm:1.1.0" + checksum: 8d4fdf63484d78b5c64cacc2c2e1165bc7b6a64b739d2a9db6a4dc8641d99cc9efb433cdd4dc3d3d6b00bfa6ce959694e4665e3255190339945c5f33b692b5d8 + languageName: node + linkType: hard + "escalade@npm:^3.1.1": version: 3.1.1 resolution: "escalade@npm:3.1.1" @@ -8394,6 +8462,15 @@ __metadata: languageName: node linkType: hard +"for-each@npm:^0.3.3": + version: 0.3.3 + resolution: "for-each@npm:0.3.3" + dependencies: + is-callable: ^1.1.3 + checksum: 6c48ff2bc63362319c65e2edca4a8e1e3483a2fabc72fbe7feaf8c73db94fc7861bd53bc02c8a66a0c1dd709da6b04eec42e0abdd6b40ce47305ae92a25e5d28 + languageName: node + linkType: hard + "fork-ts-checker-webpack-plugin@npm:^6.5.0": version: 6.5.2 resolution: "fork-ts-checker-webpack-plugin@npm:6.5.2" @@ -8606,6 +8683,17 @@ __metadata: languageName: node linkType: hard +"get-intrinsic@npm:^1.1.3": + version: 1.1.3 + resolution: "get-intrinsic@npm:1.1.3" + dependencies: + function-bind: ^1.1.1 + has: ^1.0.3 + has-symbols: ^1.0.3 + checksum: 152d79e87251d536cf880ba75cfc3d6c6c50e12b3a64e1ea960e73a3752b47c69f46034456eae1b0894359ce3bc64c55c186f2811f8a788b75b638b06fab228a + languageName: node + linkType: hard + "get-own-enumerable-property-symbols@npm:^3.0.0": version: 3.0.2 resolution: "get-own-enumerable-property-symbols@npm:3.0.2" @@ -9305,6 +9393,13 @@ __metadata: languageName: node linkType: hard +"is-callable@npm:^1.1.3, is-callable@npm:^1.2.7": + version: 1.2.7 + resolution: "is-callable@npm:1.2.7" + checksum: 61fd57d03b0d984e2ed3720fb1c7a897827ea174bd44402878e059542ea8c4aeedee0ea0985998aa5cc2736b2fa6e271c08587addb5b3959ac52cf665173d1ac + languageName: node + linkType: hard + "is-callable@npm:^1.1.4, is-callable@npm:^1.2.4": version: 1.2.4 resolution: "is-callable@npm:1.2.4" @@ -9360,6 +9455,15 @@ __metadata: languageName: node linkType: hard +"is-generator-function@npm:^1.0.7": + version: 1.0.10 + resolution: "is-generator-function@npm:1.0.10" + dependencies: + has-tostringtag: ^1.0.0 + checksum: d54644e7dbaccef15ceb1e5d91d680eb5068c9ee9f9eb0a9e04173eb5542c9b51b5ab52c5537f5703e48d5fddfd376817c1ca07a84a407b7115b769d4bdde72b + languageName: node + linkType: hard + "is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1": version: 4.0.3 resolution: "is-glob@npm:4.0.3" @@ -9397,6 +9501,16 @@ __metadata: languageName: node linkType: hard +"is-nan@npm:^1.2.1": + version: 1.3.2 + resolution: "is-nan@npm:1.3.2" + dependencies: + call-bind: ^1.0.0 + define-properties: ^1.1.3 + checksum: 5dfadcef6ad12d3029d43643d9800adbba21cf3ce2ec849f734b0e14ee8da4070d82b15fdb35138716d02587c6578225b9a22779cab34888a139cc43e4e3610a + languageName: node + linkType: hard + "is-negative-zero@npm:^2.0.2": version: 2.0.2 resolution: "is-negative-zero@npm:2.0.2" @@ -9506,6 +9620,19 @@ __metadata: languageName: node linkType: hard +"is-typed-array@npm:^1.1.3, is-typed-array@npm:^1.1.9": + version: 1.1.9 + resolution: "is-typed-array@npm:1.1.9" + dependencies: + available-typed-arrays: ^1.0.5 + call-bind: ^1.0.2 + es-abstract: ^1.20.0 + for-each: ^0.3.3 + has-tostringtag: ^1.0.0 + checksum: 11910f1e58755fef43bf0074e52fa5b932bf101ec65d613e0a83d40e8e4c6e3f2ee142d624ebc7624c091d3bbe921131f8db7d36ecbbb71909f2fe310c1faa65 + languageName: node + linkType: hard + "is-weakref@npm:^1.0.2": version: 1.0.2 resolution: "is-weakref@npm:1.0.2" @@ -11172,7 +11299,7 @@ __metadata: languageName: node linkType: hard -"object-inspect@npm:^1.12.0, object-inspect@npm:^1.9.0": +"object-inspect@npm:^1.12.0, object-inspect@npm:^1.12.2, object-inspect@npm:^1.9.0": version: 1.12.2 resolution: "object-inspect@npm:1.12.2" checksum: a534fc1b8534284ed71f25ce3a496013b7ea030f3d1b77118f6b7b1713829262be9e6243acbcb3ef8c626e2b64186112cb7f6db74e37b2789b9c789ca23048b2 @@ -11208,6 +11335,18 @@ __metadata: languageName: node linkType: hard +"object.assign@npm:^4.1.4": + version: 4.1.4 + resolution: "object.assign@npm:4.1.4" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.4 + has-symbols: ^1.0.3 + object-keys: ^1.1.1 + checksum: 76cab513a5999acbfe0ff355f15a6a125e71805fcf53de4e9d4e082e1989bdb81d1e329291e1e4e0ae7719f0e4ef80e88fb2d367ae60500d79d25a6224ac8864 + languageName: node + linkType: hard + "object.entries@npm:^1.1.5": version: 1.1.5 resolution: "object.entries@npm:1.1.5" @@ -12620,6 +12759,15 @@ __metadata: languageName: node linkType: hard +"python-ast@npm:^0.1.0": + version: 0.1.0 + resolution: "python-ast@npm:0.1.0" + dependencies: + antlr4ts: ^0.5.0-alpha.3 + checksum: 1fc453e89f932a1e7f6f57a0f9dc304252093e8aa6c934a426ef30c7e2c9fd0c05aad168fa3ad053ba3bb88362a2fb5097c6e9d360351539f5027ff2af0e5eab + languageName: node + linkType: hard + "qs@npm:6.10.3": version: 6.10.3 resolution: "qs@npm:6.10.3" @@ -13464,6 +13612,17 @@ __metadata: languageName: node linkType: hard +"safe-regex-test@npm:^1.0.0": + version: 1.0.0 + resolution: "safe-regex-test@npm:1.0.0" + dependencies: + call-bind: ^1.0.2 + get-intrinsic: ^1.1.3 + is-regex: ^1.1.4 + checksum: bc566d8beb8b43c01b94e67de3f070fd2781685e835959bbbaaec91cc53381145ca91f69bd837ce6ec244817afa0a5e974fc4e40a2957f0aca68ac3add1ddd34 + languageName: node + linkType: hard + "safer-buffer@npm:>= 2.1.2 < 3, safer-buffer@npm:>= 2.1.2 < 3.0.0": version: 2.1.2 resolution: "safer-buffer@npm:2.1.2" @@ -14986,6 +15145,19 @@ __metadata: languageName: node linkType: hard +"util@npm:^0.12.0, util@npm:^0.12.5": + version: 0.12.5 + resolution: "util@npm:0.12.5" + dependencies: + inherits: ^2.0.3 + is-arguments: ^1.0.4 + is-generator-function: ^1.0.7 + is-typed-array: ^1.1.3 + which-typed-array: ^1.1.2 + checksum: 705e51f0de5b446f4edec10739752ac25856541e0254ea1e7e45e5b9f9b0cb105bc4bd415736a6210edc68245a7f903bf085ffb08dd7deb8a0e847f60538a38a + languageName: node + linkType: hard + "utila@npm:~0.4": version: 0.4.0 resolution: "utila@npm:0.4.0" @@ -15348,6 +15520,20 @@ __metadata: languageName: node linkType: hard +"which-typed-array@npm:^1.1.2": + version: 1.1.8 + resolution: "which-typed-array@npm:1.1.8" + dependencies: + available-typed-arrays: ^1.0.5 + call-bind: ^1.0.2 + es-abstract: ^1.20.0 + for-each: ^0.3.3 + has-tostringtag: ^1.0.0 + is-typed-array: ^1.1.9 + checksum: bedf4d30a738e848404fe67fe0ace33433a7298cf3f5a4d4b2c624ba99c4d25f06a7fd6f3566c3d16af5f8a54f0c6293cbfded5b1208ce11812753990223b45a + languageName: node + linkType: hard + "which@npm:^1.3.1": version: 1.3.1 resolution: "which@npm:1.3.1"