pybricksMicropython: implement appending imports

This implements appending compiled imported modules for the multi-mpy6
file format.
This commit is contained in:
David Lechner
2022-10-19 18:26:37 -05:00
committed by David Lechner
parent 8b19ad5866
commit be9272ca9c
6 changed files with 410 additions and 22 deletions
+4
View File
@@ -38,6 +38,7 @@
"@types/web-locks-api": "^0.0.2",
"@types/wicg-file-system-access": "^2020.9.5",
"@types/zen-push": "^0.1.1",
"assert": "^2.0.0",
"babel-jest": "^29.2.1",
"babel-loader": "^8.2.3",
"babel-plugin-macros": "^3.0.1",
@@ -85,6 +86,7 @@
"prompts": "^2.4.2",
"prop-types": "^15.8.1",
"pyodide": "0.21.3",
"python-ast": "^0.1.0",
"react": "^16.13.1",
"react-app-polyfill": "^3.0.0",
"react-aria": "^3.20.0",
@@ -116,6 +118,7 @@
"typescript": "~4.8.4",
"usehooks-ts": "^2.9.1",
"user-agent-data-types": "^0.3.0",
"util": "^0.12.5",
"web-vitals": "^3.0.4",
"webpack": "^5.74.0",
"webpack-dev-server": "^4.11.1",
@@ -166,6 +169,7 @@
},
"packageManager": "yarn@3.2.0",
"resolutions": {
"antlr4ts@^0.5.0-alpha.3": "0.5.0-alpha.4",
"mq-polyfill@1.1.8": "patch:mq-polyfill@npm:1.1.8#.yarn/patches/mq-polyfill-npm-1.1.8-62fe162439.patch",
"react-error-overlay": "6.0.9",
"react-dev-utils@^12.0.1": "patch:react-dev-utils@npm:12.0.1#.yarn/patches/react-dev-utils-npm-12.0.1-83ba06e3ee.patch",
+1 -1
View File
@@ -29,7 +29,7 @@ export type FileMetadata = Readonly<{
*
* IMPORTANT: if this type is changed, we need to modify the database schema to match
*/
type FileContents = {
export type FileContents = {
/** The path of the file in storage. */
path: string;
/** The contents of the file. */
+96 -20
View File
@@ -7,7 +7,8 @@ import { compile as mpyCrossCompileV6 } from '@pybricks/mpy-cross-v6';
import wasmV6 from '@pybricks/mpy-cross-v6/build/mpy-cross-v6.wasm';
import { call, getContext, put, select, takeEvery } from 'typed-redux-saga/macro';
import { editorGetValue } from '../editor/sagas';
import { FileStorageDb } from '../fileStorage';
import { FileContents, FileStorageDb } from '../fileStorage';
import { findImportedModules, resolveModule } from '../pybricksMicropython/lib';
import { RootState } from '../reducers';
import {
compile,
@@ -20,6 +21,27 @@ import {
const encoder = new TextEncoder();
/**
* Converts JavaScript string to C string.
* @param str A string.
* @returns Zero-terminated, UTF-8 encoded byte array.
*/
function cString(str: string): Uint8Array {
return encoder.encode(str + '\x00');
}
/**
* Encodes *value* as a 32-bit unsigned integer in little endian order.
* @param value An integer between 0 and 2^32.
* @returns A 4-byte array containing the encoded valued.
*/
function encodeUInt32LE(value: number): ArrayBuffer {
const buf = new ArrayBuffer(4);
const view = new DataView(buf);
view.setUint32(0, value, true);
return buf;
}
/**
* Compiles a script to .mpy and dispatches either didCompile on success or
* didFailToCompile on error.
@@ -77,6 +99,12 @@ function* handleCompile(action: ReturnType<typeof compile>): Generator {
}
}
/**
* Compiles code into the Pybricks multi-mpy6 file format.
*
* This includes a __main__ module which is the file currently open in the
* editor and any imported modules that can be found in the user file system.
*/
function* handleCompileMulti6(): Generator {
// REVISIT: should we be getting the active file here or have it as an
// action parameter?
@@ -98,31 +126,79 @@ function* handleCompileMulti6(): Generator {
return;
}
const script = yield* editorGetValue();
const mainPy = yield* editorGetValue();
const result = yield* call(() =>
mpyCrossCompileV6(
metadata.path ?? '__main__.py',
script,
undefined,
// HACK: testing user agent for jsdom is needed only for getting unit tests to work
navigator.userAgent.includes('jsdom') ? undefined : wasmV6,
),
);
const pyFiles = new Map<string, FileContents>([
['__main__', { path: metadata.path ?? '__main__.py', contents: mainPy }],
]);
// TODO: add support for imports and append to blob
const checkedModules = new Set<string>(['__main__']);
const uncheckedScripts = new Array<string>(mainPy);
if (result.status === 0 && result.mpy) {
const sizeBuf = new ArrayBuffer(4);
const sizeView = new DataView(sizeBuf);
sizeView.setUint32(0, result.mpy.length, true);
for (;;) {
// parse all unchecked scripts to find imported modules that haven't
// been checked yet
const blob = new Blob([sizeBuf, encoder.encode('__main__\x00'), result.mpy]);
const uncheckedModules = new Set<string>();
yield* put(mpyDidCompileMulti6(blob));
} else {
yield* put(mpyDidFailToCompileMulti6(result.err));
for (const uncheckedScript of uncheckedScripts) {
const importedModules = findImportedModules(uncheckedScript);
for (const m of importedModules) {
if (!checkedModules.has(m)) {
uncheckedModules.add(m);
}
}
}
// all of the scripts have been checked now, so clear the unchecked list
uncheckedScripts.length = 0;
// when no more new modules are found, we are done
if (uncheckedModules.size === 0) {
break;
}
// try to resolve unchecked modules in the file system
for (const m of uncheckedModules) {
const file = yield* call(() => resolveModule(db, m));
// if found, queue the module to be compiled and to be parsed
// for additional imports
if (file) {
pyFiles.set(m, file);
uncheckedScripts.push(file.contents);
}
checkedModules.add(m);
}
}
const blobParts: BlobPart[] = [];
for (const [m, py] of pyFiles) {
const result = yield* call(() =>
mpyCrossCompileV6(
py.path,
py.contents,
undefined,
// HACK: testing user agent for jsdom is needed only for getting unit tests to work
navigator.userAgent.includes('jsdom') ? undefined : wasmV6,
),
);
if (result.status !== 0 || !result.mpy) {
yield* put(mpyDidFailToCompileMulti6(result.err));
return;
}
// each file is encoded as the size, module name, and mpy binary
blobParts.push(encodeUInt32LE(result.mpy.length));
blobParts.push(cString(m));
blobParts.push(result.mpy);
}
yield* put(mpyDidCompileMulti6(new Blob(blobParts)));
}
export default function* (): Generator {
+44
View File
@@ -3,6 +3,7 @@
import {
FileNameValidationResult,
findImportedModules,
pythonFileExtension,
pythonFileExtensionRegex,
validateFileName,
@@ -76,3 +77,46 @@ describe('validateFileName', () => {
);
});
});
test('findImportedModules', async () => {
const script = `
import a
import b, c
import d.d
import e.e as e
import f.f as f, g
from h import x
from h import x as y
from i import (x, y)
from i import (x as y, z)
from j import *
from . import x
from . import x as y
from .r import x
from ..r import x
from ...r import x
from ....r import x
`;
const modules = findImportedModules(script);
expect(modules).toEqual(
new Set([
'a',
'b',
'c',
'd.d',
'e.e',
'f.f',
'g',
'h',
'i',
'j',
'.',
'.r',
'..r',
'...r',
'....r',
]),
);
});
+78
View File
@@ -1,6 +1,9 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2022 The Pybricks Authors
import { parse, walk } from 'python-ast';
import type { FileContents, FileStorageDb } from '../fileStorage';
/** The Python file extension ('.py') */
export const pythonFileExtension = '.py';
@@ -67,3 +70,78 @@ export function validateFileName(
return FileNameValidationResult.IsOk;
}
/**
* Finds modules imported by a Python script.
*
* @param py A Python Script.
* @returns A list of the names of modules imported by this file.
*/
export function findImportedModules(py: string): ReadonlySet<string> {
const modules = new Set<string>();
const tree = parse(py);
// find all import statements in the syntax tree and collect imported modules
walk(
{
enterImport_stmt: (ctx) => {
// import statements have two forms:
// import_stmt: import_name | import_from;
// import_name: 'import' dotted_as_names
const name = ctx.import_name();
if (name) {
// dotted_as_names: dotted_as_name (',' dotted_as_name)*;
// dotted_as_name: dotted_name ('as' NAME)?;
for (const dottedAsName of name
.dotted_as_names()
.dotted_as_name()) {
// dotted_name: NAME ('.' NAME)*;
modules.add(dottedAsName.dotted_name().text);
}
}
// import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 'import' ('*' | '(' import_as_names ')' | import_as_names ));
const from = ctx.import_from();
if (from) {
// the leading dots aren't included in dotted_name, so
// we need to collect them separately
const leadingDots = from
.DOT()
.concat(from.ELLIPSIS())
.map((n) => n.symbol.text)
.join('');
// dotted_name: NAME ('.' NAME)*;
const dottedName = from.dotted_name()?.text ?? '';
modules.add(leadingDots + dottedName);
}
},
},
tree,
);
return modules;
}
export async function resolveModule(
db: FileStorageDb,
module: string,
): Promise<FileContents | undefined> {
const modulePath = module.replace(/\./g, '/') + '.py';
return await db.transaction('r', db.metadata, db._contents, async () => {
const match = await db.metadata.where('path').equals(modulePath).first();
if (!match) {
return undefined;
}
const file = await db._contents.get(match.path);
return file;
});
}
+187 -1
View File
@@ -2434,6 +2434,7 @@ __metadata:
"@types/zen-push": ^0.1.1
"@typescript-eslint/eslint-plugin": ^5.40.1
"@typescript-eslint/parser": ^5.40.1
assert: ^2.0.0
babel-jest: ^29.2.1
babel-loader: ^8.2.3
babel-plugin-macros: ^3.0.1
@@ -2492,6 +2493,7 @@ __metadata:
prompts: ^2.4.2
prop-types: ^15.8.1
pyodide: 0.21.3
python-ast: ^0.1.0
react: ^16.13.1
react-app-polyfill: ^3.0.0
react-aria: ^3.20.0
@@ -2523,6 +2525,7 @@ __metadata:
typescript: ~4.8.4
usehooks-ts: ^2.9.1
user-agent-data-types: ^0.3.0
util: ^0.12.5
web-vitals: ^3.0.4
webpack: ^5.74.0
webpack-dev-server: ^4.11.1
@@ -5483,6 +5486,13 @@ __metadata:
languageName: node
linkType: hard
"antlr4ts@npm:0.5.0-alpha.4":
version: 0.5.0-alpha.4
resolution: "antlr4ts@npm:0.5.0-alpha.4"
checksum: 37948499d59477f5b5a8ea71dfb8b5330e71d5a7cee60f57351dd744219b8619fa6aac1a5b6ec1a9991846e8ddc9ca47680eb166c59b44333369b3115e7aa358
languageName: node
linkType: hard
"anymatch@npm:^3.0.3, anymatch@npm:~3.1.2":
version: 3.1.2
resolution: "anymatch@npm:3.1.2"
@@ -5631,6 +5641,18 @@ __metadata:
languageName: node
linkType: hard
"assert@npm:^2.0.0":
version: 2.0.0
resolution: "assert@npm:2.0.0"
dependencies:
es6-object-assign: ^1.1.0
is-nan: ^1.2.1
object-is: ^1.0.1
util: ^0.12.0
checksum: bb91f181a86d10588ee16c5e09c280f9811373974c29974cbe401987ea34e966699d7989a812b0e19377b511ea0bc627f5905647ce569311824848ede382cae8
languageName: node
linkType: hard
"ast-types-flow@npm:^0.0.7":
version: 0.0.7
resolution: "ast-types-flow@npm:0.0.7"
@@ -5684,6 +5706,13 @@ __metadata:
languageName: node
linkType: hard
"available-typed-arrays@npm:^1.0.5":
version: 1.0.5
resolution: "available-typed-arrays@npm:1.0.5"
checksum: 20eb47b3cefd7db027b9bbb993c658abd36d4edd3fe1060e83699a03ee275b0c9b216cc076ff3f2db29073225fb70e7613987af14269ac1fe2a19803ccc97f1a
languageName: node
linkType: hard
"axe-core@npm:^4.3.5":
version: 4.4.1
resolution: "axe-core@npm:4.4.1"
@@ -7563,6 +7592,38 @@ __metadata:
languageName: node
linkType: hard
"es-abstract@npm:^1.20.0":
version: 1.20.4
resolution: "es-abstract@npm:1.20.4"
dependencies:
call-bind: ^1.0.2
es-to-primitive: ^1.2.1
function-bind: ^1.1.1
function.prototype.name: ^1.1.5
get-intrinsic: ^1.1.3
get-symbol-description: ^1.0.0
has: ^1.0.3
has-property-descriptors: ^1.0.0
has-symbols: ^1.0.3
internal-slot: ^1.0.3
is-callable: ^1.2.7
is-negative-zero: ^2.0.2
is-regex: ^1.1.4
is-shared-array-buffer: ^1.0.2
is-string: ^1.0.7
is-weakref: ^1.0.2
object-inspect: ^1.12.2
object-keys: ^1.1.1
object.assign: ^4.1.4
regexp.prototype.flags: ^1.4.3
safe-regex-test: ^1.0.0
string.prototype.trimend: ^1.0.5
string.prototype.trimstart: ^1.0.5
unbox-primitive: ^1.0.2
checksum: 89297cc785c31aedf961a603d5a07ed16471e435d3a1b6d070b54f157cf48454b95cda2ac55e4b86ff4fe3276e835fcffd2771578e6fa634337da49b26826141
languageName: node
linkType: hard
"es-module-lexer@npm:^0.9.0":
version: 0.9.3
resolution: "es-module-lexer@npm:0.9.3"
@@ -7590,6 +7651,13 @@ __metadata:
languageName: node
linkType: hard
"es6-object-assign@npm:^1.1.0":
version: 1.1.0
resolution: "es6-object-assign@npm:1.1.0"
checksum: 8d4fdf63484d78b5c64cacc2c2e1165bc7b6a64b739d2a9db6a4dc8641d99cc9efb433cdd4dc3d3d6b00bfa6ce959694e4665e3255190339945c5f33b692b5d8
languageName: node
linkType: hard
"escalade@npm:^3.1.1":
version: 3.1.1
resolution: "escalade@npm:3.1.1"
@@ -8394,6 +8462,15 @@ __metadata:
languageName: node
linkType: hard
"for-each@npm:^0.3.3":
version: 0.3.3
resolution: "for-each@npm:0.3.3"
dependencies:
is-callable: ^1.1.3
checksum: 6c48ff2bc63362319c65e2edca4a8e1e3483a2fabc72fbe7feaf8c73db94fc7861bd53bc02c8a66a0c1dd709da6b04eec42e0abdd6b40ce47305ae92a25e5d28
languageName: node
linkType: hard
"fork-ts-checker-webpack-plugin@npm:^6.5.0":
version: 6.5.2
resolution: "fork-ts-checker-webpack-plugin@npm:6.5.2"
@@ -8606,6 +8683,17 @@ __metadata:
languageName: node
linkType: hard
"get-intrinsic@npm:^1.1.3":
version: 1.1.3
resolution: "get-intrinsic@npm:1.1.3"
dependencies:
function-bind: ^1.1.1
has: ^1.0.3
has-symbols: ^1.0.3
checksum: 152d79e87251d536cf880ba75cfc3d6c6c50e12b3a64e1ea960e73a3752b47c69f46034456eae1b0894359ce3bc64c55c186f2811f8a788b75b638b06fab228a
languageName: node
linkType: hard
"get-own-enumerable-property-symbols@npm:^3.0.0":
version: 3.0.2
resolution: "get-own-enumerable-property-symbols@npm:3.0.2"
@@ -9305,6 +9393,13 @@ __metadata:
languageName: node
linkType: hard
"is-callable@npm:^1.1.3, is-callable@npm:^1.2.7":
version: 1.2.7
resolution: "is-callable@npm:1.2.7"
checksum: 61fd57d03b0d984e2ed3720fb1c7a897827ea174bd44402878e059542ea8c4aeedee0ea0985998aa5cc2736b2fa6e271c08587addb5b3959ac52cf665173d1ac
languageName: node
linkType: hard
"is-callable@npm:^1.1.4, is-callable@npm:^1.2.4":
version: 1.2.4
resolution: "is-callable@npm:1.2.4"
@@ -9360,6 +9455,15 @@ __metadata:
languageName: node
linkType: hard
"is-generator-function@npm:^1.0.7":
version: 1.0.10
resolution: "is-generator-function@npm:1.0.10"
dependencies:
has-tostringtag: ^1.0.0
checksum: d54644e7dbaccef15ceb1e5d91d680eb5068c9ee9f9eb0a9e04173eb5542c9b51b5ab52c5537f5703e48d5fddfd376817c1ca07a84a407b7115b769d4bdde72b
languageName: node
linkType: hard
"is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1":
version: 4.0.3
resolution: "is-glob@npm:4.0.3"
@@ -9397,6 +9501,16 @@ __metadata:
languageName: node
linkType: hard
"is-nan@npm:^1.2.1":
version: 1.3.2
resolution: "is-nan@npm:1.3.2"
dependencies:
call-bind: ^1.0.0
define-properties: ^1.1.3
checksum: 5dfadcef6ad12d3029d43643d9800adbba21cf3ce2ec849f734b0e14ee8da4070d82b15fdb35138716d02587c6578225b9a22779cab34888a139cc43e4e3610a
languageName: node
linkType: hard
"is-negative-zero@npm:^2.0.2":
version: 2.0.2
resolution: "is-negative-zero@npm:2.0.2"
@@ -9506,6 +9620,19 @@ __metadata:
languageName: node
linkType: hard
"is-typed-array@npm:^1.1.3, is-typed-array@npm:^1.1.9":
version: 1.1.9
resolution: "is-typed-array@npm:1.1.9"
dependencies:
available-typed-arrays: ^1.0.5
call-bind: ^1.0.2
es-abstract: ^1.20.0
for-each: ^0.3.3
has-tostringtag: ^1.0.0
checksum: 11910f1e58755fef43bf0074e52fa5b932bf101ec65d613e0a83d40e8e4c6e3f2ee142d624ebc7624c091d3bbe921131f8db7d36ecbbb71909f2fe310c1faa65
languageName: node
linkType: hard
"is-weakref@npm:^1.0.2":
version: 1.0.2
resolution: "is-weakref@npm:1.0.2"
@@ -11172,7 +11299,7 @@ __metadata:
languageName: node
linkType: hard
"object-inspect@npm:^1.12.0, object-inspect@npm:^1.9.0":
"object-inspect@npm:^1.12.0, object-inspect@npm:^1.12.2, object-inspect@npm:^1.9.0":
version: 1.12.2
resolution: "object-inspect@npm:1.12.2"
checksum: a534fc1b8534284ed71f25ce3a496013b7ea030f3d1b77118f6b7b1713829262be9e6243acbcb3ef8c626e2b64186112cb7f6db74e37b2789b9c789ca23048b2
@@ -11208,6 +11335,18 @@ __metadata:
languageName: node
linkType: hard
"object.assign@npm:^4.1.4":
version: 4.1.4
resolution: "object.assign@npm:4.1.4"
dependencies:
call-bind: ^1.0.2
define-properties: ^1.1.4
has-symbols: ^1.0.3
object-keys: ^1.1.1
checksum: 76cab513a5999acbfe0ff355f15a6a125e71805fcf53de4e9d4e082e1989bdb81d1e329291e1e4e0ae7719f0e4ef80e88fb2d367ae60500d79d25a6224ac8864
languageName: node
linkType: hard
"object.entries@npm:^1.1.5":
version: 1.1.5
resolution: "object.entries@npm:1.1.5"
@@ -12620,6 +12759,15 @@ __metadata:
languageName: node
linkType: hard
"python-ast@npm:^0.1.0":
version: 0.1.0
resolution: "python-ast@npm:0.1.0"
dependencies:
antlr4ts: ^0.5.0-alpha.3
checksum: 1fc453e89f932a1e7f6f57a0f9dc304252093e8aa6c934a426ef30c7e2c9fd0c05aad168fa3ad053ba3bb88362a2fb5097c6e9d360351539f5027ff2af0e5eab
languageName: node
linkType: hard
"qs@npm:6.10.3":
version: 6.10.3
resolution: "qs@npm:6.10.3"
@@ -13464,6 +13612,17 @@ __metadata:
languageName: node
linkType: hard
"safe-regex-test@npm:^1.0.0":
version: 1.0.0
resolution: "safe-regex-test@npm:1.0.0"
dependencies:
call-bind: ^1.0.2
get-intrinsic: ^1.1.3
is-regex: ^1.1.4
checksum: bc566d8beb8b43c01b94e67de3f070fd2781685e835959bbbaaec91cc53381145ca91f69bd837ce6ec244817afa0a5e974fc4e40a2957f0aca68ac3add1ddd34
languageName: node
linkType: hard
"safer-buffer@npm:>= 2.1.2 < 3, safer-buffer@npm:>= 2.1.2 < 3.0.0":
version: 2.1.2
resolution: "safer-buffer@npm:2.1.2"
@@ -14986,6 +15145,19 @@ __metadata:
languageName: node
linkType: hard
"util@npm:^0.12.0, util@npm:^0.12.5":
version: 0.12.5
resolution: "util@npm:0.12.5"
dependencies:
inherits: ^2.0.3
is-arguments: ^1.0.4
is-generator-function: ^1.0.7
is-typed-array: ^1.1.3
which-typed-array: ^1.1.2
checksum: 705e51f0de5b446f4edec10739752ac25856541e0254ea1e7e45e5b9f9b0cb105bc4bd415736a6210edc68245a7f903bf085ffb08dd7deb8a0e847f60538a38a
languageName: node
linkType: hard
"utila@npm:~0.4":
version: 0.4.0
resolution: "utila@npm:0.4.0"
@@ -15348,6 +15520,20 @@ __metadata:
languageName: node
linkType: hard
"which-typed-array@npm:^1.1.2":
version: 1.1.8
resolution: "which-typed-array@npm:1.1.8"
dependencies:
available-typed-arrays: ^1.0.5
call-bind: ^1.0.2
es-abstract: ^1.20.0
for-each: ^0.3.3
has-tostringtag: ^1.0.0
is-typed-array: ^1.1.9
checksum: bedf4d30a738e848404fe67fe0ace33433a7298cf3f5a4d4b2c624ba99c4d25f06a7fd6f3566c3d16af5f8a54f0c6293cbfded5b1208ce11812753990223b45a
languageName: node
linkType: hard
"which@npm:^1.3.1":
version: 1.3.1
resolution: "which@npm:1.3.1"