Michel2/pybricks-code
1
0
Fork 0
mirror of https://github.com/pybricks/pybricks-code.git synced 2026-09-12 17:45:22 +00:00
Code Issues Releases Activity

pybricksMicropython/lib: replace parser

Browse Source 
This replaces the Python parsing library. The python-ast package has
really poor performance (many seconds to parse a 500 line file). The
new package can parse a file about 100 times faster.
This commit is contained in:
David Lechner
2022-10-24 15:21:52 -05:00
parent a673a45b03
commit 0e2a0f909a
6 changed files with 40 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+5
View File
@@ -4,6 +4,11 @@
## [Unreleased]
### Fixed
- Fixed UI temporary freeze when downloading and running programs ([support#750]).
[support#750]: https://github.com/pybricks/support/issues/750
## [2.0.0-beta.6] - 2022-10-21
### Added
config/webpack.config.js
+1
View File
@@ -802,5 +802,6 @@ module.exports = function (webpackEnv) {
// Turn off performance processing because we utilize
// our own hints via the FileSizeReporter
performance: false,
ignoreWarnings: [/Failed to parse source map/],
};
};
package.json
+1 -4
View File
@@ -18,6 +18,7 @@
"@pybricks/jedi": "^1.2.0",
"@pybricks/mpy-cross-v5": "^2.0.0",
"@pybricks/mpy-cross-v6": "^2.0.0",
"@qoretechnologies/python-parser": "^0.4.10",
"@reduxjs/toolkit": "^1.8.6",
"@shopify/react-i18n": "^7.2.4",
"@svgr/webpack": "^6.5.0",
@@ -38,7 +39,6 @@
"@types/web-locks-api": "^0.0.2",
"@types/wicg-file-system-access": "^2020.9.5",
"@types/zen-push": "^0.1.1",
"assert": "^2.0.0",
"babel-jest": "^29.2.1",
"babel-loader": "^8.2.3",
"babel-plugin-macros": "^3.0.1",
@@ -86,7 +86,6 @@
"prompts": "^2.4.2",
"prop-types": "^15.8.1",
"pyodide": "0.21.3",
"python-ast": "^0.1.0",
"react": "^16.13.1",
"react-app-polyfill": "^3.0.0",
"react-aria": "^3.20.0",
@@ -118,7 +117,6 @@
"typescript": "~4.8.4",
"usehooks-ts": "^2.9.1",
"user-agent-data-types": "^0.3.0",
"util": "^0.12.5",
"web-vitals": "^3.0.4",
"webpack": "^5.74.0",
"webpack-dev-server": "^4.11.1",
@@ -169,7 +167,6 @@
},
"packageManager": "yarn@3.2.0",
"resolutions": {
"antlr4ts@^0.5.0-alpha.3": "0.5.0-alpha.4",
"mq-polyfill@1.1.8": "patch:mq-polyfill@npm:1.1.8#.yarn/patches/mq-polyfill-npm-1.1.8-62fe162439.patch",
"react-error-overlay": "6.0.9",
"react-dev-utils@^12.0.1": "patch:react-dev-utils@npm:12.0.1#.yarn/patches/react-dev-utils-npm-12.0.1-83ba06e3ee.patch",
src/pybricksMicropython/lib.test.ts
+11
View File
@@ -96,6 +96,17 @@ from .r import x
from ..r import x
from ...r import x
from ....r import x
# import q
# from q import q
"""
import q
from q import q
"""
'''
import q
from q import q
'''
`;
const modules = findImportedModules(script);
src/pybricksMicropython/lib.ts
+10 -40
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2022 The Pybricks Authors
import { parse, walk } from 'python-ast';
import { parse, walk } from '@qoretechnologies/python-parser';
import type { FileContents, FileStorageDb } from '../fileStorage';
/** The Python file extension ('.py') */
@@ -82,47 +82,17 @@ export function findImportedModules(py: string): ReadonlySet<string> {
const tree = parse(py);
// find all import statements in the syntax tree and collect imported modules
walk(
{
enterImport_stmt: (ctx) => {
// import statements have two forms:
// import_stmt: import_name | import_from;
// import_name: 'import' dotted_as_names
const name = ctx.import_name();
if (name) {
// dotted_as_names: dotted_as_name (',' dotted_as_name)*;
// dotted_as_name: dotted_name ('as' NAME)?;
for (const dottedAsName of name
.dotted_as_names()
.dotted_as_name()) {
// dotted_name: NAME ('.' NAME)*;
modules.add(dottedAsName.dotted_name().text);
}
walk(tree, {
onEnterNode(node, _ancestors) {
if (node.type === 'import') {
for (const name of node.names) {
modules.add(name.path);
}
// import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 'import' ('*' | '(' import_as_names ')' | import_as_names ));
const from = ctx.import_from();
if (from) {
// the leading dots aren't included in dotted_name, so
// we need to collect them separately
const leadingDots = from
.DOT()
.concat(from.ELLIPSIS())
.map((n) => n.symbol.text)
.join('');
// dotted_name: NAME ('.' NAME)*;
const dottedName = from.dotted_name()?.text ?? '';
modules.add(leadingDots + dottedName);
}
},
} else if (node.type === 'from') {
modules.add(node.base);
}
},
tree,
);
});
return modules;
}
yarn.lock
+12 -115
View File
@@ -2412,6 +2412,7 @@ __metadata:
"@pybricks/jedi": ^1.2.0
"@pybricks/mpy-cross-v5": ^2.0.0
"@pybricks/mpy-cross-v6": ^2.0.0
"@qoretechnologies/python-parser": ^0.4.10
"@reduxjs/toolkit": ^1.8.6
"@shopify/react-i18n": ^7.2.4
"@svgr/webpack": ^6.5.0
@@ -2434,7 +2435,6 @@ __metadata:
"@types/zen-push": ^0.1.1
"@typescript-eslint/eslint-plugin": ^5.40.1
"@typescript-eslint/parser": ^5.40.1
assert: ^2.0.0
babel-jest: ^29.2.1
babel-loader: ^8.2.3
babel-plugin-macros: ^3.0.1
@@ -2493,7 +2493,6 @@ __metadata:
prompts: ^2.4.2
prop-types: ^15.8.1
pyodide: 0.21.3
python-ast: ^0.1.0
react: ^16.13.1
react-app-polyfill: ^3.0.0
react-aria: ^3.20.0
@@ -2525,7 +2524,6 @@ __metadata:
typescript: ~4.8.4
usehooks-ts: ^2.9.1
user-agent-data-types: ^0.3.0
util: ^0.12.5
web-vitals: ^3.0.4
webpack: ^5.74.0
webpack-dev-server: ^4.11.1
@@ -2537,6 +2535,15 @@ __metadata:
languageName: unknown
linkType: soft
"@qoretechnologies/python-parser@npm:^0.4.10":
version: 0.4.10
resolution: "@qoretechnologies/python-parser@npm:0.4.10"
dependencies:
lodash: ^4.17.15
checksum: ab0d91719a8301b64f9c7fb1a7711721d52d78325abbf0c7c75940a48fd4a3d2b07456c95f8507af19faa333bf87116dd62f18e6c8e76486a57673e3224ac3ee
languageName: node
linkType: hard
"@react-aria/breadcrumbs@npm:^3.3.2":
version: 3.3.2
resolution: "@react-aria/breadcrumbs@npm:3.3.2"
@@ -5425,13 +5432,6 @@ __metadata:
languageName: node
linkType: hard
"antlr4ts@npm:0.5.0-alpha.4":
version: 0.5.0-alpha.4
resolution: "antlr4ts@npm:0.5.0-alpha.4"
checksum: 37948499d59477f5b5a8ea71dfb8b5330e71d5a7cee60f57351dd744219b8619fa6aac1a5b6ec1a9991846e8ddc9ca47680eb166c59b44333369b3115e7aa358
languageName: node
linkType: hard
"anymatch@npm:^3.0.3, anymatch@npm:~3.1.2":
version: 3.1.2
resolution: "anymatch@npm:3.1.2"
@@ -5580,18 +5580,6 @@ __metadata:
languageName: node
linkType: hard
"assert@npm:^2.0.0":
version: 2.0.0
resolution: "assert@npm:2.0.0"
dependencies:
es6-object-assign: ^1.1.0
is-nan: ^1.2.1
object-is: ^1.0.1
util: ^0.12.0
checksum: bb91f181a86d10588ee16c5e09c280f9811373974c29974cbe401987ea34e966699d7989a812b0e19377b511ea0bc627f5905647ce569311824848ede382cae8
languageName: node
linkType: hard
"ast-types-flow@npm:^0.0.7":
version: 0.0.7
resolution: "ast-types-flow@npm:0.0.7"
@@ -5645,13 +5633,6 @@ __metadata:
languageName: node
linkType: hard
"available-typed-arrays@npm:^1.0.5":
version: 1.0.5
resolution: "available-typed-arrays@npm:1.0.5"
checksum: 20eb47b3cefd7db027b9bbb993c658abd36d4edd3fe1060e83699a03ee275b0c9b216cc076ff3f2db29073225fb70e7613987af14269ac1fe2a19803ccc97f1a
languageName: node
linkType: hard
"axe-core@npm:^4.3.5":
version: 4.4.1
resolution: "axe-core@npm:4.4.1"
@@ -7500,7 +7481,7 @@ __metadata:
languageName: node
linkType: hard
"es-abstract@npm:^1.19.0, es-abstract@npm:^1.19.1, es-abstract@npm:^1.19.2, es-abstract@npm:^1.19.5, es-abstract@npm:^1.20.0":
"es-abstract@npm:^1.19.0, es-abstract@npm:^1.19.1, es-abstract@npm:^1.19.2, es-abstract@npm:^1.19.5":
version: 1.20.4
resolution: "es-abstract@npm:1.20.4"
dependencies:
@@ -7559,13 +7540,6 @@ __metadata:
languageName: node
linkType: hard
"es6-object-assign@npm:^1.1.0":
version: 1.1.0
resolution: "es6-object-assign@npm:1.1.0"
checksum: 8d4fdf63484d78b5c64cacc2c2e1165bc7b6a64b739d2a9db6a4dc8641d99cc9efb433cdd4dc3d3d6b00bfa6ce959694e4665e3255190339945c5f33b692b5d8
languageName: node
linkType: hard
"escalade@npm:^3.1.1":
version: 3.1.1
resolution: "escalade@npm:3.1.1"
@@ -8371,15 +8345,6 @@ __metadata:
languageName: node
linkType: hard
"for-each@npm:^0.3.3":
version: 0.3.3
resolution: "for-each@npm:0.3.3"
dependencies:
is-callable: ^1.1.3
checksum: 6c48ff2bc63362319c65e2edca4a8e1e3483a2fabc72fbe7feaf8c73db94fc7861bd53bc02c8a66a0c1dd709da6b04eec42e0abdd6b40ce47305ae92a25e5d28
languageName: node
linkType: hard
"fork-ts-checker-webpack-plugin@npm:^6.5.0":
version: 6.5.2
resolution: "fork-ts-checker-webpack-plugin@npm:6.5.2"
@@ -9291,7 +9256,7 @@ __metadata:
languageName: node
linkType: hard
"is-callable@npm:^1.1.3, is-callable@npm:^1.1.4, is-callable@npm:^1.2.7":
"is-callable@npm:^1.1.4, is-callable@npm:^1.2.7":
version: 1.2.7
resolution: "is-callable@npm:1.2.7"
checksum: 61fd57d03b0d984e2ed3720fb1c7a897827ea174bd44402878e059542ea8c4aeedee0ea0985998aa5cc2736b2fa6e271c08587addb5b3959ac52cf665173d1ac
@@ -9346,15 +9311,6 @@ __metadata:
languageName: node
linkType: hard
"is-generator-function@npm:^1.0.7":
version: 1.0.10
resolution: "is-generator-function@npm:1.0.10"
dependencies:
has-tostringtag: ^1.0.0
checksum: d54644e7dbaccef15ceb1e5d91d680eb5068c9ee9f9eb0a9e04173eb5542c9b51b5ab52c5537f5703e48d5fddfd376817c1ca07a84a407b7115b769d4bdde72b
languageName: node
linkType: hard
"is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1":
version: 4.0.3
resolution: "is-glob@npm:4.0.3"
@@ -9392,16 +9348,6 @@ __metadata:
languageName: node
linkType: hard
"is-nan@npm:^1.2.1":
version: 1.3.2
resolution: "is-nan@npm:1.3.2"
dependencies:
call-bind: ^1.0.0
define-properties: ^1.1.3
checksum: 5dfadcef6ad12d3029d43643d9800adbba21cf3ce2ec849f734b0e14ee8da4070d82b15fdb35138716d02587c6578225b9a22779cab34888a139cc43e4e3610a
languageName: node
linkType: hard
"is-negative-zero@npm:^2.0.2":
version: 2.0.2
resolution: "is-negative-zero@npm:2.0.2"
@@ -9518,19 +9464,6 @@ __metadata:
languageName: node
linkType: hard
"is-typed-array@npm:^1.1.3, is-typed-array@npm:^1.1.9":
version: 1.1.9
resolution: "is-typed-array@npm:1.1.9"
dependencies:
available-typed-arrays: ^1.0.5
call-bind: ^1.0.2
es-abstract: ^1.20.0
for-each: ^0.3.3
has-tostringtag: ^1.0.0
checksum: 11910f1e58755fef43bf0074e52fa5b932bf101ec65d613e0a83d40e8e4c6e3f2ee142d624ebc7624c091d3bbe921131f8db7d36ecbbb71909f2fe310c1faa65
languageName: node
linkType: hard
"is-weakref@npm:^1.0.2":
version: 1.0.2
resolution: "is-weakref@npm:1.0.2"
@@ -12645,15 +12578,6 @@ __metadata:
languageName: node
linkType: hard
"python-ast@npm:^0.1.0":
version: 0.1.0
resolution: "python-ast@npm:0.1.0"
dependencies:
antlr4ts: ^0.5.0-alpha.3
checksum: 1fc453e89f932a1e7f6f57a0f9dc304252093e8aa6c934a426ef30c7e2c9fd0c05aad168fa3ad053ba3bb88362a2fb5097c6e9d360351539f5027ff2af0e5eab
languageName: node
linkType: hard
"qs@npm:6.10.3":
version: 6.10.3
resolution: "qs@npm:6.10.3"
@@ -15032,19 +14956,6 @@ __metadata:
languageName: node
linkType: hard
"util@npm:^0.12.0, util@npm:^0.12.5":
version: 0.12.5
resolution: "util@npm:0.12.5"
dependencies:
inherits: ^2.0.3
is-arguments: ^1.0.4
is-generator-function: ^1.0.7
is-typed-array: ^1.1.3
which-typed-array: ^1.1.2
checksum: 705e51f0de5b446f4edec10739752ac25856541e0254ea1e7e45e5b9f9b0cb105bc4bd415736a6210edc68245a7f903bf085ffb08dd7deb8a0e847f60538a38a
languageName: node
linkType: hard
"utila@npm:~0.4":
version: 0.4.0
resolution: "utila@npm:0.4.0"
@@ -15407,20 +15318,6 @@ __metadata:
languageName: node
linkType: hard
"which-typed-array@npm:^1.1.2":
version: 1.1.8
resolution: "which-typed-array@npm:1.1.8"
dependencies:
available-typed-arrays: ^1.0.5
call-bind: ^1.0.2
es-abstract: ^1.20.0
for-each: ^0.3.3
has-tostringtag: ^1.0.0
is-typed-array: ^1.1.9
checksum: bedf4d30a738e848404fe67fe0ace33433a7298cf3f5a4d4b2c624ba99c4d25f06a7fd6f3566c3d16af5f8a54f0c6293cbfded5b1208ce11812753990223b45a
languageName: node
linkType: hard
"which@npm:^1.3.1":
version: 1.3.1
resolution: "which@npm:1.3.1"