v4.0.0b8

We used to separately build jedi and then use another Python step to package it.

We can just do it from the same environment to avoid packaging issues.

.github/workflows/publish-jedi.yml

@@ -38,7 +38,7 @@ jobs:
       with:
         node-version: '24'
         registry-url: 'https://registry.npmjs.org'
-    - run: ./build.py "$NPM_VERSION"
-      working-directory: npm/jedi
+    - run: poetry run python build.py "$NPM_VERSION"
+      working-directory: jedi
     - run: npm publish --tag "$NPM_TAG"
-      working-directory: npm/jedi/build
+      working-directory: jedi/npm-build

.gitignore

@@ -25,6 +25,7 @@
 build/
 build-*/
 _build/
+npm-build/
 
 # Tests
 ######################

build-all.sh

@@ -19,20 +19,15 @@ cd "$REPO_ROOT"
 flake8
 doc8
 
-# pybricks-jedi wheel (from local source)
+# pybricks-jedi tests
 echo "==> Testing pybricks-jedi"
 cd "$REPO_ROOT/jedi"
 poetry run pytest -vv
 
-echo "==> Building pybricks-jedi wheel"
-cd "$REPO_ROOT/jedi"
-rm -rf dist/
-poetry build --format=wheel
-
 # @pybricks/jedi npm package
 echo "==> Building @pybricks/jedi"
-cd "$REPO_ROOT"
-python3 npm/jedi/build.py "$NPM_VERSION"
+cd "$REPO_ROOT/jedi"
+poetry run python build.py "$NPM_VERSION"
 
 # @pybricks/ide-docs npm package
 echo "==> Building @pybricks/ide-docs"
@@ -43,5 +38,5 @@ yarn build
 
 echo ""
 echo "Build complete."
-echo "  jedi npm package : npm/jedi/build/"
+echo "  jedi npm package : jedi/npm-build/"
 echo "  ide-docs         : npm/ide-docs/html/"

jedi/build.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# Run with: poetry run python build.py <npm-version>
+# Builds the @pybricks/jedi npm package into npm-build/.
+
+import email.parser
+import importlib.metadata
+import json
+import pathlib
+import shutil
+import subprocess
+import sys
+import zipfile
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} <version>", file=sys.stderr)
+    sys.exit(1)
+
+VERSION = sys.argv[1]
+
+ROOT_DIR = pathlib.Path(__file__).parent.resolve()
+REPO_ROOT_DIR = (ROOT_DIR / "..").resolve()
+BUILD_DIR = ROOT_DIR / "npm-build"
+
+package_json = {
+    "name": "@pybricks/jedi",
+    "version": VERSION,
+    "description": "Binary distribution of pybricks-jedi Python package and dependencies for use with Pyodide.",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/pybricks/pybricks-api.git",
+        "directory": "jedi",
+    },
+    "publishConfig": {"registry": "https://registry.npmjs.org", "access": "public"},
+}
+
+license_identifiers: set[str] = set()
+license_text: dict[str, str] = {}
+whl_map: dict[str, str] = {}
+
+# ensure empty build directory so we don't end up with stale files
+shutil.rmtree(BUILD_DIR, True)
+BUILD_DIR.mkdir()
+
+# build pybricks api wheel from local source so pip uses it instead of fetching from PyPI
+subprocess.check_call(["poetry", "build", "--format=wheel"], cwd=REPO_ROOT_DIR)
+
+# copy locally built pybricks wheel to build dir
+for whl in (REPO_ROOT_DIR / "dist").glob("pybricks-*.whl"):
+    shutil.copy(whl, BUILD_DIR)
+
+# build pybricks-jedi wheel from local source
+subprocess.check_call(["poetry", "build", "--format=wheel"], cwd=ROOT_DIR)
+
+# copy locally built wheel to build dir
+for whl in (ROOT_DIR / "dist").glob("pybricks_jedi-*.whl"):
+    shutil.copy(whl, BUILD_DIR)
+
+# download transitive dependencies using the versions already installed in the venv
+# (installed by poetry from the lockfile, so versions are pinned correctly)
+transitive_packages = ["jedi", "parso", "docstring-parser", "typing-extensions"]
+transitive = [
+    f"{pkg}=={importlib.metadata.version(pkg)}" for pkg in transitive_packages
+]
+subprocess.check_call(
+    [sys.executable, "-m", "pip", "download", "--only-binary=any"] + transitive,
+    cwd=BUILD_DIR,
+)
+
+# extract info from wheel files to generate javascript package files
+for whl in BUILD_DIR.glob("*.whl"):
+    with zipfile.ZipFile(whl) as f:
+
+        def is_dist_info_metadata(info: zipfile.ZipInfo):
+            return info.filename.endswith(".dist-info/METADATA")
+
+        def is_dist_info_license(info: zipfile.ZipInfo):
+            return ".dist-info" in info.filename and "LICENSE" in info.filename
+
+        metadata = next(filter(is_dist_info_metadata, f.filelist))
+
+        with f.open(metadata.filename) as mf:
+            meta = email.parser.BytesParser().parse(mf)
+
+            # extract package name from metadata
+            name = meta["Name"]
+
+            if not name:
+                raise RuntimeError(f"missing 'Name' in {whl.name} METADATA")
+
+            whl_map[name] = whl.name
+
+            # extract license identifier from metadata
+
+            license = meta["License"]
+
+            if not license:
+                # some packages are missing license in metadata
+                if whl.name.startswith("typing_extensions-"):
+                    license = "Python-2.0"
+                else:
+                    raise RuntimeError(f"missing 'License' in {whl.name} METADATA")
+
+            license_identifiers.add(license)
+
+        if whl.name.startswith("pybricks_jedi-"):
+            with open(ROOT_DIR / "LICENSE") as lf:
+                license_text[whl.name] = lf.read()
+        else:
+            try:
+                license = next(filter(is_dist_info_license, f.filelist))
+            except StopIteration:
+                raise RuntimeError(f"missing license for {whl.name}")
+
+            with f.open(license) as lf:
+                license_text[whl.name] = lf.read().decode()
+
+
+# generate package.json file
+
+# create "license" item from collected license identifiers
+package_json["license"] = (
+    license_identifiers[0]
+    if len(license_identifiers) < 2
+    else f"({' AND '.join(license_identifiers)})"
+)
+
+# create "exports" item from collect whl map
+package_json["exports"] = {f"./{k}.whl": f"./{v}" for k, v in whl_map.items()}
+
+with open(BUILD_DIR / "package.json", "w") as f:
+    json.dump(package_json, f, indent=2)
+
+# generate LICENSE file
+
+NEWLINE = "\n"
+DIVIDER = "=" * 80 + NEWLINE
+with open(BUILD_DIR / "LICENSE", "w") as f:
+    for whl, text in license_text.items():
+        f.write(DIVIDER)
+        f.write(whl)
+        f.writelines([NEWLINE, DIVIDER, NEWLINE])
+        f.write(text)
+        f.write(NEWLINE)
+
+# copy additional files
+
+for file in ("README.md",):
+    shutil.copy(ROOT_DIR / file, BUILD_DIR / file)

npm/jedi/build.py

@@ -53,17 +53,23 @@ subprocess.check_call(["poetry", "build", "--format=wheel"], cwd=JEDI_SRC_DIR)
 for whl in (JEDI_SRC_DIR / "dist").glob("pybricks_jedi-*.whl"):
     shutil.copy(whl, BUILD_DIR)
 
-# download transitive dependencies from PyPI, using the local wheels to satisfy pybricks-jedi and pybricks itself
+# download transitive dependencies using the versions already installed in the venv
+# (installed by poetry from the lockfile, so versions are pinned correctly)
+transitive_packages = ["jedi", "parso", "docstring-parser", "typing-extensions"]
+# use the jedi venv's Python to query metadata since packages are installed there
+jedi_venv_python = JEDI_SRC_DIR / ".venv" / "bin" / "python"
+pkg_versions = json.loads(
+    subprocess.check_output(
+        [
+            jedi_venv_python,
+            "-c",
+            f"import importlib.metadata, json; print(json.dumps({{p: importlib.metadata.version(p) for p in {transitive_packages!r}}}))",
+        ]
+    )
+)
+transitive = [f"{pkg}=={pkg_versions[pkg]}" for pkg in transitive_packages]
 subprocess.check_call(
-    [
-        sys.executable,
-        "-m",
-        "pip",
-        "download",
-        "--only-binary=any",
-        f"--find-links={BUILD_DIR}",
-        "pybricks-jedi",
-    ],
+    [sys.executable, "-m", "pip", "download", "--only-binary=any"] + transitive,
     cwd=BUILD_DIR,
 )
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybricks"
-version = "4.0.0b6"
+version = "4.0.0b8"
 description = "Documentation and user-API stubs for Pybricks MicroPython"
 authors = ["The Pybricks Authors <team@pybricks.com>"]
 maintainers = ["Laurens Valk <laurens@pybricks.com>", "David Lechner <david@pybricks.com>" ]