npm/jedi: Further trim down build logic.

We used to separately build jedi and then use another Python step to package it.

We can just do it from the same environment to avoid packaging issues.

.github/workflows/publish-jedi.yml

@@ -38,7 +38,7 @@ jobs:
       with:
         node-version: '24'
         registry-url: 'https://registry.npmjs.org'
-    - run: ./build.py "$NPM_VERSION"
-      working-directory: npm/jedi
+    - run: poetry run python build.py "$NPM_VERSION"
+      working-directory: jedi
     - run: npm publish --tag "$NPM_TAG"
-      working-directory: npm/jedi/build
+      working-directory: jedi/npm-build

.gitignore

@@ -25,6 +25,7 @@
 build/
 build-*/
 _build/
+npm-build/
 
 # Tests
 ######################

build-all.sh

@@ -19,20 +19,15 @@ cd "$REPO_ROOT"
 flake8
 doc8
 
-# pybricks-jedi wheel (from local source)
+# pybricks-jedi tests
 echo "==> Testing pybricks-jedi"
 cd "$REPO_ROOT/jedi"
 poetry run pytest -vv
 
-echo "==> Building pybricks-jedi wheel"
-cd "$REPO_ROOT/jedi"
-rm -rf dist/
-poetry build --format=wheel
-
 # @pybricks/jedi npm package
 echo "==> Building @pybricks/jedi"
-cd "$REPO_ROOT"
-python3 npm/jedi/build.py "$NPM_VERSION"
+cd "$REPO_ROOT/jedi"
+poetry run python build.py "$NPM_VERSION"
 
 # @pybricks/ide-docs npm package
 echo "==> Building @pybricks/ide-docs"
@@ -43,5 +38,5 @@ yarn build
 
 echo ""
 echo "Build complete."
-echo "  jedi npm package : npm/jedi/build/"
+echo "  jedi npm package : jedi/npm-build/"
 echo "  ide-docs         : npm/ide-docs/html/"

jedi/build.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# Run with: poetry run python build.py <npm-version>
+# Builds the @pybricks/jedi npm package into npm-build/.
+
+import email.parser
+import importlib.metadata
+import json
+import pathlib
+import shutil
+import subprocess
+import sys
+import zipfile
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} <version>", file=sys.stderr)
+    sys.exit(1)
+
+VERSION = sys.argv[1]
+
+ROOT_DIR = pathlib.Path(__file__).parent.resolve()
+REPO_ROOT_DIR = (ROOT_DIR / "..").resolve()
+BUILD_DIR = ROOT_DIR / "npm-build"
+
+package_json = {
+    "name": "@pybricks/jedi",
+    "version": VERSION,
+    "description": "Binary distribution of pybricks-jedi Python package and dependencies for use with Pyodide.",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/pybricks/pybricks-api.git",
+        "directory": "jedi",
+    },
+    "publishConfig": {"registry": "https://registry.npmjs.org", "access": "public"},
+}
+
+license_identifiers: set[str] = set()
+license_text: dict[str, str] = {}
+whl_map: dict[str, str] = {}
+
+# ensure empty build directory so we don't end up with stale files
+shutil.rmtree(BUILD_DIR, True)
+BUILD_DIR.mkdir()
+
+# build pybricks api wheel from local source so pip uses it instead of fetching from PyPI
+subprocess.check_call(["poetry", "build", "--format=wheel"], cwd=REPO_ROOT_DIR)
+
+# copy locally built pybricks wheel to build dir
+for whl in (REPO_ROOT_DIR / "dist").glob("pybricks-*.whl"):
+    shutil.copy(whl, BUILD_DIR)
+
+# build pybricks-jedi wheel from local source
+subprocess.check_call(["poetry", "build", "--format=wheel"], cwd=ROOT_DIR)
+
+# copy locally built wheel to build dir
+for whl in (ROOT_DIR / "dist").glob("pybricks_jedi-*.whl"):
+    shutil.copy(whl, BUILD_DIR)
+
+# download transitive dependencies using the versions already installed in the venv
+# (installed by poetry from the lockfile, so versions are pinned correctly)
+transitive_packages = ["jedi", "parso", "docstring-parser", "typing-extensions"]
+transitive = [
+    f"{pkg}=={importlib.metadata.version(pkg)}" for pkg in transitive_packages
+]
+subprocess.check_call(
+    [sys.executable, "-m", "pip", "download", "--only-binary=any"] + transitive,
+    cwd=BUILD_DIR,
+)
+
+# extract info from wheel files to generate javascript package files
+for whl in BUILD_DIR.glob("*.whl"):
+    with zipfile.ZipFile(whl) as f:
+
+        def is_dist_info_metadata(info: zipfile.ZipInfo):
+            return info.filename.endswith(".dist-info/METADATA")
+
+        def is_dist_info_license(info: zipfile.ZipInfo):
+            return ".dist-info" in info.filename and "LICENSE" in info.filename
+
+        metadata = next(filter(is_dist_info_metadata, f.filelist))
+
+        with f.open(metadata.filename) as mf:
+            meta = email.parser.BytesParser().parse(mf)
+
+            # extract package name from metadata
+            name = meta["Name"]
+
+            if not name:
+                raise RuntimeError(f"missing 'Name' in {whl.name} METADATA")
+
+            whl_map[name] = whl.name
+
+            # extract license identifier from metadata
+
+            license = meta["License"]
+
+            if not license:
+                # some packages are missing license in metadata
+                if whl.name.startswith("typing_extensions-"):
+                    license = "Python-2.0"
+                else:
+                    raise RuntimeError(f"missing 'License' in {whl.name} METADATA")
+
+            license_identifiers.add(license)
+
+        if whl.name.startswith("pybricks_jedi-"):
+            with open(ROOT_DIR / "LICENSE") as lf:
+                license_text[whl.name] = lf.read()
+        else:
+            try:
+                license = next(filter(is_dist_info_license, f.filelist))
+            except StopIteration:
+                raise RuntimeError(f"missing license for {whl.name}")
+
+            with f.open(license) as lf:
+                license_text[whl.name] = lf.read().decode()
+
+
+# generate package.json file
+
+# create "license" item from collected license identifiers
+package_json["license"] = (
+    license_identifiers[0]
+    if len(license_identifiers) < 2
+    else f"({' AND '.join(license_identifiers)})"
+)
+
+# create "exports" item from collect whl map
+package_json["exports"] = {f"./{k}.whl": f"./{v}" for k, v in whl_map.items()}
+
+with open(BUILD_DIR / "package.json", "w") as f:
+    json.dump(package_json, f, indent=2)
+
+# generate LICENSE file
+
+NEWLINE = "\n"
+DIVIDER = "=" * 80 + NEWLINE
+with open(BUILD_DIR / "LICENSE", "w") as f:
+    for whl, text in license_text.items():
+        f.write(DIVIDER)
+        f.write(whl)
+        f.writelines([NEWLINE, DIVIDER, NEWLINE])
+        f.write(text)
+        f.write(NEWLINE)
+
+# copy additional files
+
+for file in ("README.md",):
+    shutil.copy(ROOT_DIR / file, BUILD_DIR / file)

npm/jedi/build.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import email.parser
-import importlib.metadata
 import json
 import pathlib
 import shutil
@@ -57,9 +56,18 @@ for whl in (JEDI_SRC_DIR / "dist").glob("pybricks_jedi-*.whl"):
 # download transitive dependencies using the versions already installed in the venv
 # (installed by poetry from the lockfile, so versions are pinned correctly)
 transitive_packages = ["jedi", "parso", "docstring-parser", "typing-extensions"]
-transitive = [
-    f"{pkg}=={importlib.metadata.version(pkg)}" for pkg in transitive_packages
-]
+# use the jedi venv's Python to query metadata since packages are installed there
+jedi_venv_python = JEDI_SRC_DIR / ".venv" / "bin" / "python"
+pkg_versions = json.loads(
+    subprocess.check_output(
+        [
+            jedi_venv_python,
+            "-c",
+            f"import importlib.metadata, json; print(json.dumps({{p: importlib.metadata.version(p) for p in {transitive_packages!r}}}))",
+        ]
+    )
+)
+transitive = [f"{pkg}=={pkg_versions[pkg]}" for pkg in transitive_packages]
 subprocess.check_call(
     [sys.executable, "-m", "pip", "download", "--only-binary=any"] + transitive,
     cwd=BUILD_DIR,