mirror of
https://github.com/python/cpython
synced 2024-09-05 08:17:45 +00:00
gh-95853: Add script to automate WASM build (GH-95828)
Automate WASM build with a new Python script. The script provides several build profiles with configure flags for Emscripten flavors and WASI. The script can detect and use Emscripten SDK and WASI SDK from default locations or env vars. ``configure`` now detects Node arguments and creates HOSTRUNNER arguments for Node 16. It also sets correct arguments for ``wasm64-emscripten``. Co-authored-by: Brett Cannon <brett@python.org>
This commit is contained in:
Christian Heimes
GitHub
parent
0b329f4f03
commit
32ac98e899
|
|
@ -6,6 +6,7 @@
|
|||
import warnings
|
||||
from unicodedata import normalize
|
||||
from test.support import os_helper
|
||||
from test import support
|
||||
|
||||
|
||||
filenames = [
|
||||
|
|
@ -123,6 +124,10 @@ def test_open(self):
|
|||
# NFKD in Python is useless, because darwin will normalize it later and so
|
||||
# open(), os.stat(), etc. don't raise any exception.
|
||||
@unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
|
||||
@unittest.skipIf(
|
||||
support.is_emscripten or support.is_wasi,
|
||||
"test fails on Emscripten/WASI when host platform is macOS."
|
||||
)
|
||||
def test_normalize(self):
|
||||
files = set(self.files)
|
||||
others = set()
|
||||
|
|
|
|||
|
|
@ -487,7 +487,14 @@ def test_warn_explicit_non_ascii_filename(self):
|
|||
module=self.module) as w:
|
||||
self.module.resetwarnings()
|
||||
self.module.filterwarnings("always", category=UserWarning)
|
||||
for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"):
|
||||
filenames = ["nonascii\xe9\u20ac"]
|
||||
if not support.is_emscripten:
|
||||
# JavaScript does not like surrogates.
|
||||
# Invalid UTF-8 leading byte 0x80 encountered when
|
||||
# deserializing a UTF-8 string in wasm memory to a JS
|
||||
# string!
|
||||
filenames.append("surrogate\udc80")
|
||||
for filename in filenames:
|
||||
try:
|
||||
os.fsencode(filename)
|
||||
except UnicodeEncodeError:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and
|
||||
test steps for building CPython on WebAssembly platforms.
|
||||
|
|
@ -2789,14 +2789,18 @@ EM_JS(char *, _Py_emscripten_runtime, (void), {
|
|||
if (typeof navigator == 'object') {
|
||||
info = navigator.userAgent;
|
||||
} else if (typeof process == 'object') {
|
||||
info = "Node.js ".concat(process.version)
|
||||
info = "Node.js ".concat(process.version);
|
||||
} else {
|
||||
info = "UNKNOWN"
|
||||
info = "UNKNOWN";
|
||||
}
|
||||
var len = lengthBytesUTF8(info) + 1;
|
||||
var res = _malloc(len);
|
||||
stringToUTF8(info, res, len);
|
||||
if (res) stringToUTF8(info, res, len);
|
||||
#if __wasm64__
|
||||
return BigInt(res);
|
||||
#else
|
||||
return res;
|
||||
#endif
|
||||
});
|
||||
|
||||
static PyObject *
|
||||
|
|
|
|||
|
|
@ -35,7 +35,13 @@ # other
|
|||
|
||||
### Compile a build Python interpreter
|
||||
|
||||
From within the container, run the following commands:
|
||||
From within the container, run the following command:
|
||||
|
||||
```shell
|
||||
./Tools/wasm/wasm_build.py build
|
||||
```
|
||||
|
||||
The command is roughly equivalent to:
|
||||
|
||||
```shell
|
||||
mkdir -p builddir/build
|
||||
|
|
@ -45,13 +51,13 @@ ### Compile a build Python interpreter
|
|||
popd
|
||||
```
|
||||
|
||||
### Fetch and build additional emscripten ports
|
||||
### Cross-compile to wasm32-emscripten for browser
|
||||
|
||||
```shell
|
||||
embuilder build zlib bzip2
|
||||
./Tools/wasm/wasm_build.py emscripten-browser
|
||||
```
|
||||
|
||||
### Cross compile to wasm32-emscripten for browser
|
||||
The command is roughly equivalent to:
|
||||
|
||||
```shell
|
||||
mkdir -p builddir/emscripten-browser
|
||||
|
|
@ -85,14 +91,21 @@ ### Cross compile to wasm32-emscripten for browser
|
|||
### Cross compile to wasm32-emscripten for node
|
||||
|
||||
```shell
|
||||
mkdir -p builddir/emscripten-node
|
||||
pushd builddir/emscripten-node
|
||||
./Tools/wasm/wasm_build.py emscripten-browser-dl
|
||||
```
|
||||
|
||||
The command is roughly equivalent to:
|
||||
|
||||
```shell
|
||||
mkdir -p builddir/emscripten-node-dl
|
||||
pushd builddir/emscripten-node-dl
|
||||
|
||||
CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
|
||||
emconfigure ../../configure -C \
|
||||
--host=wasm32-unknown-emscripten \
|
||||
--build=$(../../config.guess) \
|
||||
--with-emscripten-target=node \
|
||||
--enable-wasm-dynamic-linking \
|
||||
--with-build-python=$(pwd)/../build/python
|
||||
|
||||
emmake make -j$(nproc)
|
||||
|
|
@ -100,7 +113,7 @@ ### Cross compile to wasm32-emscripten for node
|
|||
```
|
||||
|
||||
```shell
|
||||
node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node/python.js
|
||||
node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js
|
||||
```
|
||||
|
||||
(``--experimental-wasm-bigint`` is not needed with recent NodeJS versions)
|
||||
|
|
@ -199,6 +212,15 @@ ## wasm32-emscripten in node
|
|||
- Node RawFS allows direct access to the host file system without need to
|
||||
perform ``FS.mount()`` call.
|
||||
|
||||
## wasm64-emscripten
|
||||
|
||||
- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``.
|
||||
- ``EM_JS`` functions must return ``BigInt()``.
|
||||
- ``Py_BuildValue()`` format strings must match size of types. Confusing 32
|
||||
and 64 bits types leads to memory corruption, see
|
||||
[gh-95876](https://github.com/python/cpython/issues/95876) and
|
||||
[gh-95878](https://github.com/python/cpython/issues/95878).
|
||||
|
||||
# Hosting Python WASM builds
|
||||
|
||||
The simple REPL terminal uses SharedArrayBuffer. For security reasons
|
||||
|
|
@ -234,6 +256,12 @@ ## Cross-compile to wasm32-wasi
|
|||
``pkg-config`` overrides. The script assumes that WASI-SDK is installed in
|
||||
``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``.
|
||||
|
||||
```shell
|
||||
./Tools/wasm/wasm_build.py wasi
|
||||
```
|
||||
|
||||
The command is roughly equivalent to:
|
||||
|
||||
```shell
|
||||
mkdir -p builddir/wasi
|
||||
pushd builddir/wasi
|
||||
|
|
|
|||
|
|
@ -72,4 +72,5 @@ export CFLAGS LDFLAGS
|
|||
export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR
|
||||
export PATH
|
||||
|
||||
exec "$@"
|
||||
# no exec, it makes arvg[0] path absolute.
|
||||
"$@"
|
||||
|
|
|
|||
567
Tools/wasm/wasm_build.py
Executable file
567
Tools/wasm/wasm_build.py
Executable file
|
|
@ -0,0 +1,567 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Build script for Python on WebAssembly platforms.
|
||||
|
||||
$ ./Tools/wasm/wasm_builder.py emscripten-browser compile
|
||||
$ ./Tools/wasm/wasm_builder.py emscripten-node-dl test
|
||||
$ ./Tools/wasm/wasm_builder.py wasi test
|
||||
|
||||
Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
|
||||
"emscripten-browser", and "wasi".
|
||||
|
||||
Emscripten builds require a recent Emscripten SDK. The tools looks for an
|
||||
activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
|
||||
(Debian, Homebrew) are not supported.
|
||||
|
||||
WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
|
||||
and falls back to /opt/wasi-sdk.
|
||||
"""
|
||||
import argparse
|
||||
import enum
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sysconfig
|
||||
|
||||
# for Python 3.8
|
||||
from typing import Any, Dict, Callable, Iterable, List, Optional, Union
|
||||
|
||||
SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
|
||||
WASMTOOLS = SRCDIR / "Tools" / "wasm"
|
||||
BUILDDIR = SRCDIR / "builddir"
|
||||
CONFIGURE = SRCDIR / "configure"
|
||||
SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
|
||||
|
||||
HAS_CCACHE = shutil.which("ccache") is not None
|
||||
|
||||
# path to WASI-SDK root
|
||||
WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
|
||||
|
||||
# path to Emscripten SDK config file.
|
||||
# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
|
||||
EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
|
||||
# 3.1.16 has broken utime()
|
||||
EMSDK_MIN_VERSION = (3, 1, 17)
|
||||
_MISSING = pathlib.PurePath("MISSING")
|
||||
|
||||
# WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py"
|
||||
|
||||
CLEAN_SRCDIR = f"""
|
||||
Builds require a clean source directory. Please use a clean checkout or
|
||||
run "make clean -C '{SRCDIR}'".
|
||||
"""
|
||||
|
||||
INSTALL_EMSDK = """
|
||||
wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
|
||||
https://emscripten.org/docs/getting_started/downloads.html how to install
|
||||
Emscripten and how to activate the SDK with ". /path/to/emsdk/emsdk_env.sh".
|
||||
|
||||
git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
|
||||
cd /path/to/emsdk
|
||||
./emsdk install latest
|
||||
./emsdk activate latest
|
||||
source /path/to/emsdk_env.sh
|
||||
"""
|
||||
|
||||
INSTALL_WASI_SDK = """
|
||||
wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
|
||||
https://github.com/WebAssembly/wasi-sdk/releases and install it to
|
||||
"/opt/wasi-sdk". Alternatively you can install the SDK in a different location
|
||||
and point the environment variable WASI_SDK_PATH to the root directory
|
||||
of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
|
||||
"""
|
||||
|
||||
INSTALL_WASMTIME = """
|
||||
wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
|
||||
https://wasmtime.dev/ to install wasmtime.
|
||||
"""
|
||||
|
||||
|
||||
def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.PurePath:
|
||||
"""Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT
|
||||
|
||||
The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
|
||||
environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
|
||||
subdirectory with tools like "emconfigure".
|
||||
"""
|
||||
if not emconfig.exists():
|
||||
return _MISSING
|
||||
with open(emconfig, encoding="utf-8") as f:
|
||||
code = f.read()
|
||||
# EM_CONFIG file is a Python snippet
|
||||
local: Dict[str, Any] = {}
|
||||
exec(code, globals(), local)
|
||||
return pathlib.Path(local["EMSCRIPTEN_ROOT"])
|
||||
|
||||
|
||||
EMSCRIPTEN_ROOT = get_emscripten_root()
|
||||
|
||||
|
||||
class ConditionError(ValueError):
|
||||
def __init__(self, info: str, text: str):
|
||||
self.info = info
|
||||
self.text = text
|
||||
|
||||
def __str__(self):
|
||||
return f"{type(self).__name__}: '{self.info}'\n{self.text}"
|
||||
|
||||
|
||||
class MissingDependency(ConditionError):
|
||||
pass
|
||||
|
||||
|
||||
class DirtySourceDirectory(ConditionError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Platform:
|
||||
"""Platform-specific settings
|
||||
|
||||
- CONFIG_SITE override
|
||||
- configure wrapper (e.g. emconfigure)
|
||||
- make wrapper (e.g. emmake)
|
||||
- additional environment variables
|
||||
- check function to verify SDK
|
||||
"""
|
||||
|
||||
name: str
|
||||
pythonexe: str
|
||||
config_site: Optional[pathlib.PurePath]
|
||||
configure_wrapper: Optional[pathlib.PurePath]
|
||||
make_wrapper: Optional[pathlib.PurePath]
|
||||
environ: dict
|
||||
check: Callable[[], None]
|
||||
|
||||
def getenv(self, profile: "BuildProfile") -> dict:
|
||||
return self.environ.copy()
|
||||
|
||||
|
||||
def _check_clean_src():
|
||||
candidates = [
|
||||
SRCDIR / "Programs" / "python.o",
|
||||
SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
|
||||
|
||||
|
||||
NATIVE = Platform(
|
||||
"native",
|
||||
# macOS has python.exe
|
||||
pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
|
||||
config_site=None,
|
||||
configure_wrapper=None,
|
||||
make_wrapper=None,
|
||||
environ={},
|
||||
check=_check_clean_src,
|
||||
)
|
||||
|
||||
|
||||
def _check_emscripten():
|
||||
if EMSCRIPTEN_ROOT is _MISSING:
|
||||
raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
|
||||
# sanity check
|
||||
emconfigure = EMSCRIPTEN.configure_wrapper
|
||||
if not emconfigure.exists():
|
||||
raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
|
||||
# version check
|
||||
version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
|
||||
if not version_txt.exists():
|
||||
raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
|
||||
with open(version_txt) as f:
|
||||
version = f.read().strip().strip('"')
|
||||
version_tuple = tuple(int(v) for v in version.split("."))
|
||||
if version_tuple < EMSDK_MIN_VERSION:
|
||||
raise MissingDependency(
|
||||
os.fspath(version_txt),
|
||||
f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
|
||||
"minimum required version "
|
||||
f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
|
||||
)
|
||||
_check_clean_src()
|
||||
|
||||
|
||||
EMSCRIPTEN = Platform(
|
||||
"emscripten",
|
||||
pythonexe="python.js",
|
||||
config_site=WASMTOOLS / "config.site-wasm32-emscripten",
|
||||
configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
|
||||
make_wrapper=EMSCRIPTEN_ROOT / "emmake",
|
||||
environ={"EM_COMPILER_WRAPPER": "ccache"} if HAS_CCACHE else {},
|
||||
check=_check_emscripten,
|
||||
)
|
||||
|
||||
|
||||
def _check_wasi():
|
||||
wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
|
||||
if not wasm_ld.exists():
|
||||
raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
|
||||
wasmtime = shutil.which("wasmtime")
|
||||
if wasmtime is None:
|
||||
raise MissingDependency("wasmtime", INSTALL_WASMTIME)
|
||||
_check_clean_src()
|
||||
|
||||
|
||||
WASI = Platform(
|
||||
"wasi",
|
||||
pythonexe="python.wasm",
|
||||
config_site=WASMTOOLS / "config.site-wasm32-wasi",
|
||||
configure_wrapper=WASMTOOLS / "wasi-env",
|
||||
make_wrapper=None,
|
||||
environ={
|
||||
"WASI_SDK_PATH": WASI_SDK_PATH,
|
||||
# workaround for https://github.com/python/cpython/issues/95952
|
||||
"HOSTRUNNER": (
|
||||
"wasmtime run "
|
||||
"--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-$(VERSION):/Lib "
|
||||
"--mapdir /::{srcdir} --"
|
||||
),
|
||||
},
|
||||
check=_check_wasi,
|
||||
)
|
||||
|
||||
|
||||
class Host(enum.Enum):
|
||||
"""Target host triplet"""
|
||||
|
||||
wasm32_emscripten = "wasm32-unknown-emscripten"
|
||||
wasm64_emscripten = "wasm64-unknown-emscripten"
|
||||
wasm32_wasi = "wasm32-unknown-wasi"
|
||||
wasm64_wasi = "wasm64-unknown-wasi"
|
||||
# current platform
|
||||
build = sysconfig.get_config_var("BUILD_GNU_TYPE")
|
||||
|
||||
@property
|
||||
def platform(self) -> Platform:
|
||||
if self.is_emscripten:
|
||||
return EMSCRIPTEN
|
||||
elif self.is_wasi:
|
||||
return WASI
|
||||
else:
|
||||
return NATIVE
|
||||
|
||||
@property
|
||||
def is_emscripten(self) -> bool:
|
||||
cls = type(self)
|
||||
return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
|
||||
|
||||
@property
|
||||
def is_wasi(self) -> bool:
|
||||
cls = type(self)
|
||||
return self in {cls.wasm32_wasi, cls.wasm64_wasi}
|
||||
|
||||
|
||||
class EmscriptenTarget(enum.Enum):
|
||||
"""Emscripten-specific targets (--with-emscripten-target)"""
|
||||
|
||||
browser = "browser"
|
||||
browser_debug = "browser-debug"
|
||||
node = "node"
|
||||
node_debug = "node-debug"
|
||||
|
||||
@property
|
||||
def can_execute(self) -> bool:
|
||||
cls = type(self)
|
||||
return self not in {cls.browser, cls.browser_debug}
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class BuildProfile:
|
||||
name: str
|
||||
host: Host
|
||||
target: Union[EmscriptenTarget, None] = None
|
||||
dynamic_linking: Union[bool, None] = None
|
||||
pthreads: Union[bool, None] = None
|
||||
testopts: str = "-j2"
|
||||
|
||||
@property
|
||||
def can_execute(self) -> bool:
|
||||
"""Can target run pythoninfo and tests?
|
||||
|
||||
Disabled for browser, enabled for all other targets
|
||||
"""
|
||||
return self.target is None or self.target.can_execute
|
||||
|
||||
@property
|
||||
def builddir(self) -> pathlib.Path:
|
||||
"""Path to build directory"""
|
||||
return BUILDDIR / self.name
|
||||
|
||||
@property
|
||||
def python_cmd(self) -> pathlib.Path:
|
||||
"""Path to python executable"""
|
||||
return self.builddir / self.host.platform.pythonexe
|
||||
|
||||
@property
|
||||
def makefile(self) -> pathlib.Path:
|
||||
"""Path to Makefile"""
|
||||
return self.builddir / "Makefile"
|
||||
|
||||
@property
|
||||
def configure_cmd(self) -> List[str]:
|
||||
"""Generate configure command"""
|
||||
# use relative path, so WASI tests can find lib prefix.
|
||||
# pathlib.Path.relative_to() does not work here.
|
||||
configure = os.path.relpath(CONFIGURE, self.builddir)
|
||||
cmd = [configure, "-C"]
|
||||
platform = self.host.platform
|
||||
if platform.configure_wrapper:
|
||||
cmd.insert(0, os.fspath(platform.configure_wrapper))
|
||||
|
||||
cmd.append(f"--host={self.host.value}")
|
||||
cmd.append(f"--build={Host.build.value}")
|
||||
|
||||
if self.target is not None:
|
||||
assert self.host.is_emscripten
|
||||
cmd.append(f"--with-emscripten-target={self.target.value}")
|
||||
|
||||
if self.dynamic_linking is not None:
|
||||
assert self.host.is_emscripten
|
||||
opt = "enable" if self.dynamic_linking else "disable"
|
||||
cmd.append(f"--{opt}-wasm-dynamic-linking")
|
||||
|
||||
if self.pthreads is not None:
|
||||
assert self.host.is_emscripten
|
||||
opt = "enable" if self.pthreads else "disable"
|
||||
cmd.append(f"--{opt}-wasm-pthreads")
|
||||
|
||||
if self.host != Host.build:
|
||||
cmd.append(f"--with-build-python={BUILD.python_cmd}")
|
||||
|
||||
if platform.config_site is not None:
|
||||
cmd.append(f"CONFIG_SITE={platform.config_site}")
|
||||
|
||||
return cmd
|
||||
|
||||
@property
|
||||
def make_cmd(self) -> List[str]:
|
||||
"""Generate make command"""
|
||||
cmd = ["make"]
|
||||
platform = self.host.platform
|
||||
if platform.make_wrapper:
|
||||
cmd.insert(0, os.fspath(platform.make_wrapper))
|
||||
return cmd
|
||||
|
||||
def getenv(self) -> dict:
|
||||
"""Generate environ dict for platform"""
|
||||
env = os.environ.copy()
|
||||
env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
|
||||
platenv = self.host.platform.getenv(self)
|
||||
for key, value in platenv.items():
|
||||
if isinstance(value, str):
|
||||
value = value.format(
|
||||
relbuilddir=self.builddir.relative_to(SRCDIR),
|
||||
srcdir=SRCDIR,
|
||||
)
|
||||
env[key] = value
|
||||
return env
|
||||
|
||||
def _run_cmd(self, cmd: Iterable[str], args: Iterable[str]):
|
||||
cmd = list(cmd)
|
||||
cmd.extend(args)
|
||||
return subprocess.check_call(
|
||||
cmd,
|
||||
cwd=os.fspath(self.builddir),
|
||||
env=self.getenv(),
|
||||
)
|
||||
|
||||
def _check_execute(self):
|
||||
if not self.can_execute:
|
||||
raise ValueError(f"Cannot execute on {self.target}")
|
||||
|
||||
def run_build(self, force_configure: bool = False):
|
||||
"""Run configure (if necessary) and make"""
|
||||
if force_configure or not self.makefile.exists():
|
||||
self.run_configure()
|
||||
self.run_make()
|
||||
|
||||
def run_configure(self, *args):
|
||||
"""Run configure script to generate Makefile"""
|
||||
os.makedirs(self.builddir, exist_ok=True)
|
||||
return self._run_cmd(self.configure_cmd, args)
|
||||
|
||||
def run_make(self, *args):
|
||||
"""Run make (defaults to build all)"""
|
||||
return self._run_cmd(self.make_cmd, args)
|
||||
|
||||
def run_pythoninfo(self):
|
||||
"""Run 'make pythoninfo'"""
|
||||
self._check_execute()
|
||||
return self.run_make("pythoninfo")
|
||||
|
||||
def run_test(self):
|
||||
"""Run buildbottests"""
|
||||
self._check_execute()
|
||||
return self.run_make("buildbottest", f"TESTOPTS={self.testopts}")
|
||||
|
||||
def run_py(self, *args):
|
||||
"""Run Python with hostrunner"""
|
||||
self._check_execute()
|
||||
self.run_make(
|
||||
"--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
|
||||
)
|
||||
|
||||
def clean(self, all: bool = False):
|
||||
"""Clean build directory"""
|
||||
if all:
|
||||
if self.builddir.exists():
|
||||
shutil.rmtree(self.builddir)
|
||||
elif self.makefile.exists():
|
||||
self.run_make("clean")
|
||||
|
||||
|
||||
# native build (build Python)
|
||||
BUILD = BuildProfile(
|
||||
"build",
|
||||
host=Host.build,
|
||||
)
|
||||
|
||||
_profiles = [
|
||||
BUILD,
|
||||
# wasm32-emscripten
|
||||
BuildProfile(
|
||||
"emscripten-browser",
|
||||
host=Host.wasm32_emscripten,
|
||||
target=EmscriptenTarget.browser,
|
||||
dynamic_linking=True,
|
||||
),
|
||||
BuildProfile(
|
||||
"emscripten-browser-debug",
|
||||
host=Host.wasm32_emscripten,
|
||||
target=EmscriptenTarget.browser_debug,
|
||||
dynamic_linking=True,
|
||||
),
|
||||
BuildProfile(
|
||||
"emscripten-node-dl",
|
||||
host=Host.wasm32_emscripten,
|
||||
target=EmscriptenTarget.node,
|
||||
dynamic_linking=True,
|
||||
),
|
||||
BuildProfile(
|
||||
"emscripten-node-dl-debug",
|
||||
host=Host.wasm32_emscripten,
|
||||
target=EmscriptenTarget.node_debug,
|
||||
dynamic_linking=True,
|
||||
),
|
||||
BuildProfile(
|
||||
"emscripten-node-pthreads",
|
||||
host=Host.wasm32_emscripten,
|
||||
target=EmscriptenTarget.node,
|
||||
pthreads=True,
|
||||
),
|
||||
BuildProfile(
|
||||
"emscripten-node-pthreads-debug",
|
||||
host=Host.wasm32_emscripten,
|
||||
target=EmscriptenTarget.node_debug,
|
||||
pthreads=True,
|
||||
),
|
||||
# wasm64-emscripten (currently not working)
|
||||
BuildProfile(
|
||||
"wasm64-emscripten-node-debug",
|
||||
host=Host.wasm64_emscripten,
|
||||
target=EmscriptenTarget.node_debug,
|
||||
# MEMORY64 is not compatible with dynamic linking
|
||||
dynamic_linking=False,
|
||||
pthreads=False,
|
||||
),
|
||||
# wasm32-wasi
|
||||
BuildProfile(
|
||||
"wasi",
|
||||
host=Host.wasm32_wasi,
|
||||
# skip sysconfig test_srcdir
|
||||
testopts="-i '*.test_srcdir' -j2",
|
||||
),
|
||||
# no SDK available yet
|
||||
# BuildProfile(
|
||||
# "wasm64-wasi",
|
||||
# host=Host.wasm64_wasi,
|
||||
# ),
|
||||
]
|
||||
|
||||
PROFILES = {p.name: p for p in _profiles}
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
"wasm_build.py",
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean", "-c", help="Clean build directories first", action="store_true"
|
||||
)
|
||||
|
||||
platforms = list(PROFILES) + ["cleanall"]
|
||||
parser.add_argument(
|
||||
"platform",
|
||||
metavar="PLATFORM",
|
||||
help=f"Build platform: {', '.join(platforms)}",
|
||||
choices=platforms,
|
||||
)
|
||||
|
||||
ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall"]
|
||||
parser.add_argument(
|
||||
"op",
|
||||
metavar="OP",
|
||||
help=f"operation: {', '.join(ops)}",
|
||||
choices=ops,
|
||||
default="compile",
|
||||
nargs="?",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = parser.parse_args()
|
||||
if args.platform == "cleanall":
|
||||
for builder in PROFILES.values():
|
||||
builder.clean(all=True)
|
||||
parser.exit(0)
|
||||
|
||||
builder = PROFILES[args.platform]
|
||||
try:
|
||||
builder.host.platform.check()
|
||||
except ConditionError as e:
|
||||
parser.error(str(e))
|
||||
|
||||
# hack for WASI
|
||||
if builder.host.is_wasi and not SETUP_LOCAL.exists():
|
||||
SETUP_LOCAL.touch()
|
||||
|
||||
if args.op in {"compile", "pythoninfo", "repl", "test"}:
|
||||
# all targets need a build Python
|
||||
if builder is not BUILD:
|
||||
if args.clean:
|
||||
BUILD.clean(all=False)
|
||||
BUILD.run_build()
|
||||
elif not BUILD.python_cmd.exists():
|
||||
BUILD.run_build()
|
||||
|
||||
if args.clean:
|
||||
builder.clean(all=False)
|
||||
|
||||
if args.op == "compile":
|
||||
builder.run_build(force_configure=True)
|
||||
else:
|
||||
if not builder.makefile.exists():
|
||||
builder.run_configure()
|
||||
if args.op == "pythoninfo":
|
||||
builder.run_pythoninfo()
|
||||
elif args.op == "repl":
|
||||
builder.run_py()
|
||||
elif args.op == "test":
|
||||
builder.run_test()
|
||||
elif args.op == "clean":
|
||||
builder.clean(all=False)
|
||||
elif args.op == "cleanall":
|
||||
builder.clean(all=True)
|
||||
else:
|
||||
raise ValueError(args.op)
|
||||
|
||||
print(builder.builddir)
|
||||
parser.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
166
configure
generated
vendored
166
configure
generated
vendored
|
|
@ -906,6 +906,7 @@ AR
|
|||
LINK_PYTHON_OBJS
|
||||
LINK_PYTHON_DEPS
|
||||
LIBRARY_DEPS
|
||||
NODE
|
||||
HOSTRUNNER
|
||||
STATIC_LIBPYTHON
|
||||
GNULD
|
||||
|
|
@ -4079,6 +4080,16 @@ if test -z "$CFLAGS"; then
|
|||
CFLAGS=
|
||||
fi
|
||||
|
||||
case $host in #(
|
||||
wasm64-*-emscripten) :
|
||||
|
||||
as_fn_append CFLAGS " -sMEMORY64=1"
|
||||
as_fn_append LDFLAGS " -sMEMORY64=1"
|
||||
;; #(
|
||||
*) :
|
||||
;;
|
||||
esac
|
||||
|
||||
if test "$ac_sys_system" = "Darwin"
|
||||
then
|
||||
# Extract the first word of "xcrun", so it can be a program name with args.
|
||||
|
|
@ -6220,7 +6231,7 @@ cat > conftest.c <<EOF
|
|||
# error unknown wasm32 platform
|
||||
# endif
|
||||
#elif defined(__wasm64__)
|
||||
# if defined(__EMSCRIPTEN)
|
||||
# if defined(__EMSCRIPTEN__)
|
||||
wasm64-emscripten
|
||||
# elif defined(__wasi__)
|
||||
wasm64-wasi
|
||||
|
|
@ -6840,19 +6851,162 @@ if test "$cross_compiling" = yes; then
|
|||
fi
|
||||
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5
|
||||
$as_echo_n "checking HOSTRUNNER... " >&6; }
|
||||
if test -z "$HOSTRUNNER"
|
||||
then
|
||||
case $ac_sys_system/$ac_sys_emscripten_target in #(
|
||||
Emscripten/node*) :
|
||||
|
||||
if test -n "$ac_tool_prefix"; then
|
||||
# Extract the first word of "${ac_tool_prefix}node", so it can be a program name with args.
|
||||
set dummy ${ac_tool_prefix}node; ac_word=$2
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
|
||||
$as_echo_n "checking for $ac_word... " >&6; }
|
||||
if ${ac_cv_path_NODE+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
case $NODE in
|
||||
[\\/]* | ?:[\\/]*)
|
||||
ac_cv_path_NODE="$NODE" # Let the user override the test with a path.
|
||||
;;
|
||||
*)
|
||||
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
|
||||
for as_dir in $PATH
|
||||
do
|
||||
IFS=$as_save_IFS
|
||||
test -z "$as_dir" && as_dir=.
|
||||
for ac_exec_ext in '' $ac_executable_extensions; do
|
||||
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
|
||||
ac_cv_path_NODE="$as_dir/$ac_word$ac_exec_ext"
|
||||
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
|
||||
break 2
|
||||
fi
|
||||
done
|
||||
done
|
||||
IFS=$as_save_IFS
|
||||
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
NODE=$ac_cv_path_NODE
|
||||
if test -n "$NODE"; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $NODE" >&5
|
||||
$as_echo "$NODE" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
$as_echo "no" >&6; }
|
||||
fi
|
||||
|
||||
|
||||
fi
|
||||
if test -z "$ac_cv_path_NODE"; then
|
||||
ac_pt_NODE=$NODE
|
||||
# Extract the first word of "node", so it can be a program name with args.
|
||||
set dummy node; ac_word=$2
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
|
||||
$as_echo_n "checking for $ac_word... " >&6; }
|
||||
if ${ac_cv_path_ac_pt_NODE+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
case $ac_pt_NODE in
|
||||
[\\/]* | ?:[\\/]*)
|
||||
ac_cv_path_ac_pt_NODE="$ac_pt_NODE" # Let the user override the test with a path.
|
||||
;;
|
||||
*)
|
||||
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
|
||||
for as_dir in $PATH
|
||||
do
|
||||
IFS=$as_save_IFS
|
||||
test -z "$as_dir" && as_dir=.
|
||||
for ac_exec_ext in '' $ac_executable_extensions; do
|
||||
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
|
||||
ac_cv_path_ac_pt_NODE="$as_dir/$ac_word$ac_exec_ext"
|
||||
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
|
||||
break 2
|
||||
fi
|
||||
done
|
||||
done
|
||||
IFS=$as_save_IFS
|
||||
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
ac_pt_NODE=$ac_cv_path_ac_pt_NODE
|
||||
if test -n "$ac_pt_NODE"; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_NODE" >&5
|
||||
$as_echo "$ac_pt_NODE" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
$as_echo "no" >&6; }
|
||||
fi
|
||||
|
||||
if test "x$ac_pt_NODE" = x; then
|
||||
NODE="node"
|
||||
else
|
||||
case $cross_compiling:$ac_tool_warned in
|
||||
yes:)
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
|
||||
$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
|
||||
ac_tool_warned=yes ;;
|
||||
esac
|
||||
NODE=$ac_pt_NODE
|
||||
fi
|
||||
else
|
||||
NODE="$ac_cv_path_NODE"
|
||||
fi
|
||||
|
||||
HOSTRUNNER="$NODE"
|
||||
# bigint for ctypes c_longlong, c_longdouble
|
||||
HOSTRUNNER="node --experimental-wasm-bigint"
|
||||
# no longer available in Node 16
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bigint" >&5
|
||||
$as_echo_n "checking for node --experimental-wasm-bigint... " >&6; }
|
||||
if ${ac_cv_tool_node_wasm_bigint+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
|
||||
if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then
|
||||
ac_cv_tool_node_wasm_bigint=yes
|
||||
else
|
||||
ac_cv_tool_node_wasm_bigint=no
|
||||
fi
|
||||
|
||||
fi
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bigint" >&5
|
||||
$as_echo "$ac_cv_tool_node_wasm_bigint" >&6; }
|
||||
if test "x$ac_cv_tool_node_wasm_bigint" = xyes; then :
|
||||
|
||||
as_fn_append HOSTRUNNER " --experimental-wasm-bigint"
|
||||
|
||||
fi
|
||||
|
||||
if test "x$enable_wasm_pthreads" = xyes; then :
|
||||
|
||||
HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory"
|
||||
as_fn_append HOSTRUNNER " --experimental-wasm-threads"
|
||||
# no longer available in Node 16
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bulk-memory" >&5
|
||||
$as_echo_n "checking for node --experimental-wasm-bulk-memory... " >&6; }
|
||||
if ${ac_cv_tool_node_wasm_bulk_memory+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
|
||||
if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then
|
||||
ac_cv_tool_node_wasm_bulk_memory=yes
|
||||
else
|
||||
ac_cv_tool_node_wasm_bulk_memory=no
|
||||
fi
|
||||
|
||||
fi
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bulk_memory" >&5
|
||||
$as_echo "$ac_cv_tool_node_wasm_bulk_memory" >&6; }
|
||||
if test "x$ac_cv_tool_node_wasm_bulk_memory" = xyes; then :
|
||||
|
||||
as_fn_append HOSTRUNNER " --experimental-wasm-bulk-memory"
|
||||
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
if test "x$host_cpu" = xwasm64; then :
|
||||
as_fn_append HOSTRUNNER " --experimental-wasm-memory64"
|
||||
fi
|
||||
;; #(
|
||||
WASI/*) :
|
||||
|
|
@ -6863,6 +7017,8 @@ fi
|
|||
esac
|
||||
fi
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5
|
||||
$as_echo_n "checking HOSTRUNNER... " >&6; }
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5
|
||||
$as_echo "$HOSTRUNNER" >&6; }
|
||||
|
||||
|
|
|
|||
46
configure.ac
46
configure.ac
|
|
@ -753,6 +753,16 @@ if test -z "$CFLAGS"; then
|
|||
CFLAGS=
|
||||
fi
|
||||
|
||||
dnl Emscripten SDK and WASI SDK default to wasm32.
|
||||
dnl On Emscripten use MEMORY64 setting to build target wasm64-emscripten.
|
||||
dnl for wasm64.
|
||||
AS_CASE([$host],
|
||||
[wasm64-*-emscripten], [
|
||||
AS_VAR_APPEND([CFLAGS], [" -sMEMORY64=1"])
|
||||
AS_VAR_APPEND([LDFLAGS], [" -sMEMORY64=1"])
|
||||
],
|
||||
)
|
||||
|
||||
if test "$ac_sys_system" = "Darwin"
|
||||
then
|
||||
dnl look for SDKROOT
|
||||
|
|
@ -1048,7 +1058,7 @@ cat > conftest.c <<EOF
|
|||
# error unknown wasm32 platform
|
||||
# endif
|
||||
#elif defined(__wasm64__)
|
||||
# if defined(__EMSCRIPTEN)
|
||||
# if defined(__EMSCRIPTEN__)
|
||||
wasm64-emscripten
|
||||
# elif defined(__wasi__)
|
||||
wasm64-wasi
|
||||
|
|
@ -1515,16 +1525,41 @@ if test "$cross_compiling" = yes; then
|
|||
fi
|
||||
|
||||
AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform])
|
||||
AC_MSG_CHECKING([HOSTRUNNER])
|
||||
if test -z "$HOSTRUNNER"
|
||||
then
|
||||
AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
|
||||
[Emscripten/node*], [
|
||||
AC_PATH_TOOL([NODE], [node], [node])
|
||||
HOSTRUNNER="$NODE"
|
||||
# bigint for ctypes c_longlong, c_longdouble
|
||||
HOSTRUNNER="node --experimental-wasm-bigint"
|
||||
AS_VAR_IF([enable_wasm_pthreads], [yes], [
|
||||
HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory"
|
||||
# no longer available in Node 16
|
||||
AC_CACHE_CHECK([for node --experimental-wasm-bigint], [ac_cv_tool_node_wasm_bigint], [
|
||||
if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then
|
||||
ac_cv_tool_node_wasm_bigint=yes
|
||||
else
|
||||
ac_cv_tool_node_wasm_bigint=no
|
||||
fi
|
||||
])
|
||||
AS_VAR_IF([ac_cv_tool_node_wasm_bigint], [yes], [
|
||||
AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bigint"])
|
||||
])
|
||||
|
||||
AS_VAR_IF([enable_wasm_pthreads], [yes], [
|
||||
AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"])
|
||||
# no longer available in Node 16
|
||||
AC_CACHE_CHECK([for node --experimental-wasm-bulk-memory], [ac_cv_tool_node_wasm_bulk_memory], [
|
||||
if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then
|
||||
ac_cv_tool_node_wasm_bulk_memory=yes
|
||||
else
|
||||
ac_cv_tool_node_wasm_bulk_memory=no
|
||||
fi
|
||||
])
|
||||
AS_VAR_IF([ac_cv_tool_node_wasm_bulk_memory], [yes], [
|
||||
AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bulk-memory"])
|
||||
])
|
||||
])
|
||||
|
||||
AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])])
|
||||
],
|
||||
dnl TODO: support other WASI runtimes
|
||||
dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the
|
||||
|
|
@ -1534,6 +1569,7 @@ then
|
|||
)
|
||||
fi
|
||||
AC_SUBST([HOSTRUNNER])
|
||||
AC_MSG_CHECKING([HOSTRUNNER])
|
||||
AC_MSG_RESULT([$HOSTRUNNER])
|
||||
|
||||
if test -n "$HOSTRUNNER"; then
|
||||
|
|
|
|||
Loading…
Reference in a new issue