mirror of
https://github.com/python/cpython
synced 2024-10-14 15:21:52 +00:00
bpo-40280: WASM docs and smaller browser builds (GH-32412)
Co-authored-by: Brett Cannon <brett@python.org>
This commit is contained in:
parent
dc14e33eff
commit
defbbd68f7
|
@ -1183,7 +1183,9 @@ always available.
|
||||||
System ``platform`` value
|
System ``platform`` value
|
||||||
================ ===========================
|
================ ===========================
|
||||||
AIX ``'aix'``
|
AIX ``'aix'``
|
||||||
|
Emscripten ``'emscripten'``
|
||||||
Linux ``'linux'``
|
Linux ``'linux'``
|
||||||
|
WASI ``'wasi'``
|
||||||
Windows ``'win32'``
|
Windows ``'win32'``
|
||||||
Windows/Cygwin ``'cygwin'``
|
Windows/Cygwin ``'cygwin'``
|
||||||
macOS ``'darwin'``
|
macOS ``'darwin'``
|
||||||
|
|
|
@ -812,8 +812,9 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
|
||||||
# --preload-file turns a relative asset path into an absolute path.
|
# --preload-file turns a relative asset path into an absolute path.
|
||||||
|
|
||||||
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
|
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
|
||||||
pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py \
|
$(srcdir)/Tools/wasm/wasm_assets.py \
|
||||||
python.html python.worker.js
|
Makefile pybuilddir.txt Modules/Setup.local \
|
||||||
|
python.html python.worker.js
|
||||||
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
|
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
|
||||||
--builddir . --prefix $(prefix)
|
--builddir . --prefix $(prefix)
|
||||||
|
|
||||||
|
|
|
@ -9,9 +9,21 @@ # Python WebAssembly (WASM) build
|
||||||
|
|
||||||
## wasm32-emscripten build
|
## wasm32-emscripten build
|
||||||
|
|
||||||
Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
|
Cross compiling to the wasm32-emscripten platform needs the
|
||||||
tool chain and a build Python interpreter.
|
[Emscripten](https://emscripten.org/) SDK and a build Python interpreter.
|
||||||
All commands below are relative to a repository checkout.
|
Emscripten 3.1.8 or newer are recommended. All commands below are relative
|
||||||
|
to a repository checkout.
|
||||||
|
|
||||||
|
Christian Heimes maintains a container image with Emscripten SDK, Python
|
||||||
|
build dependencies, WASI-SDK, wasmtime, and several additional tools.
|
||||||
|
|
||||||
|
```
|
||||||
|
# Fedora, RHEL, CentOS
|
||||||
|
podman run --rm -ti -v $(pwd):/python-wasm/cpython:Z quay.io/tiran/cpythonbuild:emsdk3
|
||||||
|
|
||||||
|
# other
|
||||||
|
docker run --rm -ti -v $(pwd):/python-wasm/cpython quay.io/tiran/cpythonbuild:emsdk3
|
||||||
|
```
|
||||||
|
|
||||||
### Compile a build Python interpreter
|
### Compile a build Python interpreter
|
||||||
|
|
||||||
|
@ -167,3 +179,77 @@ ## wasm32-emscripten in node
|
||||||
- pthread support requires WASM threads and SharedArrayBuffer (bulk memory).
|
- pthread support requires WASM threads and SharedArrayBuffer (bulk memory).
|
||||||
The runtime keeps a pool of web workers around. Each web worker uses
|
The runtime keeps a pool of web workers around. Each web worker uses
|
||||||
several file descriptors (eventfd, epoll, pipe).
|
several file descriptors (eventfd, epoll, pipe).
|
||||||
|
|
||||||
|
# Hosting Python WASM builds
|
||||||
|
|
||||||
|
The simple REPL terminal uses SharedArrayBuffer. For security reasons
|
||||||
|
browsers only provide the feature in secure environents with cross-origin
|
||||||
|
isolation. The webserver must send cross-origin headers and correct MIME types
|
||||||
|
for the JavaScript and WASM files. Otherwise the terminal will fail to load
|
||||||
|
with an error message like ``Browsers disable shared array buffer``.
|
||||||
|
|
||||||
|
## Apache HTTP .htaccess
|
||||||
|
|
||||||
|
Place a ``.htaccess`` file in the same directory as ``python.wasm``.
|
||||||
|
|
||||||
|
```
|
||||||
|
# .htaccess
|
||||||
|
Header set Cross-Origin-Opener-Policy same-origin
|
||||||
|
Header set Cross-Origin-Embedder-Policy require-corp
|
||||||
|
|
||||||
|
AddType application/javascript js
|
||||||
|
AddType application/wasm wasm
|
||||||
|
|
||||||
|
<IfModule mod_deflate.c>
|
||||||
|
AddOutputFilterByType DEFLATE text/html application/javascript application/wasm
|
||||||
|
</IfModule>
|
||||||
|
```
|
||||||
|
|
||||||
|
# Detect WebAssembly builds
|
||||||
|
|
||||||
|
## Python code
|
||||||
|
|
||||||
|
```# python
|
||||||
|
import os, sys
|
||||||
|
|
||||||
|
if sys.platform == "emscripten":
|
||||||
|
# Python on Emscripten
|
||||||
|
if sys.platform == "wasi":
|
||||||
|
# Python on WASI
|
||||||
|
|
||||||
|
if os.name == "posix":
|
||||||
|
# WASM platforms identify as POSIX-like.
|
||||||
|
# Windows does not provide os.uname().
|
||||||
|
machine = os.uname().machine
|
||||||
|
if machine.startswith("wasm"):
|
||||||
|
# WebAssembly (wasm32 or wasm64)
|
||||||
|
```
|
||||||
|
|
||||||
|
## C code
|
||||||
|
|
||||||
|
Emscripten SDK and WASI SDK define several built-in macros. You can dump a
|
||||||
|
full list of built-ins with ``emcc -dM -E - < /dev/null`` and
|
||||||
|
``/path/to/wasi-sdk/bin/clang -dM -E - < /dev/null``.
|
||||||
|
|
||||||
|
```# C
|
||||||
|
#ifdef __EMSCRIPTEN__
|
||||||
|
// Python on Emscripten
|
||||||
|
#endif
|
||||||
|
```
|
||||||
|
|
||||||
|
* WebAssembly ``__wasm__`` (also ``__wasm``)
|
||||||
|
* wasm32 ``__wasm32__`` (also ``__wasm32``)
|
||||||
|
* wasm64 ``__wasm64__``
|
||||||
|
* Emscripten ``__EMSCRIPTEN__`` (also ``EMSCRIPTEN``)
|
||||||
|
* Emscripten version ``__EMSCRIPTEN_major__``, ``__EMSCRIPTEN_minor__``, ``__EMSCRIPTEN_tiny__``
|
||||||
|
* WASI ``__wasi__``
|
||||||
|
|
||||||
|
Feature detection flags:
|
||||||
|
|
||||||
|
* ``__EMSCRIPTEN_PTHREADS__``
|
||||||
|
* ``__EMSCRIPTEN_SHARED_MEMORY__``
|
||||||
|
* ``__wasm_simd128__``
|
||||||
|
* ``__wasm_sign_ext__``
|
||||||
|
* ``__wasm_bulk_memory__``
|
||||||
|
* ``__wasm_atomics__``
|
||||||
|
* ``__wasm_mutable_globals__``
|
||||||
|
|
15
Tools/wasm/Setup.local.example
Normal file
15
Tools/wasm/Setup.local.example
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
# Module/Setup.local with reduced stdlib
|
||||||
|
*disabled*
|
||||||
|
_asyncio
|
||||||
|
audioop
|
||||||
|
_bz2
|
||||||
|
_crypt
|
||||||
|
_decimal
|
||||||
|
_pickle
|
||||||
|
pyexpat _elementtree
|
||||||
|
_sha3 _blake2
|
||||||
|
_zoneinfo
|
||||||
|
xxsubtype
|
||||||
|
|
||||||
|
# cjk codecs
|
||||||
|
#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw
|
|
@ -20,7 +20,11 @@
|
||||||
SRCDIR_LIB = SRCDIR / "Lib"
|
SRCDIR_LIB = SRCDIR / "Lib"
|
||||||
|
|
||||||
# sysconfig data relative to build dir.
|
# sysconfig data relative to build dir.
|
||||||
SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
|
SYSCONFIGDATA = pathlib.PurePath(
|
||||||
|
"build",
|
||||||
|
f"lib.emscripten-wasm32-{sys.version_info.major}.{sys.version_info.minor}",
|
||||||
|
"_sysconfigdata__emscripten_wasm32-emscripten.py",
|
||||||
|
)
|
||||||
|
|
||||||
# Library directory relative to $(prefix).
|
# Library directory relative to $(prefix).
|
||||||
WASM_LIB = pathlib.PurePath("lib")
|
WASM_LIB = pathlib.PurePath("lib")
|
||||||
|
@ -38,33 +42,44 @@
|
||||||
OMIT_FILES = (
|
OMIT_FILES = (
|
||||||
# regression tests
|
# regression tests
|
||||||
"test/",
|
"test/",
|
||||||
# user interfaces: TK, curses
|
|
||||||
"curses/",
|
|
||||||
"idlelib/",
|
|
||||||
"tkinter/",
|
|
||||||
"turtle.py",
|
|
||||||
"turtledemo/",
|
|
||||||
# package management
|
# package management
|
||||||
"ensurepip/",
|
"ensurepip/",
|
||||||
"venv/",
|
"venv/",
|
||||||
# build system
|
# build system
|
||||||
"distutils/",
|
"distutils/",
|
||||||
"lib2to3/",
|
"lib2to3/",
|
||||||
# concurrency
|
|
||||||
"concurrent/",
|
|
||||||
"multiprocessing/",
|
|
||||||
# deprecated
|
# deprecated
|
||||||
"asyncore.py",
|
"asyncore.py",
|
||||||
"asynchat.py",
|
"asynchat.py",
|
||||||
# Synchronous network I/O and protocols are not supported; for example,
|
"uu.py",
|
||||||
# socket.create_connection() raises an exception:
|
"xdrlib.py",
|
||||||
# "BlockingIOError: [Errno 26] Operation in progress".
|
# other platforms
|
||||||
|
"_aix_support.py",
|
||||||
|
"_bootsubprocess.py",
|
||||||
|
"_osx_support.py",
|
||||||
|
# webbrowser
|
||||||
|
"antigravity.py",
|
||||||
|
"webbrowser.py",
|
||||||
|
# Pure Python implementations of C extensions
|
||||||
|
"_pydecimal.py",
|
||||||
|
"_pyio.py",
|
||||||
|
# Misc unused or large files
|
||||||
|
"pydoc_data/",
|
||||||
|
"msilib/",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Synchronous network I/O and protocols are not supported; for example,
|
||||||
|
# socket.create_connection() raises an exception:
|
||||||
|
# "BlockingIOError: [Errno 26] Operation in progress".
|
||||||
|
OMIT_NETWORKING_FILES = (
|
||||||
"cgi.py",
|
"cgi.py",
|
||||||
"cgitb.py",
|
"cgitb.py",
|
||||||
"email/",
|
"email/",
|
||||||
"ftplib.py",
|
"ftplib.py",
|
||||||
"http/",
|
"http/",
|
||||||
"imaplib.py",
|
"imaplib.py",
|
||||||
|
"mailbox.py",
|
||||||
|
"mailcap.py",
|
||||||
"nntplib.py",
|
"nntplib.py",
|
||||||
"poplib.py",
|
"poplib.py",
|
||||||
"smtpd.py",
|
"smtpd.py",
|
||||||
|
@ -77,26 +92,28 @@
|
||||||
"urllib/response.py",
|
"urllib/response.py",
|
||||||
"urllib/robotparser.py",
|
"urllib/robotparser.py",
|
||||||
"wsgiref/",
|
"wsgiref/",
|
||||||
"xmlrpc/",
|
|
||||||
# dbm / gdbm
|
|
||||||
"dbm/",
|
|
||||||
# other platforms
|
|
||||||
"_aix_support.py",
|
|
||||||
"_bootsubprocess.py",
|
|
||||||
"_osx_support.py",
|
|
||||||
# webbrowser
|
|
||||||
"antigravity.py",
|
|
||||||
"webbrowser.py",
|
|
||||||
# ctypes
|
|
||||||
"ctypes/",
|
|
||||||
# Pure Python implementations of C extensions
|
|
||||||
"_pydecimal.py",
|
|
||||||
"_pyio.py",
|
|
||||||
# Misc unused or large files
|
|
||||||
"pydoc_data/",
|
|
||||||
"msilib/",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
OMIT_MODULE_FILES = {
|
||||||
|
"_asyncio": ["asyncio/"],
|
||||||
|
"audioop": ["aifc.py", "sunau.py", "wave.py"],
|
||||||
|
"_crypt": ["crypt.py"],
|
||||||
|
"_curses": ["curses/"],
|
||||||
|
"_ctypes": ["ctypes/"],
|
||||||
|
"_decimal": ["decimal.py"],
|
||||||
|
"_dbm": ["dbm/ndbm.py"],
|
||||||
|
"_gdbm": ["dbm/gnu.py"],
|
||||||
|
"_json": ["json/"],
|
||||||
|
"_multiprocessing": ["concurrent/", "multiprocessing/"],
|
||||||
|
"pyexpat": ["xml/", "xmlrpc/"],
|
||||||
|
"readline": ["rlcompleter.py"],
|
||||||
|
"_sqlite3": ["sqlite3/"],
|
||||||
|
"_ssl": ["ssl.py"],
|
||||||
|
"_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"],
|
||||||
|
|
||||||
|
"_zoneinfo": ["zoneinfo/"],
|
||||||
|
}
|
||||||
|
|
||||||
# regression test sub directories
|
# regression test sub directories
|
||||||
OMIT_SUBDIRS = (
|
OMIT_SUBDIRS = (
|
||||||
"ctypes/test/",
|
"ctypes/test/",
|
||||||
|
@ -105,34 +122,59 @@
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
|
|
||||||
OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
|
|
||||||
|
|
||||||
|
|
||||||
def filterfunc(name: str) -> bool:
|
|
||||||
return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
|
|
||||||
|
|
||||||
|
|
||||||
def create_stdlib_zip(
|
def create_stdlib_zip(
|
||||||
args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
|
args: argparse.Namespace,
|
||||||
|
*,
|
||||||
|
optimize: int = 0,
|
||||||
) -> None:
|
) -> None:
|
||||||
sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
|
def filterfunc(name: str) -> bool:
|
||||||
if not sysconfig_data:
|
return not name.startswith(args.omit_subdirs_absolute)
|
||||||
raise ValueError("No sysconfigdata file found")
|
|
||||||
|
|
||||||
with zipfile.PyZipFile(
|
with zipfile.PyZipFile(
|
||||||
args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
|
args.wasm_stdlib_zip, mode="w", compression=args.compression, optimize=optimize
|
||||||
) as pzf:
|
) as pzf:
|
||||||
|
if args.compresslevel is not None:
|
||||||
|
pzf.compresslevel = args.compresslevel
|
||||||
|
pzf.writepy(args.sysconfig_data)
|
||||||
for entry in sorted(args.srcdir_lib.iterdir()):
|
for entry in sorted(args.srcdir_lib.iterdir()):
|
||||||
if entry.name == "__pycache__":
|
if entry.name == "__pycache__":
|
||||||
continue
|
continue
|
||||||
if entry in OMIT_ABSOLUTE:
|
if entry in args.omit_files_absolute:
|
||||||
continue
|
continue
|
||||||
if entry.name.endswith(".py") or entry.is_dir():
|
if entry.name.endswith(".py") or entry.is_dir():
|
||||||
# writepy() writes .pyc files (bytecode).
|
# writepy() writes .pyc files (bytecode).
|
||||||
pzf.writepy(entry, filterfunc=filterfunc)
|
pzf.writepy(entry, filterfunc=filterfunc)
|
||||||
for entry in sysconfig_data:
|
|
||||||
pzf.writepy(entry)
|
|
||||||
|
def detect_extension_modules(args: argparse.Namespace):
|
||||||
|
modules = {}
|
||||||
|
|
||||||
|
# disabled by Modules/Setup.local ?
|
||||||
|
with open(args.builddir / "Makefile") as f:
|
||||||
|
for line in f:
|
||||||
|
if line.startswith("MODDISABLED_NAMES="):
|
||||||
|
disabled = line.split("=", 1)[1].strip().split()
|
||||||
|
for modname in disabled:
|
||||||
|
modules[modname] = False
|
||||||
|
break
|
||||||
|
|
||||||
|
# disabled by configure?
|
||||||
|
with open(args.sysconfig_data) as f:
|
||||||
|
data = f.read()
|
||||||
|
loc = {}
|
||||||
|
exec(data, globals(), loc)
|
||||||
|
|
||||||
|
for name, value in loc["build_time_vars"].items():
|
||||||
|
if value not in {"yes", "missing", "disabled", "n/a"}:
|
||||||
|
continue
|
||||||
|
if not name.startswith("MODULE_"):
|
||||||
|
continue
|
||||||
|
if name.endswith(("_CFLAGS", "_DEPS", "_LDFLAGS")):
|
||||||
|
continue
|
||||||
|
modname = name.removeprefix("MODULE_").lower()
|
||||||
|
if modname not in modules:
|
||||||
|
modules[modname] = value == "yes"
|
||||||
|
return modules
|
||||||
|
|
||||||
|
|
||||||
def path(val: str) -> pathlib.Path:
|
def path(val: str) -> pathlib.Path:
|
||||||
|
@ -147,7 +189,10 @@ def path(val: str) -> pathlib.Path:
|
||||||
type=path,
|
type=path,
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
|
"--prefix",
|
||||||
|
help="install prefix",
|
||||||
|
default=pathlib.Path("/usr/local"),
|
||||||
|
type=path,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -162,6 +207,27 @@ def main():
|
||||||
args.wasm_stdlib = args.wasm_root / WASM_STDLIB
|
args.wasm_stdlib = args.wasm_root / WASM_STDLIB
|
||||||
args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
|
args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
|
||||||
|
|
||||||
|
# bpo-17004: zipimport supports only zlib compression.
|
||||||
|
# Emscripten ZIP_STORED + -sLZ4=1 linker flags results in larger file.
|
||||||
|
args.compression = zipfile.ZIP_DEFLATED
|
||||||
|
args.compresslevel = 9
|
||||||
|
|
||||||
|
args.sysconfig_data = args.builddir / SYSCONFIGDATA
|
||||||
|
if not args.sysconfig_data.is_file():
|
||||||
|
raise ValueError(f"sysconfigdata file {SYSCONFIGDATA} missing.")
|
||||||
|
|
||||||
|
extmods = detect_extension_modules(args)
|
||||||
|
omit_files = list(OMIT_FILES)
|
||||||
|
omit_files.extend(OMIT_NETWORKING_FILES)
|
||||||
|
for modname, modfiles in OMIT_MODULE_FILES.items():
|
||||||
|
if not extmods.get(modname):
|
||||||
|
omit_files.extend(modfiles)
|
||||||
|
|
||||||
|
args.omit_files_absolute = {args.srcdir_lib / name for name in omit_files}
|
||||||
|
args.omit_subdirs_absolute = tuple(
|
||||||
|
str(args.srcdir_lib / name) for name in OMIT_SUBDIRS
|
||||||
|
)
|
||||||
|
|
||||||
# Empty, unused directory for dynamic libs, but required for site initialization.
|
# Empty, unused directory for dynamic libs, but required for site initialization.
|
||||||
args.wasm_dynload.mkdir(parents=True, exist_ok=True)
|
args.wasm_dynload.mkdir(parents=True, exist_ok=True)
|
||||||
marker = args.wasm_dynload / ".empty"
|
marker = args.wasm_dynload / ".empty"
|
||||||
|
@ -170,7 +236,7 @@ def main():
|
||||||
shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
|
shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
|
||||||
# The rest of stdlib that's useful in a WASM context.
|
# The rest of stdlib that's useful in a WASM context.
|
||||||
create_stdlib_zip(args)
|
create_stdlib_zip(args)
|
||||||
size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
|
size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2)
|
||||||
parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
|
parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,13 @@
|
||||||
|
|
||||||
|
|
||||||
class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
|
class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
|
||||||
|
extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy()
|
||||||
|
extensions_map.update(
|
||||||
|
{
|
||||||
|
".wasm": "application/wasm",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def end_headers(self):
|
def end_headers(self):
|
||||||
self.send_my_headers()
|
self.send_my_headers()
|
||||||
super().end_headers()
|
super().end_headers()
|
||||||
|
|
Loading…
Reference in a new issue