languages/cpython
languages/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2024-07-20 09:45:46 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

gh-93939: Add script to check extension modules (#94545)

Browse source 
Add script ``Tools/scripts/check_modules.py`` to check and validate builtin
and shared extension modules. The script also handles ``Modules/Setup`` and
will eventually replace ``setup.py``.

Co-authored-by: Victor Stinner <vstinner@python.org>
Co-authored-by: Erlend Egeberg Aasland <erlend.aasland@protonmail.com>
This commit is contained in:
Christian Heimes 2022-07-05 09:25:45 +02:00 committed by GitHub
parent fd76eb547d
commit 7bd67d1d88
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 504 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Makefile.pre.in
View file

@ -918,6 +918,9 @@ oldsharedmods: $(SHAREDMODS) pybuilddir.txt
fi; \
done
checksharedmods: oldsharedmods sharedmods $(PYTHON_FOR_BUILD_DEPS)
@$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/scripts/check_extension_modules.py
Modules/Setup.local:
@# Create empty Setup.local when file was deleted by user
echo "# Edit this file for local setup changes" > $@
@ -2531,7 +2534,8 @@ update-config:
Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
# Declare targets that aren't real files
.PHONY: all build_all build_wasm sharedmods check-clean-src oldsharedmods test quicktest
.PHONY: all build_all build_wasm sharedmods check-clean-src
.PHONY: oldsharedmods checksharedmods test quicktest
.PHONY: install altinstall oldsharedinstall bininstall altbininstall
.PHONY: maninstall libinstall inclinstall libainstall sharedinstall
.PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure

3
Misc/NEWS.d/next/Tools-Demos/2022-07-04-10-02-02.gh-issue-93939.U6sW6H.rst Normal file
View file

@ -0,0 +1,3 @@
Add script ``Tools/scripts/check_modules.py`` to check and validate builtin
and shared extension modules. The script also handles ``Modules/Setup`` and
will eventually replace ``setup.py``.

489
Tools/scripts/check_extension_modules.py Normal file
View file

@ -0,0 +1,489 @@
"""Check extension modules
The script checks shared and built-in extension modules. It verifies that the
modules have been built and that they can be imported successfully. Missing
modules and failed imports are reported to the user. Shared extension
files are renamed on failed import.
Module information is parsed from several sources:
- core modules hard-coded in Modules/config.c.in
- Windows-specific modules that are hard-coded in PC/config.c
- MODULE_{name}_STATE entries in Makefile (provided through sysconfig)
- Various makesetup files:
- $(srcdir)/Modules/Setup
- Modules/Setup.[local|bootstrap|stdlib] files, which are generated
from $(srcdir)/Modules/Setup.*.in files
See --help for more information
"""
import argparse
import collections
import enum
import logging
import os
import pathlib
import re
import sys
import sysconfig
import warnings
from importlib._bootstrap import _load as bootstrap_load
from importlib.machinery import BuiltinImporter, ExtensionFileLoader, ModuleSpec
from importlib.util import spec_from_file_location, spec_from_loader
from typing import Iterable
SRC_DIR = pathlib.Path(__file__).parent.parent.parent
# core modules, hard-coded in Modules/config.h.in
CORE_MODULES = {
"_ast",
"_imp",
"_string",
"_tokenize",
"_warnings",
"builtins",
"gc",
"marshal",
"sys",
}
# Windows-only modules
WINDOWS_MODULES = {
"_msi",
"_overlapped",
"_testconsole",
"_winapi",
"msvcrt",
"nt",
"winreg",
"winsound",
}
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser(
prog="check_extension_modules",
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--verbose",
action="store_true",
help="Verbose, report builtin, shared, and unavailable modules",
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable debug logging",
)
parser.add_argument(
"--strict",
action=argparse.BooleanOptionalAction,
help=(
"Strict check, fail when a module is missing or fails to import"
"(default: no, unless env var PYTHONSTRICTEXTENSIONBUILD is set)"
),
default=bool(os.environ.get("PYTHONSTRICTEXTENSIONBUILD")),
)
parser.add_argument(
"--cross-compiling",
action=argparse.BooleanOptionalAction,
help=(
"Use cross-compiling checks "
"(default: no, unless env var _PYTHON_HOST_PLATFORM is set)."
),
default="_PYTHON_HOST_PLATFORM" in os.environ,
)
parser.add_argument(
"--list-module-names",
action="store_true",
help="Print a list of module names to stdout and exit",
)
class ModuleState(enum.Enum):
# Makefile state "yes"
BUILTIN = "builtin"
SHARED = "shared"
DISABLED = "disabled"
MISSING = "missing"
NA = "n/a"
# disabled by Setup / makesetup rule
DISABLED_SETUP = "disabled_setup"
def __bool__(self):
return self.value in {"builtin", "shared"}
ModuleInfo = collections.namedtuple("ModuleInfo", "name state")
class ModuleChecker:
pybuilddir_txt = "pybuilddir.txt"
setup_files = (
SRC_DIR / "Modules/Setup",
"Modules/Setup.local",
"Modules/Setup.bootstrap",
"Modules/Setup.stdlib",
)
def __init__(self, cross_compiling: bool = False, strict: bool = False):
self.cross_compiling = cross_compiling
self.strict_extensions_build = strict
self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
self.platform = sysconfig.get_platform()
self.builddir = self.get_builddir()
self.modules = self.get_modules()
self.builtin_ok = []
self.shared_ok = []
self.failed_on_import = []
self.missing = []
self.disabled_configure = []
self.disabled_setup = []
self.notavailable = []
def check(self):
for modinfo in self.modules:
logger.debug("Checking '%s' (%s)", modinfo.name, self.get_location(modinfo))
if modinfo.state == ModuleState.DISABLED:
self.disabled_configure.append(modinfo)
elif modinfo.state == ModuleState.DISABLED_SETUP:
self.disabled_setup.append(modinfo)
elif modinfo.state == ModuleState.MISSING:
self.missing.append(modinfo)
elif modinfo.state == ModuleState.NA:
self.notavailable.append(modinfo)
else:
try:
if self.cross_compiling:
self.check_module_cross(modinfo)
else:
self.check_module_import(modinfo)
except (ImportError, FileNotFoundError):
self.rename_module(modinfo)
self.failed_on_import.append(modinfo)
else:
if modinfo.state == ModuleState.BUILTIN:
self.builtin_ok.append(modinfo)
else:
assert modinfo.state == ModuleState.SHARED
self.shared_ok.append(modinfo)
def summary(self, *, verbose: bool = False):
longest = max([len(e.name) for e in self.modules], default=0)
def print_three_column(modinfos: list[ModuleInfo]):
names = [modinfo.name for modinfo in modinfos]
names.sort(key=str.lower)
# guarantee zip() doesn't drop anything
while len(names) % 3:
names.append("")
for l, m, r in zip(names[::3], names[1::3], names[2::3]):
print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r))
if verbose and self.builtin_ok:
print("The following *built-in* modules have been successfully built:")
print_three_column(self.builtin_ok)
print()
if verbose and self.shared_ok:
print("The following *shared* modules have been successfully built:")
print_three_column(self.shared_ok)
print()
if self.disabled_configure:
print("The following modules are *disabled* in configure script:")
print_three_column(self.disabled_configure)
print()
if self.disabled_setup:
print("The following modules are *disabled* in Modules/Setup files:")
print_three_column(self.disabled_setup)
print()
if verbose and self.notavailable:
print(
f"The following modules are not available on platform '{self.platform}':"
)
print_three_column(self.notavailable)
print()
if self.missing:
print("The necessary bits to build these optional modules were not found:")
print_three_column(self.missing)
print("To find the necessary bits, look in configure.ac and config.log.")
print()
if self.failed_on_import:
print(
"Following modules built successfully "
"but were removed because they could not be imported:"
)
print_three_column(self.failed_on_import)
print()
if any(
modinfo.name == "_ssl" for modinfo in self.missing + self.failed_on_import
):
print("Could not build the ssl module!")
print("Python requires a OpenSSL 1.1.1 or newer")
if sysconfig.get_config_var("OPENSSL_LDFLAGS"):
print("Custom linker flags may require --with-openssl-rpath=auto")
print()
disabled = len(self.disabled_configure) + len(self.disabled_setup)
print(
f"Checked {len(self.modules)} modules ("
f"{len(self.builtin_ok)} built-in, "
f"{len(self.shared_ok)} shared, "
f"{len(self.notavailable)} n/a on {self.platform}, "
f"{disabled} disabled, "
f"{len(self.missing)} missing, "
f"{len(self.failed_on_import)} failed on import)"
)
def check_strict_build(self):
"""Fail if modules are missing and it's a strict build"""
if self.strict_extensions_build and (self.failed_on_import or self.missing):
raise RuntimeError("Failed to build some stdlib modules")
def list_module_names(self, *, all: bool = False) -> set:
names = {modinfo.name for modinfo in self.modules}
if all:
names.update(WINDOWS_MODULES)
return names
def get_builddir(self) -> pathlib.Path:
try:
with open(self.pybuilddir_txt, encoding="utf-8") as f:
builddir = f.read()
except FileNotFoundError:
logger.error("%s must be run from the top build directory", __file__)
raise
builddir = pathlib.Path(builddir)
logger.debug("%s: %s", self.pybuilddir_txt, builddir)
return builddir
def get_modules(self) -> list[ModuleInfo]:
"""Get module info from sysconfig and Modules/Setup* files"""
seen = set()
modules = []
# parsing order is important, first entry wins
for modinfo in self.get_core_modules():
modules.append(modinfo)
seen.add(modinfo.name)
for setup_file in self.setup_files:
for modinfo in self.parse_setup_file(setup_file):
if modinfo.name not in seen:
modules.append(modinfo)
seen.add(modinfo.name)
for modinfo in self.get_sysconfig_modules():
if modinfo.name not in seen:
modules.append(modinfo)
seen.add(modinfo.name)
logger.debug("Found %i modules in total", len(modules))
modules.sort()
return modules
def get_core_modules(self) -> Iterable[ModuleInfo]:
"""Get hard-coded core modules"""
for name in CORE_MODULES:
modinfo = ModuleInfo(name, ModuleState.BUILTIN)
logger.debug("Found core module %s", modinfo)
yield modinfo
def get_sysconfig_modules(self) -> Iterable[ModuleInfo]:
"""Get modules defined in Makefile through sysconfig
MODBUILT_NAMES: modules in *static* block
MODSHARED_NAMES: modules in *shared* block
MODDISABLED_NAMES: modules in *disabled* block
Modules built by setup.py addext() have a MODULE_{modname}_STATE entry,
but are not listed in MODSHARED_NAMES.
Modules built by old-style setup.py add() have neither a MODULE_{modname}
entry nor an entry in MODSHARED_NAMES.
"""
moddisabled = set(sysconfig.get_config_var("MODDISABLED_NAMES").split())
if self.cross_compiling:
modbuiltin = set(sysconfig.get_config_var("MODBUILT_NAMES").split())
else:
modbuiltin = set(sys.builtin_module_names)
for key, value in sysconfig.get_config_vars().items():
if not key.startswith("MODULE_") or not key.endswith("_STATE"):
continue
if value not in {"yes", "disabled", "missing", "n/a"}:
raise ValueError(f"Unsupported value '{value}' for {key}")
modname = key[7:-6].lower()
if modname in moddisabled:
# Setup "*disabled*" rule
state = ModuleState.DISABLED_SETUP
elif value in {"disabled", "missing", "n/a"}:
state = ModuleState(value)
elif modname in modbuiltin:
assert value == "yes"
state = ModuleState.BUILTIN
else:
assert value == "yes"
state = ModuleState.SHARED
modinfo = ModuleInfo(modname, state)
logger.debug("Found %s in Makefile", modinfo)
yield modinfo
def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
"""Parse a Modules/Setup file"""
assign_var = re.compile(r"^\w+=") # EGG_SPAM=foo
# default to static module
state = ModuleState.BUILTIN
logger.debug("Parsing Setup file %s", setup_file)
with open(setup_file, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#") or assign_var.match(line):
continue
match line.split():
case ["*shared*"]:
state = ModuleState.SHARED
case ["*static*"]:
state = ModuleState.BUILTIN
case ["*disabled*"]:
state = ModuleState.DISABLED
case ["*noconfig*"]:
state = None
case [*items]:
if state == ModuleState.DISABLED:
# *disabled* can disable multiple modules per line
for item in items:
modinfo = ModuleInfo(item, state)
logger.debug("Found %s in %s", modinfo, setup_file)
yield modinfo
elif state in {ModuleState.SHARED, ModuleState.BUILTIN}:
# *shared* and *static*, first item is the name of the module.
modinfo = ModuleInfo(items[0], state)
logger.debug("Found %s in %s", modinfo, setup_file)
yield modinfo
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
"""Get ModuleSpec for builtin or extension module"""
if modinfo.state == ModuleState.SHARED:
location = os.fspath(self.get_location(modinfo))
loader = ExtensionFileLoader(modinfo.name, location)
return spec_from_file_location(modinfo.name, location, loader=loader)
elif modinfo.state == ModuleState.BUILTIN:
return spec_from_loader(modinfo.name, loader=BuiltinImporter)
else:
raise ValueError(modinfo)
def get_location(self, modinfo: ModuleInfo) -> pathlib.Path:
"""Get shared library location in build directory"""
if modinfo.state == ModuleState.SHARED:
return self.builddir / f"{modinfo.name}{self.ext_suffix}"
else:
return None
def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec):
"""Check that the module file is present and not empty"""
if spec.loader is BuiltinImporter:
return
try:
st = os.stat(spec.origin)
except FileNotFoundError:
logger.error("%s (%s) is missing", modinfo.name, spec.origin)
raise
if not st.st_size:
raise ImportError(f"{spec.origin} is an empty file")
def check_module_import(self, modinfo: ModuleInfo):
"""Attempt to import module and report errors"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
try:
with warnings.catch_warnings():
# ignore deprecation warning from deprecated modules
warnings.simplefilter("ignore", DeprecationWarning)
bootstrap_load(spec)
except ImportError as e:
logger.error("%s failed to import: %s", modinfo.name, e)
raise
except Exception as e:
logger.exception("Importing extension '%s' failed!", modinfo.name)
raise
def check_module_cross(self, modinfo: ModuleInfo):
"""Sanity check for cross compiling"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
def rename_module(self, modinfo: ModuleInfo) -> None:
"""Rename module file"""
if modinfo.state == ModuleState.BUILTIN:
logger.error("Cannot mark builtin module '%s' as failed!", modinfo.name)
return
failed_name = f"{modinfo.name}_failed{self.ext_suffix}"
builddir_path = self.get_location(modinfo)
if builddir_path.is_symlink():
symlink = builddir_path
module_path = builddir_path.resolve().relative_to(os.getcwd())
failed_path = module_path.parent / failed_name
else:
symlink = None
module_path = builddir_path
failed_path = self.builddir / failed_name
# remove old failed file
failed_path.unlink(missing_ok=True)
# remove symlink
if symlink is not None:
symlink.unlink(missing_ok=True)
# rename shared extension file
try:
module_path.rename(failed_path)
except FileNotFoundError:
logger.debug("Shared extension file '%s' does not exist.", module_path)
else:
logger.debug("Rename '%s' -> '%s'", module_path, failed_path)
def main():
args = parser.parse_args()
if args.debug:
args.verbose = True
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO,
format="[%(levelname)s] %(message)s",
)
checker = ModuleChecker(
cross_compiling=args.cross_compiling,
strict=args.strict,
)
if args.list_module_names:
names = checker.list_module_names(all=True)
for name in sorted(names):
print(name)
else:
checker.check()
checker.summary(verbose=args.verbose)
try:
checker.check_strict_build()
except RuntimeError as e:
parser.exit(1, f"\nError: {e}\n")
if __name__ == "__main__":
main()

47
Tools/scripts/generate_stdlib_module_names.py
View file

@ -7,10 +7,11 @@
import sys
import sysconfig
from check_extension_modules import ModuleChecker
SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup')
SETUP_PY = os.path.join(SRC_DIR, 'setup.py')
IGNORE = {
@ -41,23 +42,6 @@
'xxsubtype',
}
# Windows extension modules
WINDOWS_MODULES = (
'_msi',
'_overlapped',
'_testconsole',
'_winapi',
'msvcrt',
'nt',
'winreg',
'winsound'
)
# macOS extension modules
MACOS_MODULES = (
'_scproxy',
)
# Pure Python modules (Lib/*.py)
def list_python_modules(names):
for filename in os.listdir(STDLIB_PATH):
@ -89,28 +73,11 @@ def list_setup_extensions(names):
names |= set(extensions)
# Built-in and extension modules built by Modules/Setup
# Built-in and extension modules built by Modules/Setup*
# includes Windows and macOS extensions.
def list_modules_setup_extensions(names):
assign_var = re.compile("^[A-Z]+=")
with open(MODULES_SETUP, encoding="utf-8") as modules_fp:
for line in modules_fp:
# Strip comment
line = line.partition("#")[0]
line = line.rstrip()
if not line:
continue
if assign_var.match(line):
# Ignore "VAR=VALUE"
continue
if line in ("*disabled*", "*shared*"):
continue
parts = line.split()
if len(parts) < 2:
continue
# "errno errnomodule.c" => write "errno"
name = parts[0]
names.add(name)
checker = ModuleChecker()
names.update(checker.list_module_names(all=True))
# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
@ -134,7 +101,7 @@ def list_frozen(names):
def list_modules():
names = set(sys.builtin_module_names) | set(WINDOWS_MODULES) | set(MACOS_MODULES)
names = set(sys.builtin_module_names)
list_modules_setup_extensions(names)
list_setup_extensions(names)
list_packages(names)