mirror of
https://github.com/python/cpython
synced 2024-10-14 15:57:14 +00:00
gh-109649: Use os.process_cpu_count() (#110165)
Replace os.cpu_count() with os.process_cpu_count() in modules: * compileall * concurrent.futures * multiprocessing Replace os.cpu_count() with os.process_cpu_count() in programs: * _decimal deccheck.py test * freeze.py * multissltests.py * python -m test (regrtest) * wasm_build.py Other changes: * test.pythoninfo logs os.process_cpu_count(). * regrtest gets os.process_cpu_count() / os.cpu_count() in headers.
This commit is contained in:
parent
53eb9a676f
commit
a46e960768
|
@ -90,7 +90,7 @@ compile Python sources.
|
||||||
.. cmdoption:: -j N
|
.. cmdoption:: -j N
|
||||||
|
|
||||||
Use *N* workers to compile the files within the given directory.
|
Use *N* workers to compile the files within the given directory.
|
||||||
If ``0`` is used, then the result of :func:`os.cpu_count()`
|
If ``0`` is used, then the result of :func:`os.process_cpu_count()`
|
||||||
will be used.
|
will be used.
|
||||||
|
|
||||||
.. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash]
|
.. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash]
|
||||||
|
|
|
@ -188,6 +188,10 @@ And::
|
||||||
ThreadPoolExecutor now reuses idle worker threads before starting
|
ThreadPoolExecutor now reuses idle worker threads before starting
|
||||||
*max_workers* worker threads too.
|
*max_workers* worker threads too.
|
||||||
|
|
||||||
|
.. versionchanged:: 3.13
|
||||||
|
Default value of *max_workers* is changed to
|
||||||
|
``min(32, (os.process_cpu_count() or 1) + 4)``.
|
||||||
|
|
||||||
|
|
||||||
.. _threadpoolexecutor-example:
|
.. _threadpoolexecutor-example:
|
||||||
|
|
||||||
|
@ -243,7 +247,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
|
||||||
|
|
||||||
An :class:`Executor` subclass that executes calls asynchronously using a pool
|
An :class:`Executor` subclass that executes calls asynchronously using a pool
|
||||||
of at most *max_workers* processes. If *max_workers* is ``None`` or not
|
of at most *max_workers* processes. If *max_workers* is ``None`` or not
|
||||||
given, it will default to the number of processors on the machine.
|
given, it will default to :func:`os.process_cpu_count`.
|
||||||
If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError`
|
If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError`
|
||||||
will be raised.
|
will be raised.
|
||||||
On Windows, *max_workers* must be less than or equal to ``61``. If it is not
|
On Windows, *max_workers* must be less than or equal to ``61``. If it is not
|
||||||
|
@ -301,6 +305,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
|
||||||
different start method. See the :func:`os.fork` documentation for
|
different start method. See the :func:`os.fork` documentation for
|
||||||
further explanation.
|
further explanation.
|
||||||
|
|
||||||
|
.. versionchanged:: 3.13
|
||||||
|
*max_workers* uses :func:`os.process_cpu_count` by default, instead of
|
||||||
|
:func:`os.cpu_count`.
|
||||||
|
|
||||||
.. _processpoolexecutor-example:
|
.. _processpoolexecutor-example:
|
||||||
|
|
||||||
ProcessPoolExecutor Example
|
ProcessPoolExecutor Example
|
||||||
|
|
|
@ -996,13 +996,13 @@ Miscellaneous
|
||||||
|
|
||||||
This number is not equivalent to the number of CPUs the current process can
|
This number is not equivalent to the number of CPUs the current process can
|
||||||
use. The number of usable CPUs can be obtained with
|
use. The number of usable CPUs can be obtained with
|
||||||
``len(os.sched_getaffinity(0))``
|
:func:`os.process_cpu_count`.
|
||||||
|
|
||||||
When the number of CPUs cannot be determined a :exc:`NotImplementedError`
|
When the number of CPUs cannot be determined a :exc:`NotImplementedError`
|
||||||
is raised.
|
is raised.
|
||||||
|
|
||||||
.. seealso::
|
.. seealso::
|
||||||
:func:`os.cpu_count`
|
:func:`os.cpu_count` and :func:`os.process_cpu_count`
|
||||||
|
|
||||||
.. function:: current_process()
|
.. function:: current_process()
|
||||||
|
|
||||||
|
@ -2214,7 +2214,7 @@ with the :class:`Pool` class.
|
||||||
callbacks and has a parallel map implementation.
|
callbacks and has a parallel map implementation.
|
||||||
|
|
||||||
*processes* is the number of worker processes to use. If *processes* is
|
*processes* is the number of worker processes to use. If *processes* is
|
||||||
``None`` then the number returned by :func:`os.cpu_count` is used.
|
``None`` then the number returned by :func:`os.process_cpu_count` is used.
|
||||||
|
|
||||||
If *initializer* is not ``None`` then each worker process will call
|
If *initializer* is not ``None`` then each worker process will call
|
||||||
``initializer(*initargs)`` when it starts.
|
``initializer(*initargs)`` when it starts.
|
||||||
|
@ -2249,6 +2249,10 @@ with the :class:`Pool` class.
|
||||||
.. versionadded:: 3.4
|
.. versionadded:: 3.4
|
||||||
*context*
|
*context*
|
||||||
|
|
||||||
|
.. versionchanged:: 3.13
|
||||||
|
*processes* uses :func:`os.process_cpu_count` by default, instead of
|
||||||
|
:func:`os.cpu_count`.
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
|
|
||||||
Worker processes within a :class:`Pool` typically live for the complete
|
Worker processes within a :class:`Pool` typically live for the complete
|
||||||
|
@ -2775,7 +2779,7 @@ worker threads rather than worker processes.
|
||||||
:meth:`~multiprocessing.pool.Pool.terminate` manually.
|
:meth:`~multiprocessing.pool.Pool.terminate` manually.
|
||||||
|
|
||||||
*processes* is the number of worker threads to use. If *processes* is
|
*processes* is the number of worker threads to use. If *processes* is
|
||||||
``None`` then the number returned by :func:`os.cpu_count` is used.
|
``None`` then the number returned by :func:`os.process_cpu_count` is used.
|
||||||
|
|
||||||
If *initializer* is not ``None`` then each worker process will call
|
If *initializer* is not ``None`` then each worker process will call
|
||||||
``initializer(*initargs)`` when it starts.
|
``initializer(*initargs)`` when it starts.
|
||||||
|
|
|
@ -91,6 +91,13 @@ Other Language Changes
|
||||||
of the ``optimize`` argument.
|
of the ``optimize`` argument.
|
||||||
(Contributed by Irit Katriel in :gh:`108113`).
|
(Contributed by Irit Katriel in :gh:`108113`).
|
||||||
|
|
||||||
|
* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
|
||||||
|
Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
|
||||||
|
default number of worker threads and processes. Get the CPU affinity
|
||||||
|
if supported.
|
||||||
|
(Contributed by Victor Stinner in :gh:`109649`.)
|
||||||
|
|
||||||
|
|
||||||
New Modules
|
New Modules
|
||||||
===========
|
===========
|
||||||
|
|
||||||
|
|
|
@ -666,7 +666,7 @@ def __init__(self, max_workers=None, mp_context=None,
|
||||||
_check_system_limits()
|
_check_system_limits()
|
||||||
|
|
||||||
if max_workers is None:
|
if max_workers is None:
|
||||||
self._max_workers = os.cpu_count() or 1
|
self._max_workers = os.process_cpu_count() or 1
|
||||||
if sys.platform == 'win32':
|
if sys.platform == 'win32':
|
||||||
self._max_workers = min(_MAX_WINDOWS_WORKERS,
|
self._max_workers = min(_MAX_WINDOWS_WORKERS,
|
||||||
self._max_workers)
|
self._max_workers)
|
||||||
|
|
|
@ -139,10 +139,10 @@ def __init__(self, max_workers=None, thread_name_prefix='',
|
||||||
# * CPU bound task which releases GIL
|
# * CPU bound task which releases GIL
|
||||||
# * I/O bound task (which releases GIL, of course)
|
# * I/O bound task (which releases GIL, of course)
|
||||||
#
|
#
|
||||||
# We use cpu_count + 4 for both types of tasks.
|
# We use process_cpu_count + 4 for both types of tasks.
|
||||||
# But we limit it to 32 to avoid consuming surprisingly large resource
|
# But we limit it to 32 to avoid consuming surprisingly large resource
|
||||||
# on many core machine.
|
# on many core machine.
|
||||||
max_workers = min(32, (os.cpu_count() or 1) + 4)
|
max_workers = min(32, (os.process_cpu_count() or 1) + 4)
|
||||||
if max_workers <= 0:
|
if max_workers <= 0:
|
||||||
raise ValueError("max_workers must be greater than 0")
|
raise ValueError("max_workers must be greater than 0")
|
||||||
|
|
||||||
|
|
|
@ -200,7 +200,7 @@ def __init__(self, processes=None, initializer=None, initargs=(),
|
||||||
self._initargs = initargs
|
self._initargs = initargs
|
||||||
|
|
||||||
if processes is None:
|
if processes is None:
|
||||||
processes = os.cpu_count() or 1
|
processes = os.process_cpu_count() or 1
|
||||||
if processes < 1:
|
if processes < 1:
|
||||||
raise ValueError("Number of processes must be at least 1")
|
raise ValueError("Number of processes must be at least 1")
|
||||||
if maxtasksperchild is not None:
|
if maxtasksperchild is not None:
|
||||||
|
|
|
@ -426,7 +426,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
|
||||||
if self.num_workers < 0:
|
if self.num_workers < 0:
|
||||||
# Use all CPUs + 2 extra worker processes for tests
|
# Use all CPUs + 2 extra worker processes for tests
|
||||||
# that like to sleep
|
# that like to sleep
|
||||||
self.num_workers = (os.cpu_count() or 1) + 2
|
self.num_workers = (os.process_cpu_count() or 1) + 2
|
||||||
|
|
||||||
# For a partial run, we do not need to clutter the output.
|
# For a partial run, we do not need to clutter the output.
|
||||||
if (self.want_header
|
if (self.want_header
|
||||||
|
|
|
@ -546,6 +546,9 @@ def display_header(use_resources: tuple[str, ...],
|
||||||
|
|
||||||
cpu_count = os.cpu_count()
|
cpu_count = os.cpu_count()
|
||||||
if cpu_count:
|
if cpu_count:
|
||||||
|
process_cpu_count = os.process_cpu_count()
|
||||||
|
if process_cpu_count and process_cpu_count != cpu_count:
|
||||||
|
cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)"
|
||||||
print("== CPU count:", cpu_count)
|
print("== CPU count:", cpu_count)
|
||||||
print("== encodings: locale=%s, FS=%s"
|
print("== encodings: locale=%s, FS=%s"
|
||||||
% (locale.getencoding(), sys.getfilesystemencoding()))
|
% (locale.getencoding(), sys.getfilesystemencoding()))
|
||||||
|
|
|
@ -239,6 +239,7 @@ def format_attr(attr, value):
|
||||||
'getresgid',
|
'getresgid',
|
||||||
'getresuid',
|
'getresuid',
|
||||||
'getuid',
|
'getuid',
|
||||||
|
'process_cpu_count',
|
||||||
'uname',
|
'uname',
|
||||||
):
|
):
|
||||||
call_func(info_add, 'os.%s' % func, os, func)
|
call_func(info_add, 'os.%s' % func, os, func)
|
||||||
|
|
|
@ -25,7 +25,7 @@ def record_finished(n):
|
||||||
|
|
||||||
def test_default_workers(self):
|
def test_default_workers(self):
|
||||||
executor = self.executor_type()
|
executor = self.executor_type()
|
||||||
expected = min(32, (os.cpu_count() or 1) + 4)
|
expected = min(32, (os.process_cpu_count() or 1) + 4)
|
||||||
self.assertEqual(executor._max_workers, expected)
|
self.assertEqual(executor._max_workers, expected)
|
||||||
|
|
||||||
def test_saturation(self):
|
def test_saturation(self):
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
:mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`:
|
||||||
|
Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the
|
||||||
|
default number of worker threads and processes. Get the CPU affinity if
|
||||||
|
supported. Patch by Victor Stinner.
|
|
@ -1301,7 +1301,7 @@ def tfunc():
|
||||||
out, _ = p.communicate()
|
out, _ = p.communicate()
|
||||||
write_output(out, p.returncode)
|
write_output(out, p.returncode)
|
||||||
|
|
||||||
N = os.cpu_count()
|
N = os.process_cpu_count()
|
||||||
t = N * [None]
|
t = N * [None]
|
||||||
|
|
||||||
for i in range(N):
|
for i in range(N):
|
||||||
|
|
|
@ -130,7 +130,7 @@ def prepare(script=None, outdir=None):
|
||||||
if not MAKE:
|
if not MAKE:
|
||||||
raise UnsupportedError('make')
|
raise UnsupportedError('make')
|
||||||
|
|
||||||
cores = os.cpu_count()
|
cores = os.process_cpu_count()
|
||||||
if cores and cores >= 3:
|
if cores and cores >= 3:
|
||||||
# this test is most often run as part of the whole suite with a lot
|
# this test is most often run as part of the whole suite with a lot
|
||||||
# of other tests running in parallel, from 1-2 vCPU systems up to
|
# of other tests running in parallel, from 1-2 vCPU systems up to
|
||||||
|
|
|
@ -151,7 +151,10 @@ class AbstractBuilder(object):
|
||||||
build_template = None
|
build_template = None
|
||||||
depend_target = None
|
depend_target = None
|
||||||
install_target = 'install'
|
install_target = 'install'
|
||||||
jobs = os.cpu_count()
|
if hasattr(os, 'process_cpu_count'):
|
||||||
|
jobs = os.process_cpu_count()
|
||||||
|
else:
|
||||||
|
jobs = os.cpu_count()
|
||||||
|
|
||||||
module_files = (
|
module_files = (
|
||||||
os.path.join(PYTHONROOT, "Modules/_ssl.c"),
|
os.path.join(PYTHONROOT, "Modules/_ssl.c"),
|
||||||
|
|
|
@ -516,7 +516,11 @@ def make_cmd(self) -> List[str]:
|
||||||
def getenv(self) -> Dict[str, Any]:
|
def getenv(self) -> Dict[str, Any]:
|
||||||
"""Generate environ dict for platform"""
|
"""Generate environ dict for platform"""
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
|
if hasattr(os, 'process_cpu_count'):
|
||||||
|
cpu_count = os.process_cpu_count()
|
||||||
|
else:
|
||||||
|
cpu_count = os.cpu_count()
|
||||||
|
env.setdefault("MAKEFLAGS", f"-j{cpu_count}")
|
||||||
platenv = self.host.platform.getenv(self)
|
platenv = self.host.platform.getenv(self)
|
||||||
for key, value in platenv.items():
|
for key, value in platenv.items():
|
||||||
if value is None:
|
if value is None:
|
||||||
|
|
Loading…
Reference in a new issue