From 1037ca5a8ea001bfa2a198e08655620234e9befd Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 10:09:48 -0800 Subject: [PATCH] bpo-45850: Implement deep-freeze on Windows (#29648) Implement changes to build with deep-frozen modules on Windows. Note that we now require Python 3.10 as the "bootstrap" or "host" Python. This causes a modest startup speed (around 7%) on Windows. --- .../2021-11-20-00-06-59.bpo-45850.q9lofz.rst | 2 + PCbuild/_freeze_module.vcxproj | 58 +++- PCbuild/find_python.bat | 6 +- PCbuild/pythoncore.vcxproj | 24 ++ Python/frozen.c | 5 - Tools/scripts/deepfreeze.py | 58 +++- Tools/scripts/freeze_modules.py | 21 +- Tools/scripts/startuptime.py | 22 ++ Tools/scripts/umarshal.py | 328 ++++++++++++++++++ 9 files changed, 500 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst create mode 100644 Tools/scripts/startuptime.py create mode 100644 Tools/scripts/umarshal.py diff --git a/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst new file mode 100644 index 00000000000..a84e1feb750 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst @@ -0,0 +1,2 @@ +Implement changes to build with deep-frozen modules on Windows. +Note that we now require Python 3.10 as the "bootstrap" or "host" Python. diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 6a91776b9d8..54fef9ca629 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -236,101 +236,141 @@ importlib._bootstrap $(IntDir)importlib._bootstrap.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h + $(IntDir)importlib._bootstrap.g.c + $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap.c importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h + $(IntDir)importlib._bootstrap_external.g.c + $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap_external.c zipimport $(IntDir)zipimport.g.h $(PySourcePath)Python\frozen_modules\zipimport.h + $(IntDir)zipimport.g.c + $(PySourcePath)Python\deepfreeze\df.zipimport.c abc $(IntDir)abc.g.h $(PySourcePath)Python\frozen_modules\abc.h + $(IntDir)abc.g.c + $(PySourcePath)Python\deepfreeze\df.abc.c codecs $(IntDir)codecs.g.h $(PySourcePath)Python\frozen_modules\codecs.h + $(IntDir)codecs.g.c + $(PySourcePath)Python\deepfreeze\df.codecs.c io $(IntDir)io.g.h $(PySourcePath)Python\frozen_modules\io.h + $(IntDir)io.g.c + $(PySourcePath)Python\deepfreeze\df.io.c _collections_abc $(IntDir)_collections_abc.g.h $(PySourcePath)Python\frozen_modules\_collections_abc.h + $(IntDir)_collections_abc.g.c + $(PySourcePath)Python\deepfreeze\df._collections_abc.c _sitebuiltins $(IntDir)_sitebuiltins.g.h $(PySourcePath)Python\frozen_modules\_sitebuiltins.h + $(IntDir)_sitebuiltins.g.c + $(PySourcePath)Python\deepfreeze\df._sitebuiltins.c genericpath $(IntDir)genericpath.g.h $(PySourcePath)Python\frozen_modules\genericpath.h + $(IntDir)genericpath.g.c + $(PySourcePath)Python\deepfreeze\df.genericpath.c ntpath $(IntDir)ntpath.g.h $(PySourcePath)Python\frozen_modules\ntpath.h + $(IntDir)ntpath.g.c + $(PySourcePath)Python\deepfreeze\df.ntpath.c posixpath $(IntDir)posixpath.g.h $(PySourcePath)Python\frozen_modules\posixpath.h + $(IntDir)posixpath.g.c + $(PySourcePath)Python\deepfreeze\df.posixpath.c os $(IntDir)os.g.h $(PySourcePath)Python\frozen_modules\os.h + $(IntDir)os.g.c + $(PySourcePath)Python\deepfreeze\df.os.c site $(IntDir)site.g.h $(PySourcePath)Python\frozen_modules\site.h + $(IntDir)site.g.c + $(PySourcePath)Python\deepfreeze\df.site.c stat $(IntDir)stat.g.h $(PySourcePath)Python\frozen_modules\stat.h + $(IntDir)stat.g.c + $(PySourcePath)Python\deepfreeze\df.stat.c __hello__ $(IntDir)__hello__.g.h $(PySourcePath)Python\frozen_modules\__hello__.h + $(IntDir)__hello__.g.c + $(PySourcePath)Python\deepfreeze\df.__hello__.c __phello__ $(IntDir)__phello__.g.h $(PySourcePath)Python\frozen_modules\__phello__.h + $(IntDir)__phello__.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.c __phello__.ham $(IntDir)__phello__.ham.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.h + $(IntDir)__phello__.ham.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.ham.c __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h + $(IntDir)__phello__.ham.eggs.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.ham.eggs.c __phello__.spam $(IntDir)__phello__.spam.g.h $(PySourcePath)Python\frozen_modules\__phello__.spam.h + $(IntDir)__phello__.spam.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.spam.c frozen_only $(IntDir)frozen_only.g.h $(PySourcePath)Python\frozen_modules\frozen_only.h + $(IntDir)frozen_only.g.c + $(PySourcePath)Python\deepfreeze\df.frozen_only.c @@ -338,17 +378,29 @@ - + - + Condition="!Exists(%(None.OutFile)) or (Exists(%(None.DeepIntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(` `, ` `))' != '$([System.IO.File]::ReadAllText(%(None.DeepIntFile)).Replace(` `, ` `))')"> + + + + + + + + + diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d0e4a86b01f..a9f14c5277f 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -31,13 +31,13 @@ @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) @rem If we have Python in externals, use that one -@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" @rem If HOST_PYTHON is recent enough, use that -@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 70f05563fa3..e1d59de7bc8 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -502,6 +502,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Python/frozen.c b/Python/frozen.c index 1565c9a3d73..9f43db70886 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -61,12 +61,7 @@ #include "frozen_modules/frozen_only.h" /* End includes */ -#ifdef MS_WINDOWS -/* Deepfreeze isn't supported on Windows yet. */ -#define GET_CODE(name) NULL -#else #define GET_CODE(name) _Py_get_##name##_toplevel -#endif /* Start extern declarations */ extern PyObject *_Py_get_importlib__bootstrap_toplevel(void); diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 074127f9492..b6d52b74549 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -1,13 +1,16 @@ import argparse +import ast import builtins import collections import contextlib import os -import sys +import re import time import types import typing +import umarshal + verbose = False @@ -55,7 +58,8 @@ def get_localsplus_counts(code: types.CodeType, nplaincellvars += 1 elif kind & CO_FAST_FREE: nfreevars += 1 - assert nlocals == len(code.co_varnames) == code.co_nlocals + assert nlocals == len(code.co_varnames) == code.co_nlocals, \ + (nlocals, len(code.co_varnames), code.co_nlocals) assert ncellvars == len(code.co_cellvars) assert nfreevars == len(code.co_freevars) assert len(names) == nlocals + nplaincellvars + nfreevars @@ -274,14 +278,7 @@ def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: self.write(item + ",") return f"& {name}._object.ob_base.ob_base" - def generate_int(self, name: str, i: int) -> str: - maxint = sys.maxsize - if maxint == 2**31 - 1: - digit = 2**15 - elif maxint == 2**63 - 1: - digit = 2**30 - else: - assert False, f"What int size is this system?!? {maxint=}" + def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: sign = -1 if i < 0 else 0 if i == 0 else +1 i = abs(i) digits: list[int] = [] @@ -298,6 +295,20 @@ def generate_int(self, name: str, i: int) -> str: if digits: ds = ", ".join(map(str, digits)) self.write(f".ob_digit = {{ {ds} }},") + + def generate_int(self, name: str, i: int) -> str: + if abs(i) < 2**15: + self._generate_int_for_bits(name, i, 2**15) + else: + connective = "if" + for bits_in_digit in 15, 30: + self.write(f"#{connective} PYLONG_BITS_IN_DIGIT == {bits_in_digit}") + self._generate_int_for_bits(name, i, 2**bits_in_digit) + connective = "elif" + self.write("#else") + self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"') + self.write("#endif") + # If neither clause applies, it won't compile return f"& {name}.ob_base.ob_base" def generate_float(self, name: str, x: float) -> str: @@ -326,7 +337,7 @@ def generate(self, name: str, obj: object) -> str: return self.cache[key] self.misses += 1 match obj: - case types.CodeType() as code: + case types.CodeType() | umarshal.Code() as code: val = self.generate_code(name, code) case tuple(t): val = self.generate_tuple(name, t) @@ -367,8 +378,31 @@ def generate(self, name: str, obj: object) -> str: } """ +FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */" + +FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" + + +def is_frozen_header(source: str) -> bool: + return source.startswith(FROZEN_COMMENT) + + +def decode_frozen_data(source: str) -> types.CodeType: + lines = source.splitlines() + while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None: + del lines[0] + while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: + del lines[-1] + values: tuple[int, ...] = ast.literal_eval("".join(lines)) + data = bytes(values) + return umarshal.loads(data) + + def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None: - code = compile(source, filename, "exec") + if is_frozen_header(source): + code = decode_frozen_data(source) + else: + code = compile(source, filename, "exec") printer = Printer(file) printer.generate("toplevel", code) printer.write("") diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index ccea4e11ab6..61ccae61e4f 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -11,7 +11,6 @@ import platform import subprocess import sys -import textwrap import time from update_file import updating_file_with_tmpfile, update_file_with_tmpfile @@ -55,6 +54,7 @@ def find_tool(): MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters') +PCBUILD_PYTHONCORE = os.path.join(ROOT_DIR, 'PCbuild', 'pythoncore.vcxproj') OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath' @@ -717,20 +717,28 @@ def regen_makefile(modules): def regen_pcbuild(modules): projlines = [] filterlines = [] + corelines = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) + deepbase = "df." + src.id + deepoutfile = f"Python\\deepfreeze\\{deepbase}.c" intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' + deepintfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.c' projlines.append(f' ') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') projlines.append(f' $(PySourcePath){header}') + projlines.append(f' $(IntDir){deepintfile}') + projlines.append(f' $(PySourcePath){deepoutfile}') projlines.append(f' ') filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') + corelines.append(f' ') + print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): lines = infile.readlines() @@ -753,6 +761,17 @@ def regen_pcbuild(modules): PCBUILD_FILTERS, ) outfile.writelines(lines) + print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') + with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + '', + '', + corelines, + PCBUILD_FILTERS, + ) + outfile.writelines(lines) ####################################### diff --git a/Tools/scripts/startuptime.py b/Tools/scripts/startuptime.py new file mode 100644 index 00000000000..1bb5b208f66 --- /dev/null +++ b/Tools/scripts/startuptime.py @@ -0,0 +1,22 @@ +# Quick script to time startup for various binaries + +import subprocess +import sys +import time + +NREPS = 100 + + +def main(): + binaries = sys.argv[1:] + for bin in binaries: + t0 = time.time() + for _ in range(NREPS): + result = subprocess.run([bin, "-c", "pass"]) + result.check_returncode() + t1 = time.time() + print(f"{(t1-t0)/NREPS:6.3f} {bin}") + + +if __name__ == "__main__": + main() diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py new file mode 100644 index 00000000000..e0d18c89e75 --- /dev/null +++ b/Tools/scripts/umarshal.py @@ -0,0 +1,328 @@ +# Implementat marshal.loads() in pure Python + +import ast + +from typing import Any + + +class Type: + # Adapted from marshal.c + NULL = ord('0') + NONE = ord('N') + FALSE = ord('F') + TRUE = ord('T') + STOPITER = ord('S') + ELLIPSIS = ord('.') + INT = ord('i') + INT64 = ord('I') + FLOAT = ord('f') + BINARY_FLOAT = ord('g') + COMPLEX = ord('x') + BINARY_COMPLEX = ord('y') + LONG = ord('l') + STRING = ord('s') + INTERNED = ord('t') + REF = ord('r') + TUPLE = ord('(') + LIST = ord('[') + DICT = ord('{') + CODE = ord('c') + UNICODE = ord('u') + UNKNOWN = ord('?') + SET = ord('<') + FROZENSET = ord('>') + ASCII = ord('a') + ASCII_INTERNED = ord('A') + SMALL_TUPLE = ord(')') + SHORT_ASCII = ord('z') + SHORT_ASCII_INTERNED = ord('Z') + + +FLAG_REF = 0x80 # with a type, add obj to index + +NULL = object() # marker + +# Cell kinds +CO_FAST_LOCAL = 0x20 +CO_FAST_CELL = 0x40 +CO_FAST_FREE = 0x80 + + +class Code: + def __init__(self, **kwds: Any): + self.__dict__.update(kwds) + + def __repr__(self) -> str: + return f"Code(**{self.__dict__})" + + co_localsplusnames: tuple[str] + co_localspluskinds: tuple[int] + + def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: + varnames: list[str] = [] + for name, kind in zip(self.co_localsplusnames, + self.co_localspluskinds): + if kind & select_kind: + varnames.append(name) + return tuple(varnames) + + @property + def co_varnames(self) -> tuple[str, ...]: + return self.get_localsplus_names(CO_FAST_LOCAL) + + @property + def co_cellvars(self) -> tuple[str, ...]: + return self.get_localsplus_names(CO_FAST_CELL) + + @property + def co_freevars(self) -> tuple[str, ...]: + return self.get_localsplus_names(CO_FAST_FREE) + + @property + def co_nlocals(self) -> int: + return len(self.co_varnames) + + +class Reader: + # A fairly literal translation of the marshal reader. + + def __init__(self, data: bytes): + self.data: bytes = data + self.end: int = len(self.data) + self.pos: int = 0 + self.refs: list[Any] = [] + self.level: int = 0 + + def r_string(self, n: int) -> bytes: + assert 0 <= n <= self.end - self.pos + buf = self.data[self.pos : self.pos + n] + self.pos += n + return buf + + def r_byte(self) -> int: + buf = self.r_string(1) + return buf[0] + + def r_short(self) -> int: + buf = self.r_string(2) + x = buf[0] + x |= buf[1] << 8 + x |= -(x & (1<<15)) # Sign-extend + return x + + def r_long(self) -> int: + buf = self.r_string(4) + x = buf[0] + x |= buf[1] << 8 + x |= buf[2] << 16 + x |= buf[3] << 24 + x |= -(x & (1<<31)) # Sign-extend + return x + + def r_long64(self) -> int: + buf = self.r_string(8) + x = buf[0] + x |= buf[1] << 8 + x |= buf[2] << 16 + x |= buf[3] << 24 + x |= buf[1] << 32 + x |= buf[1] << 40 + x |= buf[1] << 48 + x |= buf[1] << 56 + x |= -(x & (1<<63)) # Sign-extend + return x + + def r_PyLong(self) -> int: + n = self.r_long() + size = abs(n) + x = 0 + # Pray this is right + for i in range(size): + x |= self.r_short() << i*15 + if n < 0: + x = -x + return x + + def r_float_bin(self) -> float: + buf = self.r_string(8) + import struct # Lazy import to avoid breaking UNIX build + return struct.unpack("d", buf)[0] + + def r_float_str(self) -> float: + n = self.r_byte() + buf = self.r_string(n) + return ast.literal_eval(buf.decode("ascii")) + + def r_ref_reserve(self, flag: int) -> int: + if flag: + idx = len(self.refs) + self.refs.append(None) + return idx + else: + return 0 + + def r_ref_insert(self, obj: Any, idx: int, flag: int) -> Any: + if flag: + self.refs[idx] = obj + return obj + + def r_ref(self, obj: Any, flag: int) -> Any: + assert flag & FLAG_REF + self.refs.append(obj) + return obj + + def r_object(self) -> Any: + old_level = self.level + try: + return self._r_object() + finally: + self.level = old_level + + def _r_object(self) -> Any: + code = self.r_byte() + flag = code & FLAG_REF + type = code & ~FLAG_REF + # print(" "*self.level + f"{code} {flag} {type} {chr(type)!r}") + self.level += 1 + + def R_REF(obj: Any) -> Any: + if flag: + obj = self.r_ref(obj, flag) + return obj + + match type: + case Type.NULL: + return NULL + case Type.NONE: + return None + case Type.ELLIPSIS: + return Ellipsis + case Type.FALSE: + return False + case Type.TRUE: + return True + case Type.INT: + return R_REF(self.r_long()) + case Type.INT64: + return R_REF(self.r_long64()) + case Type.LONG: + return R_REF(self.r_PyLong()) + case Type.FLOAT: + return R_REF(self.r_float_str()) + case Type.BINARY_FLOAT: + return R_REF(self.r_float_bin()) + case Type.COMPLEX: + return R_REF(complex(self.r_float_str(), + self.r_float_str())) + case Type.BINARY_COMPLEX: + return R_REF(complex(self.r_float_bin(), + self.r_float_bin())) + case Type.STRING: + n = self.r_long() + return R_REF(self.r_string(n)) + case Type.ASCII_INTERNED | Type.ASCII: + n = self.r_long() + return R_REF(self.r_string(n).decode("ascii")) + case Type.SHORT_ASCII_INTERNED | Type.SHORT_ASCII: + n = self.r_byte() + return R_REF(self.r_string(n).decode("ascii")) + case Type.INTERNED | Type.UNICODE: + n = self.r_long() + return R_REF(self.r_string(n).decode("utf8", "surrogatepass")) + case Type.SMALL_TUPLE: + n = self.r_byte() + idx = self.r_ref_reserve(flag) + retval: Any = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + case Type.TUPLE: + n = self.r_long() + idx = self.r_ref_reserve(flag) + retval = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + case Type.LIST: + n = self.r_long() + retval = R_REF([]) + for _ in range(n): + retval.append(self.r_object()) + return retval + case Type.DICT: + retval = R_REF({}) + while True: + key = self.r_object() + if key == NULL: + break + val = self.r_object() + retval[key] = val + return retval + case Type.SET: + n = self.r_long() + retval = R_REF(set()) + for _ in range(n): + v = self.r_object() + retval.add(v) + return retval + case Type.FROZENSET: + n = self.r_long() + s: set[Any] = set() + idx = self.r_ref_reserve(flag) + for _ in range(n): + v = self.r_object() + s.add(v) + retval = frozenset(s) + self.r_ref_insert(retval, idx, flag) + return retval + case Type.CODE: + retval = R_REF(Code()) + retval.co_argcount = self.r_long() + retval.co_posonlyargcount = self.r_long() + retval.co_kwonlyargcount = self.r_long() + retval.co_stacksize = self.r_long() + retval.co_flags = self.r_long() + retval.co_code = self.r_object() + retval.co_consts = self.r_object() + retval.co_names = self.r_object() + retval.co_localsplusnames = self.r_object() + retval.co_localspluskinds = self.r_object() + retval.co_filename = self.r_object() + retval.co_name = self.r_object() + retval.co_qualname = self.r_object() + retval.co_firstlineno = self.r_long() + retval.co_linetable = self.r_object() + retval.co_endlinetable = self.r_object() + retval.co_columntable = self.r_object() + retval.co_exceptiontable = self.r_object() + return retval + case Type.REF: + n = self.r_long() + retval = self.refs[n] + assert retval is not None + return retval + case _: + breakpoint() + raise AssertionError(f"Unknown type {type} {chr(type)!r}") + + +def loads(data: bytes) -> Any: + assert isinstance(data, bytes) + r = Reader(data) + return r.r_object() + + +def main(): + # Test + import marshal, pprint + sample = {'foo': {(42, "bar", 3.14)}} + data = marshal.dumps(sample) + retval = loads(data) + assert retval == sample, retval + sample = main.__code__ + data = marshal.dumps(sample) + retval = loads(data) + assert isinstance(retval, Code), retval + pprint.pprint(retval.__dict__) + + +if __name__ == "__main__": + main()