bpo-39448: Add regen-frozen makefile target. (GH-18174)

Add the "regen-frozen" makefile target that regenerates the code for the
frozen __hello__ module.

Lib/ctypes/test/test_values.py

@@ -80,9 +80,9 @@ class struct_frozen(Structure):
                 continue
             items.append((entry.name.decode("ascii"), entry.size))
 
-        expected = [("__hello__", 141),
-                    ("__phello__", -141),
-                    ("__phello__.spam", 141),
+        expected = [("__hello__", 125),
+                    ("__phello__", -125),
+                    ("__phello__.spam", 125),
                     ]
         self.assertEqual(items, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")

Makefile.pre.in

@@ -761,7 +761,7 @@ regen-limited-abi: all
 
 regen-all: regen-opcode regen-opcode-targets regen-typeslots \
 	regen-token regen-ast regen-keyword regen-importlib clinic \
-	regen-pegen-metaparser regen-pegen
+	regen-pegen-metaparser regen-pegen regen-frozen
 	@echo
 	@echo "Note: make regen-stdlib-module-names and autoconf should be run manually"
 
@@ -870,6 +870,11 @@ regen-opcode:
 		$(srcdir)/Include/opcode.h.new
 	$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
 
+.PHONY: regen-frozen
+regen-frozen:
+	# Regenerate code for frozen module "__hello__".
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/freeze/regen_frozen.py $(srcdir)/Python/frozen_hello.h
+
 .PHONY: regen-token
 regen-token:
 	# Regenerate Doc/library/token-list.inc from Grammar/Tokens
@@ -974,7 +979,7 @@ Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
 		$(srcdir)/Python/condvar.h
 
 Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h \
-		$(srcdir)/Python/importlib_zipimport.h
+		$(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/frozen_hello.h
 
 # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to
 # follow our naming conventions. dtrace(1) uses the output filename to generate

Misc/NEWS.d/next/Build/2020-01-24-12-54-22.bpo-39448.k4pv14.rst

@@ -0,0 +1,2 @@
+Add the "regen-frozen" makefile target that regenerates the code for the
+frozen ``__hello__`` module.

Python/frozen.c

@@ -1,5 +1,5 @@
 
-/* Dummy frozen modules initializer */
+/* Frozen modules initializer */
 
 #include "Python.h"
 #include "importlib.h"
@@ -10,21 +10,11 @@
    define a single frozen module, __hello__.  Loading it will print
    some famous words... */
 
-/* To regenerate this data after the bytecode or marshal format has changed,
-   go to ../Tools/freeze/ and freeze the flag.py file; then copy and paste
-   the appropriate bytes from M___main__.c. */
-
-static unsigned char M___hello__[] = {
-    227,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,2,0,0,0,64,0,0,0,115,16,0,0,0,100,0,
-    90,0,101,1,100,1,131,1,1,0,100,2,83,0,41,3,
-    84,122,12,72,101,108,108,111,32,119,111,114,108,100,33,78,
-    41,2,218,11,105,110,105,116,105,97,108,105,122,101,100,218,
-    5,112,114,105,110,116,169,0,114,3,0,0,0,114,3,0,
-    0,0,250,20,84,111,111,108,115,47,102,114,101,101,122,101,
-    47,102,108,97,103,46,112,121,218,8,60,109,111,100,117,108,
-    101,62,1,0,0,0,115,2,0,0,0,4,1,
-};
+/* Run "make regen-frozen" to regen the file below (e.g. after a bytecode
+ * format change).  The file is created by Tools/frozen/regen_frozen.py.  The
+ * include file defines M___hello__ as an array of bytes.
+ */
+#include "frozen_hello.h"
 
 #define SIZE (int)sizeof(M___hello__)
 

Python/frozen_hello.h

@@ -0,0 +1,13 @@
+/* Generated with Tools/freeze/regen_frozen.py */
+static unsigned char M___hello__[] = {
+    227,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,2,0,0,0,64,0,0,0,115,
+    16,0,0,0,100,0,90,0,101,1,100,1,131,
+    1,1,0,100,2,83,0,41,3,84,122,12,72,
+    101,108,108,111,32,119,111,114,108,100,33,78,41,
+    2,90,11,105,110,105,116,105,97,108,105,122,101,
+    100,218,5,112,114,105,110,116,169,0,114,2,0,
+    0,0,114,2,0,0,0,218,4,110,111,110,101,
+    218,8,60,109,111,100,117,108,101,62,1,0,0,
+    0,115,2,0,0,0,4,1,
+};

Tools/freeze/regen_frozen.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import sys
+import os
+import marshal
+
+
+DIR = os.path.dirname(sys.argv[0])
+# source code for module to freeze
+FILE = os.path.join(DIR, 'flag.py')
+# C symbol to use for array holding frozen bytes
+SYMBOL = 'M___hello__'
+
+
+def get_module_code(filename):
+    """Compile 'filename' and return the module code as a marshalled byte
+    string.
+    """
+    with open(filename, 'r') as fp:
+        src = fp.read()
+    co = compile(src, 'none', 'exec')
+    co_bytes = marshal.dumps(co)
+    return co_bytes
+
+
+def gen_c_code(fp, co_bytes):
+    """Generate C code for the module code in 'co_bytes', write it to 'fp'.
+    """
+    def write(*args, **kwargs):
+        print(*args, **kwargs, file=fp)
+    write('/* Generated with Tools/freeze/regen_frozen.py */')
+    write('static unsigned char %s[] = {' % SYMBOL, end='')
+    bytes_per_row = 13
+    for i, opcode in enumerate(co_bytes):
+        if (i % bytes_per_row) == 0:
+            # start a new row
+            write()
+            write('    ', end='')
+        write('%d,' % opcode, end='')
+    write()
+    write('};')
+
+
+def main():
+    out_filename = sys.argv[1]
+    co_bytes = get_module_code(FILE)
+    with open(out_filename, 'w') as fp:
+        gen_c_code(fp, co_bytes)
+
+
+if __name__ == '__main__':
+    main()