GH-111485: Factor out tier 2 code generation from the rest of the interpreter code generator (GH-112968)

Include/internal/pycore_uop_ids.h

@@ -1,6 +1,6 @@
 // This file is generated by Tools/cases_generator/uop_id_generator.py
 // from:
-//   ['./Python/bytecodes.c']
+//   Python/bytecodes.c
 // Do not edit!
 #ifndef Py_CORE_UOP_IDS_H
 #define Py_CORE_UOP_IDS_H

Include/opcode_ids.h

@@ -1,6 +1,6 @@
 // This file is generated by Tools/cases_generator/opcode_id_generator.py
 // from:
-//   ['./Python/bytecodes.c']
+//   Python/bytecodes.c
 // Do not edit!
 
 #ifndef Py_OPCODE_IDS_H

Makefile.pre.in

@@ -1589,7 +1589,6 @@ regen-cases:
 		$(CASESFLAG) \
 		-t $(srcdir)/Python/opcode_targets.h.new \
 		-m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \
-		-e $(srcdir)/Python/executor_cases.c.h.new \
 		-p $(srcdir)/Lib/_opcode_metadata.py.new \
 		-a $(srcdir)/Python/abstract_interp_cases.c.h.new \
 		$(srcdir)/Python/bytecodes.c
@@ -1599,6 +1598,8 @@ regen-cases:
 	    $(srcdir)/Tools/cases_generator/uop_id_generator.py -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c
 	$(PYTHON_FOR_REGEN) \
 	    $(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
+	$(PYTHON_FOR_REGEN) \
+	    $(srcdir)/Tools/cases_generator/tier2_generator.py -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c
 	$(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new
 	$(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new
 	$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new

Python/bytecodes.c

@@ -3967,6 +3967,7 @@ dummy_func(
         }
 
         inst(EXTENDED_ARG, ( -- )) {
+            TIER_ONE_ONLY
             assert(oparg);
             opcode = next_instr->op.code;
             oparg = oparg << 8 | next_instr->op.arg;
@@ -3975,11 +3976,13 @@ dummy_func(
         }
 
         inst(CACHE, (--)) {
+            TIER_ONE_ONLY
             assert(0 && "Executing a cache.");
             Py_UNREACHABLE();
         }
 
         inst(RESERVED, (--)) {
+            TIER_ONE_ONLY
             assert(0 && "Executing RESERVED instruction.");
             Py_UNREACHABLE();
         }

Python/generated_cases.c.h

@@ -1,6 +1,6 @@
 // This file is generated by Tools/cases_generator/tier1_generator.py
 // from:
-//   ['./Python/bytecodes.c']
+//   Python/bytecodes.c
 // Do not edit!
 
 #ifdef TIER_TWO
@@ -725,6 +725,7 @@
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(CACHE);
+            TIER_ONE_ONLY
             assert(0 && "Executing a cache.");
             Py_UNREACHABLE();
         }
@@ -2364,6 +2365,7 @@
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(EXTENDED_ARG);
+            TIER_ONE_ONLY
             assert(oparg);
             opcode = next_instr->op.code;
             oparg = oparg << 8 | next_instr->op.arg;
@@ -4704,6 +4706,7 @@
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(RESERVED);
+            TIER_ONE_ONLY
             assert(0 && "Executing RESERVED instruction.");
             Py_UNREACHABLE();
         }

Tools/cases_generator/analyzer.py

@@ -15,6 +15,7 @@ class Properties:
     needs_this: bool
     always_exits: bool
     stores_sp: bool
+    tier_one_only: bool
 
     def dump(self, indent: str) -> None:
         print(indent, end="")
@@ -33,6 +34,7 @@ def from_list(properties: list["Properties"]) -> "Properties":
             needs_this=any(p.needs_this for p in properties),
             always_exits=any(p.always_exits for p in properties),
             stores_sp=any(p.stores_sp for p in properties),
+            tier_one_only=any(p.tier_one_only for p in properties),
         )
 
 
@@ -46,6 +48,7 @@ def from_list(properties: list["Properties"]) -> "Properties":
     needs_this=False,
     always_exits=False,
     stores_sp=False,
+    tier_one_only=False,
 )
 
 
@@ -124,6 +127,21 @@ def size(self) -> int:
             self._size = sum(c.size for c in self.caches)
         return self._size
 
+    def is_viable(self) -> bool:
+        if self.name == "_SAVE_RETURN_OFFSET":
+            return True  # Adjusts next_instr, but only in tier 1 code
+        if self.properties.needs_this:
+            return False
+        if "INSTRUMENTED" in self.name:
+            return False
+        if "replaced" in self.annotations:
+            return False
+        if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"):
+            return False
+        if len([c for c in self.caches if c.name != "unused"]) > 1:
+            return False
+        return True
+
 
 Part = Uop | Skip
 
@@ -292,6 +310,7 @@ def compute_properties(op: parser.InstDef) -> Properties:
         needs_this=variable_used(op, "this_instr"),
         always_exits=always_exits(op),
         stores_sp=variable_used(op, "STORE_SP"),
+        tier_one_only=variable_used(op, "TIER_ONE_ONLY"),
     )
 
 

Tools/cases_generator/generate_cases.py

@@ -128,13 +128,6 @@
 arg_parser.add_argument(
     "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
 )
-arg_parser.add_argument(
-    "-e",
-    "--executor-cases",
-    type=str,
-    help="Write executor cases to this file",
-    default=DEFAULT_EXECUTOR_OUTPUT,
-)
 arg_parser.add_argument(
     "-a",
     "--abstract-interpreter-cases",
@@ -846,7 +839,6 @@ def main() -> None:
     a.assign_opcode_ids()
     a.write_opcode_targets(args.opcode_targets_h)
     a.write_metadata(args.metadata, args.pymetadata)
-    a.write_executor_instructions(args.executor_cases, args.emit_line_directives)
     a.write_abstract_interpreter_instructions(
         args.abstract_interpreter_cases, args.emit_line_directives
     )

Tools/cases_generator/generators_common.py

@@ -1,19 +1,186 @@
 from pathlib import Path
 from typing import TextIO
 
+from analyzer import (
+    Analysis,
+    Instruction,
+    Uop,
+    Part,
+    analyze_files,
+    Skip,
+    StackItem,
+    analysis_error,
+)
+from cwriter import CWriter
+from typing import Callable, Mapping, TextIO, Iterator
+from lexer import Token
+from stack import StackOffset, Stack
+
+
 ROOT = Path(__file__).parent.parent.parent
-DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute()
+DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute().as_posix()
 
 
 def root_relative_path(filename: str) -> str:
-    return Path(filename).relative_to(ROOT).as_posix()
+    return Path(filename).absolute().relative_to(ROOT).as_posix()
 
 
-def write_header(generator: str, source: str, outfile: TextIO) -> None:
+def write_header(generator: str, sources: list[str], outfile: TextIO) -> None:
     outfile.write(
         f"""// This file is generated by {root_relative_path(generator)}
 // from:
-//   {source}
+//   {", ".join(root_relative_path(src) for src in sources)}
 // Do not edit!
 """
     )
+
+
+def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None:
+    parens = 0
+    for tkn in tkn_iter:
+        if tkn.kind == end and parens == 0:
+            return
+        if tkn.kind == "LPAREN":
+            parens += 1
+        if tkn.kind == "RPAREN":
+            parens -= 1
+        out.emit(tkn)
+
+
+def replace_deopt(
+    out: CWriter,
+    tkn: Token,
+    tkn_iter: Iterator[Token],
+    uop: Uop,
+    unused: Stack,
+    inst: Instruction | None,
+) -> None:
+    out.emit_at("DEOPT_IF", tkn)
+    out.emit(next(tkn_iter))
+    emit_to(out, tkn_iter, "RPAREN")
+    next(tkn_iter)  # Semi colon
+    out.emit(", ")
+    assert inst is not None
+    assert inst.family is not None
+    out.emit(inst.family.name)
+    out.emit(");\n")
+
+
+def replace_error(
+    out: CWriter,
+    tkn: Token,
+    tkn_iter: Iterator[Token],
+    uop: Uop,
+    stack: Stack,
+    inst: Instruction | None,
+) -> None:
+    out.emit_at("if ", tkn)
+    out.emit(next(tkn_iter))
+    emit_to(out, tkn_iter, "COMMA")
+    label = next(tkn_iter).text
+    next(tkn_iter)  # RPAREN
+    next(tkn_iter)  # Semi colon
+    out.emit(") ")
+    c_offset = stack.peek_offset.to_c()
+    try:
+        offset = -int(c_offset)
+        close = ";\n"
+    except ValueError:
+        offset = None
+        out.emit(f"{{ stack_pointer += {c_offset}; ")
+        close = "; }\n"
+    out.emit("goto ")
+    if offset:
+        out.emit(f"pop_{offset}_")
+    out.emit(label)
+    out.emit(close)
+
+
+def replace_decrefs(
+    out: CWriter,
+    tkn: Token,
+    tkn_iter: Iterator[Token],
+    uop: Uop,
+    stack: Stack,
+    inst: Instruction | None,
+) -> None:
+    next(tkn_iter)
+    next(tkn_iter)
+    next(tkn_iter)
+    out.emit_at("", tkn)
+    for var in uop.stack.inputs:
+        if var.name == "unused" or var.name == "null" or var.peek:
+            continue
+        if var.size != "1":
+            out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n")
+            out.emit(f"Py_DECREF({var.name}[_i]);\n")
+            out.emit("}\n")
+        elif var.condition:
+            out.emit(f"Py_XDECREF({var.name});\n")
+        else:
+            out.emit(f"Py_DECREF({var.name});\n")
+
+
+def replace_store_sp(
+    out: CWriter,
+    tkn: Token,
+    tkn_iter: Iterator[Token],
+    uop: Uop,
+    stack: Stack,
+    inst: Instruction | None,
+) -> None:
+    next(tkn_iter)
+    next(tkn_iter)
+    next(tkn_iter)
+    out.emit_at("", tkn)
+    stack.flush(out)
+    out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n")
+
+
+def replace_check_eval_breaker(
+    out: CWriter,
+    tkn: Token,
+    tkn_iter: Iterator[Token],
+    uop: Uop,
+    stack: Stack,
+    inst: Instruction | None,
+) -> None:
+    next(tkn_iter)
+    next(tkn_iter)
+    next(tkn_iter)
+    if not uop.properties.ends_with_eval_breaker:
+        out.emit_at("CHECK_EVAL_BREAKER();", tkn)
+
+
+REPLACEMENT_FUNCTIONS = {
+    "DEOPT_IF": replace_deopt,
+    "ERROR_IF": replace_error,
+    "DECREF_INPUTS": replace_decrefs,
+    "CHECK_EVAL_BREAKER": replace_check_eval_breaker,
+    "STORE_SP": replace_store_sp,
+}
+
+ReplacementFunctionType = Callable[
+    [CWriter, Token, Iterator[Token], Uop, Stack, Instruction | None], None
+]
+
+
+def emit_tokens(
+    out: CWriter,
+    uop: Uop,
+    stack: Stack,
+    inst: Instruction | None,
+    replacement_functions: Mapping[
+        str, ReplacementFunctionType
+    ] = REPLACEMENT_FUNCTIONS,
+) -> None:
+    tkns = uop.body[1:-1]
+    if not tkns:
+        return
+    tkn_iter = iter(tkns)
+    out.start_line()
+    for tkn in tkn_iter:
+        if tkn.kind == "IDENTIFIER" and tkn.text in replacement_functions:
+            replacement_functions[tkn.text](out, tkn, tkn_iter, uop, stack, inst)
+        else:
+            out.emit(tkn)

Tools/cases_generator/opcode_id_generator.py

@@ -24,7 +24,7 @@
 DEFAULT_OUTPUT = ROOT / "Include/opcode_ids.h"
 
 
-def generate_opcode_header(filenames: str, analysis: Analysis, outfile: TextIO) -> None:
+def generate_opcode_header(filenames: list[str], analysis: Analysis, outfile: TextIO) -> None:
     write_header(__file__, filenames, outfile)
     out = CWriter(outfile, 0, False)
     out.emit("\n")

Tools/cases_generator/stack.py

@@ -2,6 +2,7 @@
 from analyzer import StackItem
 from dataclasses import dataclass
 from formatting import maybe_parenthesize
+from cwriter import CWriter
 
 
 def var_size(var: StackItem) -> str:
@@ -79,3 +80,89 @@ def to_c(self) -> str:
     def clear(self) -> None:
         self.popped = []
         self.pushed = []
+
+
+class SizeMismatch(Exception):
+    pass
+
+
+class Stack:
+    def __init__(self) -> None:
+        self.top_offset = StackOffset()
+        self.base_offset = StackOffset()
+        self.peek_offset = StackOffset()
+        self.variables: list[StackItem] = []
+        self.defined: set[str] = set()
+
+    def pop(self, var: StackItem) -> str:
+        self.top_offset.pop(var)
+        if not var.peek:
+            self.peek_offset.pop(var)
+        indirect = "&" if var.is_array() else ""
+        if self.variables:
+            popped = self.variables.pop()
+            if popped.size != var.size:
+                raise SizeMismatch(
+                    f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. "
+                    f"Expected {var.size} got {popped.size}"
+                )
+            if popped.name == var.name:
+                return ""
+            elif popped.name == "unused":
+                self.defined.add(var.name)
+                return (
+                    f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n"
+                )
+            elif var.name == "unused":
+                return ""
+            else:
+                self.defined.add(var.name)
+                return f"{var.name} = {popped.name};\n"
+        self.base_offset.pop(var)
+        if var.name == "unused":
+            return ""
+        else:
+            self.defined.add(var.name)
+        cast = f"({var.type})" if (not indirect and var.type) else ""
+        assign = (
+            f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];"
+        )
+        if var.condition:
+            return f"if ({var.condition}) {{ {assign} }}\n"
+        return f"{assign}\n"
+
+    def push(self, var: StackItem) -> str:
+        self.variables.append(var)
+        if var.is_array() and var.name not in self.defined and var.name != "unused":
+            c_offset = self.top_offset.to_c()
+            self.top_offset.push(var)
+            self.defined.add(var.name)
+            return f"{var.name} = &stack_pointer[{c_offset}];\n"
+        else:
+            self.top_offset.push(var)
+            return ""
+
+    def flush(self, out: CWriter) -> None:
+        for var in self.variables:
+            if not var.peek:
+                cast = "(PyObject *)" if var.type else ""
+                if var.name != "unused" and not var.is_array():
+                    if var.condition:
+                        out.emit(f" if ({var.condition}) ")
+                    out.emit(
+                        f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n"
+                    )
+            self.base_offset.push(var)
+        if self.base_offset.to_c() != self.top_offset.to_c():
+            print("base", self.base_offset.to_c(), "top", self.top_offset.to_c())
+            assert False
+        number = self.base_offset.to_c()
+        if number != "0":
+            out.emit(f"stack_pointer += {number};\n")
+        self.variables = []
+        self.base_offset.clear()
+        self.top_offset.clear()
+        self.peek_offset.clear()
+
+    def as_comment(self) -> str:
+        return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */"

Tools/cases_generator/tier1_generator.py

@@ -21,11 +21,12 @@
     DEFAULT_INPUT,
     ROOT,
     write_header,
+    emit_tokens,
 )
 from cwriter import CWriter
 from typing import TextIO, Iterator
 from lexer import Token
-from stack import StackOffset
+from stack import StackOffset, Stack, SizeMismatch
 
 
 DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h"
@@ -34,88 +35,6 @@
 FOOTER = "#undef TIER_ONE\n"
 
 
-class SizeMismatch(Exception):
-    pass
-
-
-class Stack:
-    def __init__(self) -> None:
-        self.top_offset = StackOffset()
-        self.base_offset = StackOffset()
-        self.peek_offset = StackOffset()
-        self.variables: list[StackItem] = []
-        self.defined: set[str] = set()
-
-    def pop(self, var: StackItem) -> str:
-        self.top_offset.pop(var)
-        if not var.peek:
-            self.peek_offset.pop(var)
-        indirect = "&" if var.is_array() else ""
-        if self.variables:
-            popped = self.variables.pop()
-            if popped.size != var.size:
-                raise SizeMismatch(
-                    f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. "
-                    f"Expected {var.size} got {popped.size}"
-                )
-            if popped.name == var.name:
-                return ""
-            elif popped.name == "unused":
-                self.defined.add(var.name)
-                return (
-                    f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n"
-                )
-            elif var.name == "unused":
-                return ""
-            else:
-                self.defined.add(var.name)
-                return f"{var.name} = {popped.name};\n"
-        self.base_offset.pop(var)
-        if var.name == "unused":
-            return ""
-        else:
-            self.defined.add(var.name)
-        assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];"
-        if var.condition:
-            return f"if ({var.condition}) {{ {assign} }}\n"
-        return f"{assign}\n"
-
-    def push(self, var: StackItem) -> str:
-        self.variables.append(var)
-        if var.is_array() and var.name not in self.defined and var.name != "unused":
-            c_offset = self.top_offset.to_c()
-            self.top_offset.push(var)
-            self.defined.add(var.name)
-            return f"{var.name} = &stack_pointer[{c_offset}];\n"
-        else:
-            self.top_offset.push(var)
-            return ""
-
-    def flush(self, out: CWriter) -> None:
-        for var in self.variables:
-            if not var.peek:
-                if var.name != "unused" and not var.is_array():
-                    if var.condition:
-                        out.emit(f" if ({var.condition}) ")
-                    out.emit(
-                        f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n"
-                    )
-            self.base_offset.push(var)
-        if self.base_offset.to_c() != self.top_offset.to_c():
-            print("base", self.base_offset.to_c(), "top", self.top_offset.to_c())
-            assert False
-        number = self.base_offset.to_c()
-        if number != "0":
-            out.emit(f"stack_pointer += {number};\n")
-        self.variables = []
-        self.base_offset.clear()
-        self.top_offset.clear()
-        self.peek_offset.clear()
-
-    def as_comment(self) -> str:
-        return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */"
-
-
 def declare_variables(inst: Instruction, out: CWriter) -> None:
     variables = {"unused"}
     for uop in inst.parts:
@@ -138,145 +57,6 @@ def declare_variables(inst: Instruction, out: CWriter) -> None:
                         out.emit(f"{type}{var.name};\n")
 
 
-def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None:
-    parens = 0
-    for tkn in tkn_iter:
-        if tkn.kind == end and parens == 0:
-            return
-        if tkn.kind == "LPAREN":
-            parens += 1
-        if tkn.kind == "RPAREN":
-            parens -= 1
-        out.emit(tkn)
-
-
-def replace_deopt(
-    out: CWriter,
-    tkn: Token,
-    tkn_iter: Iterator[Token],
-    uop: Uop,
-    unused: Stack,
-    inst: Instruction,
-) -> None:
-    out.emit_at("DEOPT_IF", tkn)
-    out.emit(next(tkn_iter))
-    emit_to(out, tkn_iter, "RPAREN")
-    next(tkn_iter)  # Semi colon
-    out.emit(", ")
-    assert inst.family is not None
-    out.emit(inst.family.name)
-    out.emit(");\n")
-
-
-def replace_error(
-    out: CWriter,
-    tkn: Token,
-    tkn_iter: Iterator[Token],
-    uop: Uop,
-    stack: Stack,
-    inst: Instruction,
-) -> None:
-    out.emit_at("if ", tkn)
-    out.emit(next(tkn_iter))
-    emit_to(out, tkn_iter, "COMMA")
-    label = next(tkn_iter).text
-    next(tkn_iter)  # RPAREN
-    next(tkn_iter)  # Semi colon
-    out.emit(") ")
-    c_offset = stack.peek_offset.to_c()
-    try:
-        offset = -int(c_offset)
-        close = ";\n"
-    except ValueError:
-        offset = None
-        out.emit(f"{{ stack_pointer += {c_offset}; ")
-        close = "; }\n"
-    out.emit("goto ")
-    if offset:
-        out.emit(f"pop_{offset}_")
-    out.emit(label)
-    out.emit(close)
-
-
-def replace_decrefs(
-    out: CWriter,
-    tkn: Token,
-    tkn_iter: Iterator[Token],
-    uop: Uop,
-    stack: Stack,
-    inst: Instruction,
-) -> None:
-    next(tkn_iter)
-    next(tkn_iter)
-    next(tkn_iter)
-    out.emit_at("", tkn)
-    for var in uop.stack.inputs:
-        if var.name == "unused" or var.name == "null" or var.peek:
-            continue
-        if var.size != "1":
-            out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n")
-            out.emit(f"Py_DECREF({var.name}[_i]);\n")
-            out.emit("}\n")
-        elif var.condition:
-            out.emit(f"Py_XDECREF({var.name});\n")
-        else:
-            out.emit(f"Py_DECREF({var.name});\n")
-
-
-def replace_store_sp(
-    out: CWriter,
-    tkn: Token,
-    tkn_iter: Iterator[Token],
-    uop: Uop,
-    stack: Stack,
-    inst: Instruction,
-) -> None:
-    next(tkn_iter)
-    next(tkn_iter)
-    next(tkn_iter)
-    out.emit_at("", tkn)
-    stack.flush(out)
-    out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n")
-
-
-def replace_check_eval_breaker(
-    out: CWriter,
-    tkn: Token,
-    tkn_iter: Iterator[Token],
-    uop: Uop,
-    stack: Stack,
-    inst: Instruction,
-) -> None:
-    next(tkn_iter)
-    next(tkn_iter)
-    next(tkn_iter)
-    if not uop.properties.ends_with_eval_breaker:
-        out.emit_at("CHECK_EVAL_BREAKER();", tkn)
-
-
-REPLACEMENT_FUNCTIONS = {
-    "DEOPT_IF": replace_deopt,
-    "ERROR_IF": replace_error,
-    "DECREF_INPUTS": replace_decrefs,
-    "CHECK_EVAL_BREAKER": replace_check_eval_breaker,
-    "STORE_SP": replace_store_sp,
-}
-
-
-# Move this to formatter
-def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None:
-    tkns = uop.body[1:-1]
-    if not tkns:
-        return
-    tkn_iter = iter(tkns)
-    out.start_line()
-    for tkn in tkn_iter:
-        if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS:
-            REPLACEMENT_FUNCTIONS[tkn.text](out, tkn, tkn_iter, uop, stack, inst)
-        else:
-            out.emit(tkn)
-
-
 def write_uop(
     uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool
 ) -> int:
@@ -334,7 +114,7 @@ def uses_this(inst: Instruction) -> bool:
 
 
 def generate_tier1(
-    filenames: str, analysis: Analysis, outfile: TextIO, lines: bool
+    filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool
 ) -> None:
     write_header(__file__, filenames, outfile)
     outfile.write(
@@ -404,7 +184,7 @@ def generate_tier1(
 if __name__ == "__main__":
     args = arg_parser.parse_args()
     if len(args.input) == 0:
-        args.input.append(DEFAULT_INPUT.as_posix())
+        args.input.append(DEFAULT_INPUT)
     data = analyze_files(args.input)
     with open(args.output, "w") as outfile:
         generate_tier1(args.input, data, outfile, args.emit_line_directives)

Tools/cases_generator/tier2_generator.py

@@ -0,0 +1,202 @@
+"""Generate the cases for the tier 2 interpreter.
+Reads the instruction definitions from bytecodes.c.
+Writes the cases to executor_cases.c.h, which is #included in ceval.c.
+"""
+
+import argparse
+import os.path
+import sys
+
+from analyzer import (
+    Analysis,
+    Instruction,
+    Uop,
+    Part,
+    analyze_files,
+    Skip,
+    StackItem,
+    analysis_error,
+)
+from generators_common import (
+    DEFAULT_INPUT,
+    ROOT,
+    write_header,
+    emit_tokens,
+    emit_to,
+    REPLACEMENT_FUNCTIONS,
+)
+from cwriter import CWriter
+from typing import TextIO, Iterator
+from lexer import Token
+from stack import StackOffset, Stack, SizeMismatch
+
+DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h"
+
+
+def declare_variables(uop: Uop, out: CWriter) -> None:
+    variables = {"unused"}
+    for var in reversed(uop.stack.inputs):
+        if var.name not in variables:
+            type = var.type if var.type else "PyObject *"
+            variables.add(var.name)
+            if var.condition:
+                out.emit(f"{type}{var.name} = NULL;\n")
+            else:
+                out.emit(f"{type}{var.name};\n")
+    for var in uop.stack.outputs:
+        if var.name not in variables:
+            variables.add(var.name)
+            type = var.type if var.type else "PyObject *"
+            if var.condition:
+                out.emit(f"{type}{var.name} = NULL;\n")
+            else:
+                out.emit(f"{type}{var.name};\n")
+
+
+def tier2_replace_error(
+    out: CWriter,
+    tkn: Token,
+    tkn_iter: Iterator[Token],
+    uop: Uop,
+    stack: Stack,
+    inst: Instruction | None,
+) -> None:
+    out.emit_at("if ", tkn)
+    out.emit(next(tkn_iter))
+    emit_to(out, tkn_iter, "COMMA")
+    label = next(tkn_iter).text
+    next(tkn_iter)  # RPAREN
+    next(tkn_iter)  # Semi colon
+    out.emit(") ")
+    c_offset = stack.peek_offset.to_c()
+    try:
+        offset = -int(c_offset)
+        close = ";\n"
+    except ValueError:
+        offset = None
+        out.emit(f"{{ stack_pointer += {c_offset}; ")
+        close = "; }\n"
+    out.emit("goto ")
+    if offset:
+        out.emit(f"pop_{offset}_")
+    out.emit(label + "_tier_two")
+    out.emit(close)
+
+
+def tier2_replace_deopt(
+    out: CWriter,
+    tkn: Token,
+    tkn_iter: Iterator[Token],
+    uop: Uop,
+    unused: Stack,
+    inst: Instruction | None,
+) -> None:
+    out.emit_at("if ", tkn)
+    out.emit(next(tkn_iter))
+    emit_to(out, tkn_iter, "RPAREN")
+    next(tkn_iter)  # Semi colon
+    out.emit(") goto deoptimize;\n")
+
+
+TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy()
+TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error
+TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt
+
+
+def is_super(uop: Uop) -> bool:
+    for tkn in uop.body:
+        if tkn.kind == "IDENTIFIER" and tkn.text == "oparg1":
+            return True
+    return False
+
+
+def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None:
+    try:
+        out.start_line()
+        if uop.properties.oparg:
+            out.emit("oparg = CURRENT_OPARG();\n")
+        for var in reversed(uop.stack.inputs):
+            out.emit(stack.pop(var))
+        if not uop.properties.stores_sp:
+            for i, var in enumerate(uop.stack.outputs):
+                out.emit(stack.push(var))
+        for cache in uop.caches:
+            if cache.name != "unused":
+                if cache.size == 4:
+                    type = "PyObject *"
+                else:
+                    type = f"uint{cache.size*16}_t"
+                out.emit(f"{type} {cache.name} = ({type})CURRENT_OPERAND();\n")
+        emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS)
+        if uop.properties.stores_sp:
+            for i, var in enumerate(uop.stack.outputs):
+                out.emit(stack.push(var))
+    except SizeMismatch as ex:
+        raise analysis_error(ex.args[0], uop.body[0])
+
+
+SKIPS = ("_EXTENDED_ARG",)
+
+
+def generate_tier2(
+    filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool
+) -> None:
+    write_header(__file__, filenames, outfile)
+    outfile.write(
+        """
+#ifdef TIER_ONE
+    #error "This file is for Tier 2 only"
+#endif
+#define TIER_TWO 2
+"""
+    )
+    out = CWriter(outfile, 2, lines)
+    out.emit("\n")
+    for name, uop in analysis.uops.items():
+        if uop.properties.tier_one_only:
+            continue
+        if is_super(uop):
+            continue
+        if not uop.is_viable():
+            out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n")
+            continue
+        out.emit(f"case {uop.name}: {{\n")
+        declare_variables(uop, out)
+        stack = Stack()
+        write_uop(uop, out, stack)
+        out.start_line()
+        if not uop.properties.always_exits:
+            stack.flush(out)
+            if uop.properties.ends_with_eval_breaker:
+                out.emit("CHECK_EVAL_BREAKER();\n")
+            out.emit("break;\n")
+        out.start_line()
+        out.emit("}")
+        out.emit("\n\n")
+    outfile.write("#undef TIER_TWO\n")
+
+
+arg_parser = argparse.ArgumentParser(
+    description="Generate the code for the tier 2 interpreter.",
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+)
+
+arg_parser.add_argument(
+    "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
+)
+
+arg_parser.add_argument(
+    "-l", "--emit-line-directives", help="Emit #line directives", action="store_true"
+)
+
+arg_parser.add_argument(
+    "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
+)
+
+if __name__ == "__main__":
+    args = arg_parser.parse_args()
+    if len(args.input) == 0:
+        args.input.append(DEFAULT_INPUT)
+    data = analyze_files(args.input)
+    with open(args.output, "w") as outfile:
+        generate_tier2(args.input, data, outfile, args.emit_line_directives)

Tools/cases_generator/uop_id_generator.py

@@ -24,8 +24,11 @@
 DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_ids.h"
 
 
+OMIT = {"_CACHE", "_RESERVED", "_EXTENDED_ARG"}
+
+
 def generate_uop_ids(
-    filenames: str, analysis: Analysis, outfile: TextIO, distinct_namespace: bool
+    filenames: list[str], analysis: Analysis, outfile: TextIO, distinct_namespace: bool
 ) -> None:
     write_header(__file__, filenames, outfile)
     out = CWriter(outfile, 0, False)
@@ -45,11 +48,15 @@ def generate_uop_ids(
     next_id += 1
     out.emit(f"#define _SET_IP {next_id}\n")
     next_id += 1
-    PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP", "_CACHE", "_RESERVED", "_EXTENDED_ARG"}
+    PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"}
 
     for uop in analysis.uops.values():
         if uop.name in PRE_DEFINED:
             continue
+        # TODO: We should omit all tier-1 only uops, but
+        # generate_cases.py still generates code for those.
+        if uop.name in OMIT:
+            continue
         if uop.implicitly_created and not distinct_namespace:
             out.emit(f"#define {uop.name} {uop.name[1:]}\n")
         else:
@@ -85,7 +92,7 @@ def generate_uop_ids(
 if __name__ == "__main__":
     args = arg_parser.parse_args()
     if len(args.input) == 0:
-        args.input.append(DEFAULT_INPUT.as_posix())
+        args.input.append(DEFAULT_INPUT)
     data = analyze_files(args.input)
     with open(args.output, "w") as outfile:
         generate_uop_ids(args.input, data, outfile, args.namespace)