languages/cpython
languages/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython synced 2024-09-15 21:20:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

GH-116422: Tier2 hot/cold splitting (GH-116813)

Browse source 
Splits the "cold" path, deopts and exits, from the "hot" path, reducing the size of most jitted instructions, at the cost of slower exits.
This commit is contained in:
Mark Shannon 2024-03-26 09:35:11 +00:00 committed by GitHub
parent 61599a48f5
commit bf82f77957
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 1662 additions and 1003 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
Include/cpython/optimizer.h
View file

@ -30,16 +30,63 @@ typedef struct {
PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR).
} _PyVMData;
#define UOP_FORMAT_TARGET 0
#define UOP_FORMAT_EXIT 1
#define UOP_FORMAT_JUMP 2
#define UOP_FORMAT_UNUSED 3
/* Depending on the format,
* the 32 bits between the oparg and operand are:
* UOP_FORMAT_TARGET:
* uint32_t target;
* UOP_FORMAT_EXIT
* uint16_t exit_index;
* uint16_t error_target;
* UOP_FORMAT_JUMP
* uint16_t jump_target;
* uint16_t error_target;
*/
typedef struct {
uint16_t opcode;
uint16_t opcode:14;
uint16_t format:2;
uint16_t oparg;
union {
uint32_t target;
uint32_t exit_index;
struct {
union {
uint16_t exit_index;
uint16_t jump_target;
};
uint16_t error_target;
};
};
uint64_t operand; // A cache entry
} _PyUOpInstruction;
static inline uint32_t uop_get_target(const _PyUOpInstruction *inst)
{
assert(inst->format == UOP_FORMAT_TARGET);
return inst->target;
}
static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst)
{
assert(inst->format == UOP_FORMAT_EXIT);
return inst->exit_index;
}
static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst)
{
assert(inst->format == UOP_FORMAT_JUMP);
return inst->jump_target;
}
static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
{
assert(inst->format != UOP_FORMAT_TARGET);
return inst->error_target;
}
typedef struct _exit_data {
uint32_t target;
int16_t temperature;

116
Include/internal/pycore_opcode_metadata.h generated
View file

@ -921,6 +921,7 @@ enum InstructionFormat {
#define HAS_PURE_FLAG (2048)
#define HAS_PASSTHROUGH_FLAG (4096)
#define HAS_OPARG_AND_1_FLAG (8192)
#define HAS_ERROR_NO_POP_FLAG (16384)
#define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG))
#define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG))
#define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG))
@ -935,6 +936,7 @@ enum InstructionFormat {
#define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG))
#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG))
#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG))
#define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG))
#define OPARG_FULL 0
#define OPARG_CACHE_1 1
@ -954,17 +956,17 @@ struct opcode_metadata {
extern const struct opcode_metadata _PyOpcode_opcode_metadata[268];
#ifdef NEED_OPCODE_METADATA
const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG },
[BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG },
[BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG },
[BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -975,25 +977,25 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
[CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
[CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
[CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG },
[CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -1005,11 +1007,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -1017,40 +1019,40 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG },
[COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG },
[DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
[END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
[ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG },
[EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG },
[FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
[FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG },
[GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 },
[INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG },
[INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
@ -1059,10 +1061,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
[INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
[INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
[IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -1076,10 +1078,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
@ -1089,20 +1091,20 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG },
[LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG },
[LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -1117,15 +1119,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
[PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 },
[PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
[RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
[RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
[RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG },
[RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
[SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -1133,7 +1135,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG },
[STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
@ -1147,12 +1149,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG },
[TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
@ -1188,8 +1190,6 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256];
#ifdef NEED_OPCODE_METADATA
const struct opcode_macro_expansion
_PyOpcode_macro_expansion[256] = {
[BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { _BEFORE_ASYNC_WITH, 0, 0 } } },
[BEFORE_WITH] = { .nuops = 1, .uops = { { _BEFORE_WITH, 0, 0 } } },
[BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } },
[BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } },
[BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } },
@ -1207,7 +1207,6 @@ _PyOpcode_macro_expansion[256] = {
[BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } },
[BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } },
[BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } },
[BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } },
[BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } },
[BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } },
[BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } },
@ -1291,7 +1290,6 @@ _PyOpcode_macro_expansion[256] = {
[LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } },
[LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } },
[LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } },
[LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } },
[LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } },
[LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } },
[MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } },

2
Include/internal/pycore_optimizer.h
View file

@ -12,7 +12,7 @@ extern "C" {
#include <stdbool.h>
// This is the length of the trace we project initially.
#define UOP_MAX_TRACE_LENGTH 512
#define UOP_MAX_TRACE_LENGTH 800
#define TRACE_STACK_SIZE 5

195
Include/internal/pycore_uop_ids.h generated
View file

@ -87,45 +87,47 @@ extern "C" {
#define _DELETE_GLOBAL DELETE_GLOBAL
#define _DELETE_NAME DELETE_NAME
#define _DELETE_SUBSCR DELETE_SUBSCR
#define _DEOPT 341
#define _DICT_MERGE DICT_MERGE
#define _DICT_UPDATE DICT_UPDATE
#define _END_SEND END_SEND
#define _ERROR_POP_N 342
#define _EXIT_INIT_CHECK EXIT_INIT_CHECK
#define _FATAL_ERROR 341
#define _FATAL_ERROR 343
#define _FORMAT_SIMPLE FORMAT_SIMPLE
#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC
#define _FOR_ITER 342
#define _FOR_ITER 344
#define _FOR_ITER_GEN FOR_ITER_GEN
#define _FOR_ITER_TIER_TWO 343
#define _FOR_ITER_TIER_TWO 345
#define _GET_AITER GET_AITER
#define _GET_ANEXT GET_ANEXT
#define _GET_AWAITABLE GET_AWAITABLE
#define _GET_ITER GET_ITER
#define _GET_LEN GET_LEN
#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER
#define _GUARD_BOTH_FLOAT 344
#define _GUARD_BOTH_INT 345
#define _GUARD_BOTH_UNICODE 346
#define _GUARD_BUILTINS_VERSION 347
#define _GUARD_DORV_VALUES 348
#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 349
#define _GUARD_GLOBALS_VERSION 350
#define _GUARD_IS_FALSE_POP 351
#define _GUARD_IS_NONE_POP 352
#define _GUARD_IS_NOT_NONE_POP 353
#define _GUARD_IS_TRUE_POP 354
#define _GUARD_KEYS_VERSION 355
#define _GUARD_NOT_EXHAUSTED_LIST 356
#define _GUARD_NOT_EXHAUSTED_RANGE 357
#define _GUARD_NOT_EXHAUSTED_TUPLE 358
#define _GUARD_TYPE_VERSION 359
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 360
#define _INIT_CALL_PY_EXACT_ARGS 361
#define _INIT_CALL_PY_EXACT_ARGS_0 362
#define _INIT_CALL_PY_EXACT_ARGS_1 363
#define _INIT_CALL_PY_EXACT_ARGS_2 364
#define _INIT_CALL_PY_EXACT_ARGS_3 365
#define _INIT_CALL_PY_EXACT_ARGS_4 366
#define _GUARD_BOTH_FLOAT 346
#define _GUARD_BOTH_INT 347
#define _GUARD_BOTH_UNICODE 348
#define _GUARD_BUILTINS_VERSION 349
#define _GUARD_DORV_VALUES 350
#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351
#define _GUARD_GLOBALS_VERSION 352
#define _GUARD_IS_FALSE_POP 353
#define _GUARD_IS_NONE_POP 354
#define _GUARD_IS_NOT_NONE_POP 355
#define _GUARD_IS_TRUE_POP 356
#define _GUARD_KEYS_VERSION 357
#define _GUARD_NOT_EXHAUSTED_LIST 358
#define _GUARD_NOT_EXHAUSTED_RANGE 359
#define _GUARD_NOT_EXHAUSTED_TUPLE 360
#define _GUARD_TYPE_VERSION 361
#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 362
#define _INIT_CALL_PY_EXACT_ARGS 363
#define _INIT_CALL_PY_EXACT_ARGS_0 364
#define _INIT_CALL_PY_EXACT_ARGS_1 365
#define _INIT_CALL_PY_EXACT_ARGS_2 366
#define _INIT_CALL_PY_EXACT_ARGS_3 367
#define _INIT_CALL_PY_EXACT_ARGS_4 368
#define _INSTRUMENTED_CALL INSTRUMENTED_CALL
#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX
#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW
@ -142,65 +144,65 @@ extern "C" {
#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST
#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE
#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE
#define _INTERNAL_INCREMENT_OPT_COUNTER 367
#define _IS_NONE 368
#define _INTERNAL_INCREMENT_OPT_COUNTER 369
#define _IS_NONE 370
#define _IS_OP IS_OP
#define _ITER_CHECK_LIST 369
#define _ITER_CHECK_RANGE 370
#define _ITER_CHECK_TUPLE 371
#define _ITER_JUMP_LIST 372
#define _ITER_JUMP_RANGE 373
#define _ITER_JUMP_TUPLE 374
#define _ITER_NEXT_LIST 375
#define _ITER_NEXT_RANGE 376
#define _ITER_NEXT_TUPLE 377
#define _JUMP_TO_TOP 378
#define _ITER_CHECK_LIST 371
#define _ITER_CHECK_RANGE 372
#define _ITER_CHECK_TUPLE 373
#define _ITER_JUMP_LIST 374
#define _ITER_JUMP_RANGE 375
#define _ITER_JUMP_TUPLE 376
#define _ITER_NEXT_LIST 377
#define _ITER_NEXT_RANGE 378
#define _ITER_NEXT_TUPLE 379
#define _JUMP_TO_TOP 380
#define _LIST_APPEND LIST_APPEND
#define _LIST_EXTEND LIST_EXTEND
#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR
#define _LOAD_ATTR 379
#define _LOAD_ATTR_CLASS 380
#define _LOAD_ATTR_CLASS_0 381
#define _LOAD_ATTR_CLASS_1 382
#define _LOAD_ATTR 381
#define _LOAD_ATTR_CLASS 382
#define _LOAD_ATTR_CLASS_0 383
#define _LOAD_ATTR_CLASS_1 384
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
#define _LOAD_ATTR_INSTANCE_VALUE 383
#define _LOAD_ATTR_INSTANCE_VALUE_0 384
#define _LOAD_ATTR_INSTANCE_VALUE_1 385
#define _LOAD_ATTR_METHOD_LAZY_DICT 386
#define _LOAD_ATTR_METHOD_NO_DICT 387
#define _LOAD_ATTR_METHOD_WITH_VALUES 388
#define _LOAD_ATTR_MODULE 389
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 390
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 391
#define _LOAD_ATTR_INSTANCE_VALUE 385
#define _LOAD_ATTR_INSTANCE_VALUE_0 386
#define _LOAD_ATTR_INSTANCE_VALUE_1 387
#define _LOAD_ATTR_METHOD_LAZY_DICT 388
#define _LOAD_ATTR_METHOD_NO_DICT 389
#define _LOAD_ATTR_METHOD_WITH_VALUES 390
#define _LOAD_ATTR_MODULE 391
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 392
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 393
#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY
#define _LOAD_ATTR_SLOT 392
#define _LOAD_ATTR_SLOT_0 393
#define _LOAD_ATTR_SLOT_1 394
#define _LOAD_ATTR_WITH_HINT 395
#define _LOAD_ATTR_SLOT 394
#define _LOAD_ATTR_SLOT_0 395
#define _LOAD_ATTR_SLOT_1 396
#define _LOAD_ATTR_WITH_HINT 397
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
#define _LOAD_CONST LOAD_CONST
#define _LOAD_CONST_INLINE 396
#define _LOAD_CONST_INLINE_BORROW 397
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 398
#define _LOAD_CONST_INLINE_WITH_NULL 399
#define _LOAD_CONST_INLINE 398
#define _LOAD_CONST_INLINE_BORROW 399
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 400
#define _LOAD_CONST_INLINE_WITH_NULL 401
#define _LOAD_DEREF LOAD_DEREF
#define _LOAD_FAST 400
#define _LOAD_FAST_0 401
#define _LOAD_FAST_1 402
#define _LOAD_FAST_2 403
#define _LOAD_FAST_3 404
#define _LOAD_FAST_4 405
#define _LOAD_FAST_5 406
#define _LOAD_FAST_6 407
#define _LOAD_FAST_7 408
#define _LOAD_FAST 402
#define _LOAD_FAST_0 403
#define _LOAD_FAST_1 404
#define _LOAD_FAST_2 405
#define _LOAD_FAST_3 406
#define _LOAD_FAST_4 407
#define _LOAD_FAST_5 408
#define _LOAD_FAST_6 409
#define _LOAD_FAST_7 410
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
#define _LOAD_GLOBAL 409
#define _LOAD_GLOBAL_BUILTINS 410
#define _LOAD_GLOBAL_MODULE 411
#define _LOAD_GLOBAL 411
#define _LOAD_GLOBAL_BUILTINS 412
#define _LOAD_GLOBAL_MODULE 413
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
@ -214,48 +216,49 @@ extern "C" {
#define _MATCH_SEQUENCE MATCH_SEQUENCE
#define _NOP NOP
#define _POP_EXCEPT POP_EXCEPT
#define _POP_FRAME 412
#define _POP_JUMP_IF_FALSE 413
#define _POP_JUMP_IF_TRUE 414
#define _POP_FRAME 414
#define _POP_JUMP_IF_FALSE 415
#define _POP_JUMP_IF_TRUE 416
#define _POP_TOP POP_TOP
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 415
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 417
#define _PUSH_EXC_INFO PUSH_EXC_INFO
#define _PUSH_FRAME 416
#define _PUSH_FRAME 418
#define _PUSH_NULL PUSH_NULL
#define _REPLACE_WITH_TRUE 417
#define _REPLACE_WITH_TRUE 419
#define _RESUME_CHECK RESUME_CHECK
#define _SAVE_RETURN_OFFSET 418
#define _SEND 419
#define _SAVE_RETURN_OFFSET 420
#define _SEND 421
#define _SEND_GEN SEND_GEN
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
#define _START_EXECUTOR 420
#define _STORE_ATTR 421
#define _STORE_ATTR_INSTANCE_VALUE 422
#define _STORE_ATTR_SLOT 423
#define _SIDE_EXIT 422
#define _START_EXECUTOR 423
#define _STORE_ATTR 424
#define _STORE_ATTR_INSTANCE_VALUE 425
#define _STORE_ATTR_SLOT 426
#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT
#define _STORE_DEREF STORE_DEREF
#define _STORE_FAST 424
#define _STORE_FAST_0 425
#define _STORE_FAST_1 426
#define _STORE_FAST_2 427
#define _STORE_FAST_3 428
#define _STORE_FAST_4 429
#define _STORE_FAST_5 430
#define _STORE_FAST_6 431
#define _STORE_FAST_7 432
#define _STORE_FAST 427
#define _STORE_FAST_0 428
#define _STORE_FAST_1 429
#define _STORE_FAST_2 430
#define _STORE_FAST_3 431
#define _STORE_FAST_4 432
#define _STORE_FAST_5 433
#define _STORE_FAST_6 434
#define _STORE_FAST_7 435
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
#define _STORE_SLICE STORE_SLICE
#define _STORE_SUBSCR 433
#define _STORE_SUBSCR 436
#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
#define _SWAP SWAP
#define _TO_BOOL 434
#define _TO_BOOL 437
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
#define _TO_BOOL_LIST TO_BOOL_LIST
@ -265,12 +268,12 @@ extern "C" {
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
#define _UNPACK_SEQUENCE 435
#define _UNPACK_SEQUENCE 438
#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
#define _WITH_EXCEPT_START WITH_EXCEPT_START
#define MAX_UOP_ID 435
#define MAX_UOP_ID 438
#ifdef __cplusplus
}

536
Include/internal/pycore_uop_metadata.h generated
View file

@ -15,11 +15,13 @@ extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];
extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];
extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];
extern int _PyUop_num_popped(int opcode, int oparg);
#ifdef NEED_OPCODE_METADATA
const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_NOP] = HAS_PURE_FLAG,
[_RESUME_CHECK] = HAS_DEOPT_FLAG,
[_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG,
[_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG,
[_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG,
[_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG,
@ -49,22 +51,22 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_UNARY_NOT] = HAS_PURE_FLAG,
[_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_TO_BOOL_BOOL] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_TO_BOOL_INT] = HAS_EXIT_FLAG,
[_TO_BOOL_LIST] = HAS_EXIT_FLAG,
[_TO_BOOL_NONE] = HAS_EXIT_FLAG,
[_TO_BOOL_STR] = HAS_EXIT_FLAG,
[_REPLACE_WITH_TRUE] = 0,
[_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_BOTH_INT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG,
[_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG,
[_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG,
[_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
@ -83,13 +85,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_POP_FRAME] = HAS_ESCAPES_FLAG,
[_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_POP_EXCEPT] = HAS_ESCAPES_FLAG,
[_LOAD_ASSERTION_ERROR] = 0,
[_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
@ -98,19 +100,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG,
[_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG,
[_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG,
[_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG,
[_COPY_FREE_VARS] = HAS_ARG_FLAG,
@ -119,7 +120,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG,
[_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
@ -129,7 +129,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG,
@ -165,8 +165,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_MATCH_SEQUENCE] = 0,
[_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_ITER_NEXT_LIST] = 0,
@ -176,8 +176,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_ITER_NEXT_RANGE] = HAS_ERROR_FLAG,
[_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_PUSH_EXC_INFO] = 0,
[_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
@ -204,18 +202,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG,
[_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
[_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG,
[_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG,
@ -224,14 +222,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG,
[_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG,
[_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG,
[_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG,
[_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG,
[_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG,
[_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG,
[_SET_IP] = 0,
[_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
[_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
[_EXIT_TRACE] = HAS_EXIT_FLAG,
[_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
[_LOAD_CONST_INLINE] = HAS_PURE_FLAG,
[_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG,
@ -240,10 +238,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG,
[_CHECK_FUNCTION] = HAS_DEOPT_FLAG,
[_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
[_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
[_START_EXECUTOR] = 0,
[_FATAL_ERROR] = HAS_ESCAPES_FLAG,
[_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG,
[_DEOPT] = 0,
[_SIDE_EXIT] = 0,
[_ERROR_POP_N] = HAS_ARG_FLAG,
};
const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {
@ -253,8 +254,6 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {
};
const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH",
[_BEFORE_WITH] = "_BEFORE_WITH",
[_BINARY_OP] = "_BINARY_OP",
[_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT",
[_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT",
@ -272,7 +271,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP",
[_BUILD_LIST] = "_BUILD_LIST",
[_BUILD_MAP] = "_BUILD_MAP",
[_BUILD_SET] = "_BUILD_SET",
[_BUILD_SLICE] = "_BUILD_SLICE",
[_BUILD_STRING] = "_BUILD_STRING",
[_BUILD_TUPLE] = "_BUILD_TUPLE",
@ -323,9 +321,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_DELETE_GLOBAL] = "_DELETE_GLOBAL",
[_DELETE_NAME] = "_DELETE_NAME",
[_DELETE_SUBSCR] = "_DELETE_SUBSCR",
[_DEOPT] = "_DEOPT",
[_DICT_MERGE] = "_DICT_MERGE",
[_DICT_UPDATE] = "_DICT_UPDATE",
[_END_SEND] = "_END_SEND",
[_ERROR_POP_N] = "_ERROR_POP_N",
[_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK",
[_EXIT_TRACE] = "_EXIT_TRACE",
[_FATAL_ERROR] = "_FATAL_ERROR",
@ -416,7 +416,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS",
[_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE",
[_LOAD_LOCALS] = "_LOAD_LOCALS",
[_LOAD_NAME] = "_LOAD_NAME",
[_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR",
[_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD",
[_MAKE_CELL] = "_MAKE_CELL",
@ -442,6 +441,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE",
[_SET_IP] = "_SET_IP",
[_SET_UPDATE] = "_SET_UPDATE",
[_SIDE_EXIT] = "_SIDE_EXIT",
[_START_EXECUTOR] = "_START_EXECUTOR",
[_STORE_ATTR] = "_STORE_ATTR",
[_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE",
@ -481,6 +481,466 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE",
[_WITH_EXCEPT_START] = "_WITH_EXCEPT_START",
};
int _PyUop_num_popped(int opcode, int oparg)
{
switch(opcode) {
case _NOP:
return 0;
case _RESUME_CHECK:
return 0;
case _LOAD_FAST_CHECK:
return 0;
case _LOAD_FAST_0:
return 0;
case _LOAD_FAST_1:
return 0;
case _LOAD_FAST_2:
return 0;
case _LOAD_FAST_3:
return 0;
case _LOAD_FAST_4:
return 0;
case _LOAD_FAST_5:
return 0;
case _LOAD_FAST_6:
return 0;
case _LOAD_FAST_7:
return 0;
case _LOAD_FAST:
return 0;
case _LOAD_FAST_AND_CLEAR:
return 0;
case _LOAD_FAST_LOAD_FAST:
return 0;
case _LOAD_CONST:
return 0;
case _STORE_FAST_0:
return 1;
case _STORE_FAST_1:
return 1;
case _STORE_FAST_2:
return 1;
case _STORE_FAST_3:
return 1;
case _STORE_FAST_4:
return 1;
case _STORE_FAST_5:
return 1;
case _STORE_FAST_6:
return 1;
case _STORE_FAST_7:
return 1;
case _STORE_FAST:
return 1;
case _STORE_FAST_LOAD_FAST:
return 1;
case _STORE_FAST_STORE_FAST:
return 2;
case _POP_TOP:
return 1;
case _PUSH_NULL:
return 0;
case _END_SEND:
return 2;
case _UNARY_NEGATIVE:
return 1;
case _UNARY_NOT:
return 1;
case _TO_BOOL:
return 1;
case _TO_BOOL_BOOL:
return 1;
case _TO_BOOL_INT:
return 1;
case _TO_BOOL_LIST:
return 1;
case _TO_BOOL_NONE:
return 1;
case _TO_BOOL_STR:
return 1;
case _REPLACE_WITH_TRUE:
return 1;
case _UNARY_INVERT:
return 1;
case _GUARD_BOTH_INT:
return 2;
case _BINARY_OP_MULTIPLY_INT:
return 2;
case _BINARY_OP_ADD_INT:
return 2;
case _BINARY_OP_SUBTRACT_INT:
return 2;
case _GUARD_BOTH_FLOAT:
return 2;
case _BINARY_OP_MULTIPLY_FLOAT:
return 2;
case _BINARY_OP_ADD_FLOAT:
return 2;
case _BINARY_OP_SUBTRACT_FLOAT:
return 2;
case _GUARD_BOTH_UNICODE:
return 2;
case _BINARY_OP_ADD_UNICODE:
return 2;
case _BINARY_SUBSCR:
return 2;
case _BINARY_SLICE:
return 3;
case _STORE_SLICE:
return 4;
case _BINARY_SUBSCR_LIST_INT:
return 2;
case _BINARY_SUBSCR_STR_INT:
return 2;
case _BINARY_SUBSCR_TUPLE_INT:
return 2;
case _BINARY_SUBSCR_DICT:
return 2;
case _LIST_APPEND:
return 2 + (oparg-1);
case _SET_ADD:
return 2 + (oparg-1);
case _STORE_SUBSCR:
return 3;
case _STORE_SUBSCR_LIST_INT:
return 3;
case _STORE_SUBSCR_DICT:
return 3;
case _DELETE_SUBSCR:
return 2;
case _CALL_INTRINSIC_1:
return 1;
case _CALL_INTRINSIC_2:
return 2;
case _POP_FRAME:
return 1;
case _GET_AITER:
return 1;
case _GET_ANEXT:
return 1;
case _GET_AWAITABLE:
return 1;
case _POP_EXCEPT:
return 1;
case _LOAD_ASSERTION_ERROR:
return 0;
case _LOAD_BUILD_CLASS:
return 0;
case _STORE_NAME:
return 1;
case _DELETE_NAME:
return 0;
case _UNPACK_SEQUENCE:
return 1;
case _UNPACK_SEQUENCE_TWO_TUPLE:
return 1;
case _UNPACK_SEQUENCE_TUPLE:
return 1;
case _UNPACK_SEQUENCE_LIST:
return 1;
case _UNPACK_EX:
return 1;
case _STORE_ATTR:
return 2;
case _DELETE_ATTR:
return 1;
case _STORE_GLOBAL:
return 1;
case _DELETE_GLOBAL:
return 0;
case _LOAD_LOCALS:
return 0;
case _LOAD_FROM_DICT_OR_GLOBALS:
return 1;
case _LOAD_GLOBAL:
return 0;
case _GUARD_GLOBALS_VERSION:
return 0;
case _GUARD_BUILTINS_VERSION:
return 0;
case _LOAD_GLOBAL_MODULE:
return 0;
case _LOAD_GLOBAL_BUILTINS:
return 0;
case _DELETE_FAST:
return 0;
case _MAKE_CELL:
return 0;
case _DELETE_DEREF:
return 0;
case _LOAD_FROM_DICT_OR_DEREF:
return 1;
case _LOAD_DEREF:
return 0;
case _STORE_DEREF:
return 1;
case _COPY_FREE_VARS:
return 0;
case _BUILD_STRING:
return oparg;
case _BUILD_TUPLE:
return oparg;
case _BUILD_LIST:
return oparg;
case _LIST_EXTEND:
return 2 + (oparg-1);
case _SET_UPDATE:
return 2 + (oparg-1);
case _BUILD_MAP:
return oparg*2;
case _SETUP_ANNOTATIONS:
return 0;
case _BUILD_CONST_KEY_MAP:
return 1 + oparg;
case _DICT_UPDATE:
return 2 + (oparg - 1);
case _DICT_MERGE:
return 5 + (oparg - 1);
case _MAP_ADD:
return 3 + (oparg - 1);
case _LOAD_SUPER_ATTR_ATTR:
return 3;
case _LOAD_SUPER_ATTR_METHOD:
return 3;
case _LOAD_ATTR:
return 1;
case _GUARD_TYPE_VERSION:
return 1;
case _CHECK_MANAGED_OBJECT_HAS_VALUES:
return 1;
case _LOAD_ATTR_INSTANCE_VALUE_0:
return 1;
case _LOAD_ATTR_INSTANCE_VALUE_1:
return 1;
case _LOAD_ATTR_INSTANCE_VALUE:
return 1;
case _CHECK_ATTR_MODULE:
return 1;
case _LOAD_ATTR_MODULE:
return 1;
case _CHECK_ATTR_WITH_HINT:
return 1;
case _LOAD_ATTR_WITH_HINT:
return 1;
case _LOAD_ATTR_SLOT_0:
return 1;
case _LOAD_ATTR_SLOT_1:
return 1;
case _LOAD_ATTR_SLOT:
return 1;
case _CHECK_ATTR_CLASS:
return 1;
case _LOAD_ATTR_CLASS_0:
return 1;
case _LOAD_ATTR_CLASS_1:
return 1;
case _LOAD_ATTR_CLASS:
return 1;
case _GUARD_DORV_VALUES:
return 1;
case _STORE_ATTR_INSTANCE_VALUE:
return 2;
case _STORE_ATTR_SLOT:
return 2;
case _COMPARE_OP:
return 2;
case _COMPARE_OP_FLOAT:
return 2;
case _COMPARE_OP_INT:
return 2;
case _COMPARE_OP_STR:
return 2;
case _IS_OP:
return 2;
case _CONTAINS_OP:
return 2;
case _CONTAINS_OP_SET:
return 2;
case _CONTAINS_OP_DICT:
return 2;
case _CHECK_EG_MATCH:
return 2;
case _CHECK_EXC_MATCH:
return 2;
case _IS_NONE:
return 1;
case _GET_LEN:
return 1;
case _MATCH_CLASS:
return 3;
case _MATCH_MAPPING:
return 1;
case _MATCH_SEQUENCE:
return 1;
case _MATCH_KEYS:
return 2;
case _GET_ITER:
return 1;
case _GET_YIELD_FROM_ITER:
return 1;
case _FOR_ITER_TIER_TWO:
return 1;
case _ITER_CHECK_LIST:
return 1;
case _GUARD_NOT_EXHAUSTED_LIST:
return 1;
case _ITER_NEXT_LIST:
return 1;
case _ITER_CHECK_TUPLE:
return 1;
case _GUARD_NOT_EXHAUSTED_TUPLE:
return 1;
case _ITER_NEXT_TUPLE:
return 1;
case _ITER_CHECK_RANGE:
return 1;
case _GUARD_NOT_EXHAUSTED_RANGE:
return 1;
case _ITER_NEXT_RANGE:
return 1;
case _WITH_EXCEPT_START:
return 4;
case _PUSH_EXC_INFO:
return 1;
case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT:
return 1;
case _GUARD_KEYS_VERSION:
return 1;
case _LOAD_ATTR_METHOD_WITH_VALUES:
return 1;
case _LOAD_ATTR_METHOD_NO_DICT:
return 1;
case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES:
return 1;
case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT:
return 1;
case _CHECK_ATTR_METHOD_LAZY_DICT:
return 1;
case _LOAD_ATTR_METHOD_LAZY_DICT:
return 1;
case _CHECK_PERIODIC:
return 0;
case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS:
return 2 + oparg;
case _INIT_CALL_BOUND_METHOD_EXACT_ARGS:
return 2 + oparg;
case _CHECK_PEP_523:
return 0;
case _CHECK_FUNCTION_EXACT_ARGS:
return 2 + oparg;
case _CHECK_STACK_SPACE:
return 2 + oparg;
case _INIT_CALL_PY_EXACT_ARGS_0:
return 2 + oparg;
case _INIT_CALL_PY_EXACT_ARGS_1:
return 2 + oparg;
case _INIT_CALL_PY_EXACT_ARGS_2:
return 2 + oparg;
case _INIT_CALL_PY_EXACT_ARGS_3:
return 2 + oparg;
case _INIT_CALL_PY_EXACT_ARGS_4:
return 2 + oparg;
case _INIT_CALL_PY_EXACT_ARGS:
return 2 + oparg;
case _PUSH_FRAME:
return 1;
case _CALL_TYPE_1:
return 3;
case _CALL_STR_1:
return 3;
case _CALL_TUPLE_1:
return 3;
case _EXIT_INIT_CHECK:
return 1;
case _CALL_BUILTIN_CLASS:
return 2 + oparg;
case _CALL_BUILTIN_O:
return 2 + oparg;
case _CALL_BUILTIN_FAST:
return 2 + oparg;
case _CALL_BUILTIN_FAST_WITH_KEYWORDS:
return 2 + oparg;
case _CALL_LEN:
return 2 + oparg;
case _CALL_ISINSTANCE:
return 2 + oparg;
case _CALL_METHOD_DESCRIPTOR_O:
return 2 + oparg;
case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
return 2 + oparg;
case _CALL_METHOD_DESCRIPTOR_NOARGS:
return 2 + oparg;
case _CALL_METHOD_DESCRIPTOR_FAST:
return 2 + oparg;
case _MAKE_FUNCTION:
return 1;
case _SET_FUNCTION_ATTRIBUTE:
return 2;
case _BUILD_SLICE:
return 2 + ((oparg == 3) ? 1 : 0);
case _CONVERT_VALUE:
return 1;
case _FORMAT_SIMPLE:
return 1;
case _FORMAT_WITH_SPEC:
return 2;
case _COPY:
return 1 + (oparg-1);
case _BINARY_OP:
return 2;
case _SWAP:
return 2 + (oparg-2);
case _GUARD_IS_TRUE_POP:
return 1;
case _GUARD_IS_FALSE_POP:
return 1;
case _GUARD_IS_NONE_POP:
return 1;
case _GUARD_IS_NOT_NONE_POP:
return 1;
case _JUMP_TO_TOP:
return 0;
case _SET_IP:
return 0;
case _SAVE_RETURN_OFFSET:
return 0;
case _EXIT_TRACE:
return 0;
case _CHECK_VALIDITY:
return 0;
case _LOAD_CONST_INLINE:
return 0;
case _LOAD_CONST_INLINE_BORROW:
return 0;
case _POP_TOP_LOAD_CONST_INLINE_BORROW:
return 1;
case _LOAD_CONST_INLINE_WITH_NULL:
return 0;
case _LOAD_CONST_INLINE_BORROW_WITH_NULL:
return 0;
case _CHECK_FUNCTION:
return 0;
case _INTERNAL_INCREMENT_OPT_COUNTER:
return 1;
case _COLD_EXIT:
return 0;
case _START_EXECUTOR:
return 0;
case _FATAL_ERROR:
return 0;
case _CHECK_VALIDITY_AND_SET_IP:
return 0;
case _DEOPT:
return 0;
case _SIDE_EXIT:
return 0;
case _ERROR_POP_N:
return oparg;
default:
return -1;
}
}
#endif // NEED_OPCODE_METADATA

139
Python/bytecodes.c
View file

@ -179,7 +179,7 @@ dummy_func(
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
next_instr = this_instr;
}
@ -206,7 +206,13 @@ dummy_func(
inst(LOAD_FAST_CHECK, (-- value)) {
value = GETLOCAL(oparg);
ERROR_IF(value == NULL, unbound_local_error);
if (value == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
ERROR_IF(1, error);
}
Py_INCREF(value);
}
@ -275,7 +281,7 @@ dummy_func(
if (PyGen_Check(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
PyErr_SetRaisedException(NULL);
}
@ -290,7 +296,7 @@ dummy_func(
if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
PyErr_SetRaisedException(NULL);
}
@ -826,7 +832,7 @@ dummy_func(
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
if (err) GOTO_ERROR(error);
if (err) ERROR_NO_POP();
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -850,7 +856,7 @@ dummy_func(
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
if (err) GOTO_ERROR(error);
if (err) ERROR_NO_POP();
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -906,7 +912,7 @@ dummy_func(
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
} else {
if (type->tp_as_async != NULL){
@ -916,7 +922,7 @@ dummy_func(
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
else {
@ -924,7 +930,7 @@ dummy_func(
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
GOTO_ERROR(error);
ERROR_NO_POP();
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
@ -936,7 +942,7 @@ dummy_func(
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
GOTO_ERROR(error);
ERROR_NO_POP();
} else {
Py_DECREF(next_iter);
}
@ -1018,7 +1024,7 @@ dummy_func(
JUMPBY(oparg);
}
else {
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
Py_DECREF(v);
@ -1054,7 +1060,7 @@ dummy_func(
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, retval);
if (err) GOTO_ERROR(error);
if (err) ERROR_NO_POP();
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
@ -1108,7 +1114,7 @@ dummy_func(
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
assert(exc && PyExceptionInstance_Check(exc));
@ -1184,7 +1190,7 @@ dummy_func(
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
GOTO_ERROR(error);
ERROR_NO_POP();
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
@ -1192,7 +1198,7 @@ dummy_func(
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
@ -1312,12 +1318,12 @@ dummy_func(
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
@ -1334,21 +1340,21 @@ dummy_func(
inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
}
@ -1364,21 +1370,21 @@ dummy_func(
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
}
@ -1494,7 +1500,13 @@ dummy_func(
inst(DELETE_FAST, (--)) {
PyObject *v = GETLOCAL(oparg);
ERROR_IF(v == NULL, unbound_local_error);
if (v == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
ERROR_IF(1, error);
}
SETLOCAL(oparg, NULL);
}
@ -1504,7 +1516,7 @@ dummy_func(
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
SETLOCAL(oparg, cell);
}
@ -1516,7 +1528,7 @@ dummy_func(
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
GOTO_ERROR(error);
ERROR_NO_POP();
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
@ -1528,14 +1540,14 @@ dummy_func(
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
if (!value) {
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
GOTO_ERROR(error);
ERROR_NO_POP();
}
Py_INCREF(value);
}
@ -1615,7 +1627,7 @@ dummy_func(
inst(BUILD_SET, (values[oparg] -- set)) {
set = PySet_New(NULL);
if (set == NULL)
GOTO_ERROR(error);
ERROR_NO_POP();
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
@ -1662,12 +1674,8 @@ dummy_func(
}
inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) {
if (!PyTuple_CheckExact(keys) ||
PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
_PyErr_SetString(tstate, PyExc_SystemError,
"bad BUILD_CONST_KEY_MAP keys argument");
GOTO_ERROR(error); // Pop the keys and values.
}
assert(PyTuple_CheckExact(keys));
assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
@ -2502,7 +2510,7 @@ dummy_func(
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
GOTO_ERROR(error);
ERROR_NO_POP();
}
iter = iterable;
}
@ -2513,7 +2521,7 @@ dummy_func(
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
DECREF_INPUTS();
}
@ -2550,7 +2558,7 @@ dummy_func(
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
@ -2573,7 +2581,7 @@ dummy_func(
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
_PyErr_Clear(tstate);
}
@ -2599,7 +2607,7 @@ dummy_func(
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
@ -2779,7 +2787,7 @@ dummy_func(
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
GOTO_ERROR(error);
ERROR_NO_POP();
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
@ -2791,7 +2799,7 @@ dummy_func(
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
GOTO_ERROR(error);
ERROR_NO_POP();
}
DECREF_INPUTS();
res = PyObject_CallNoArgs(enter);
@ -2814,7 +2822,7 @@ dummy_func(
"context manager protocol",
Py_TYPE(mgr)->tp_name);
}
GOTO_ERROR(error);
ERROR_NO_POP();
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
if (exit == NULL) {
@ -2826,7 +2834,7 @@ dummy_func(
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
GOTO_ERROR(error);
ERROR_NO_POP();
}
DECREF_INPUTS();
res = PyObject_CallNoArgs(enter);
@ -3075,7 +3083,7 @@ dummy_func(
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
@ -3298,7 +3306,7 @@ dummy_func(
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
Py_DECREF(tp);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
@ -3335,7 +3343,7 @@ dummy_func(
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
GOTO_ERROR(error);
ERROR_NO_POP();
}
}
@ -3472,7 +3480,7 @@ dummy_func(
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@ -3498,7 +3506,7 @@ dummy_func(
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@ -3712,7 +3720,7 @@ dummy_func(
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
@ -3760,11 +3768,11 @@ dummy_func(
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
Py_SETREF(callargs, tuple);
}
@ -3776,7 +3784,7 @@ dummy_func(
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
if (err) GOTO_ERROR(error);
if (err) ERROR_NO_POP();
result = PyObject_Call(func, callargs, kwargs);
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
@ -3810,7 +3818,7 @@ dummy_func(
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
@ -3831,7 +3839,7 @@ dummy_func(
Py_DECREF(codeobj);
if (func_obj == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
_PyFunction_SetVersion(
@ -3871,7 +3879,7 @@ dummy_func(
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
GOTO_ERROR(error);
ERROR_NO_POP();
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -4169,7 +4177,7 @@ dummy_func(
if (optimized < 0) {
Py_DECREF(previous);
tstate->previous_executor = Py_None;
ERROR_IF(1, error);
GOTO_UNWIND();
}
GOTO_TIER_ONE(target);
}
@ -4199,6 +4207,19 @@ dummy_func(
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
}
tier2 op(_DEOPT, (--)) {
EXIT_TO_TIER1();
}
tier2 op(_SIDE_EXIT, (--)) {
EXIT_TO_TRACE();
}
tier2 op(_ERROR_POP_N, (unused[oparg] --)) {
SYNC_SP();
GOTO_UNWIND();
}
// END BYTECODES //
}

57
Python/ceval.c
View file

@ -642,7 +642,6 @@ int _Py_CheckRecursiveCallPy(
return 0;
}
static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
/* Put a NOP at the start, so that the IP points into
* the code, rather than before it */
@ -850,15 +849,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
or goto error. */
Py_UNREACHABLE();
unbound_local_error:
{
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
goto error;
}
pop_4_error:
STACK_SHRINK(1);
pop_3_error:
@ -980,12 +970,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
#undef GOTO_ERROR
#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two
#undef DEOPT_IF
#define DEOPT_IF(COND, INSTNAME) \
if ((COND)) { \
goto deoptimize;\
}
#ifdef Py_STATS
// Disable these macros that apply to Tier 1 stats when we are in Tier 2
#undef STAT_INC
@ -1013,6 +997,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
#endif
assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT);
tier2_dispatch:
for (;;) {
uopcode = next_uop->opcode;
#ifdef Py_DEBUG
@ -1054,24 +1039,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
}
}
// Jump here from ERROR_IF(..., unbound_local_error)
unbound_local_error_tier_two:
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
goto error_tier_two;
// JUMP to any of these from ERROR_IF(..., error)
pop_4_error_tier_two:
STACK_SHRINK(1);
pop_3_error_tier_two:
STACK_SHRINK(1);
pop_2_error_tier_two:
STACK_SHRINK(1);
pop_1_error_tier_two:
STACK_SHRINK(1);
error_tier_two:
jump_to_error_target:
#ifdef Py_DEBUG
if (lltrace >= 2) {
printf("Error: [UOp ");
@ -1081,15 +1049,28 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
_PyOpcode_OpName[frame->instr_ptr->op.code]);
}
#endif
assert (next_uop[-1].format == UOP_FORMAT_JUMP);
uint16_t target = uop_get_error_target(&next_uop[-1]);
next_uop = current_executor->trace + target;
goto tier2_dispatch;
error_tier_two:
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
assert(next_uop[-1].format == UOP_FORMAT_TARGET);
frame->return_offset = 0; // Don't leave this random
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(current_executor);
tstate->previous_executor = NULL;
goto resume_with_error;
// Jump here from DEOPT_IF()
deoptimize:
jump_to_jump_target:
assert(next_uop[-1].format == UOP_FORMAT_JUMP);
target = uop_get_jump_target(&next_uop[-1]);
next_uop = current_executor->trace + target;
goto tier2_dispatch;
exit_to_tier1:
assert(next_uop[-1].format == UOP_FORMAT_TARGET);
next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame));
#ifdef Py_DEBUG
if (lltrace >= 2) {
@ -1105,8 +1086,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
tstate->previous_executor = NULL;
DISPATCH();
// Jump here from EXIT_IF()
side_exit:
exit_to_trace:
assert(next_uop[-1].format == UOP_FORMAT_EXIT);
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
UOP_STAT_INC(uopcode, miss);
uint32_t exit_index = next_uop[-1].exit_index;

6
Python/ceval_macros.h
View file

@ -423,3 +423,9 @@ do { \
#define CURRENT_OPARG() (next_uop[-1].oparg)
#define CURRENT_OPERAND() (next_uop[-1].operand)
#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target
#define JUMP_TO_ERROR() goto jump_to_error_target
#define GOTO_UNWIND() goto error_tier_two
#define EXIT_TO_TRACE() goto exit_to_trace
#define EXIT_TO_TIER1() goto exit_to_tier1

738
Python/executor_cases.c.h generated
View file

File diff suppressed because it is too large Load diff

122
Python/generated_cases.c.h generated
View file

@ -25,7 +25,7 @@
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
GOTO_ERROR(error);
goto error;
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
@ -37,7 +37,7 @@
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
GOTO_ERROR(error);
goto error;
}
Py_DECREF(mgr);
res = PyObject_CallNoArgs(enter);
@ -71,7 +71,7 @@
"context manager protocol",
Py_TYPE(mgr)->tp_name);
}
GOTO_ERROR(error);
goto error;
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
if (exit == NULL) {
@ -83,7 +83,7 @@
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
GOTO_ERROR(error);
goto error;
}
Py_DECREF(mgr);
res = PyObject_CallNoArgs(enter);
@ -605,12 +605,8 @@
PyObject *map;
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
if (!PyTuple_CheckExact(keys) ||
PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
_PyErr_SetString(tstate, PyExc_SystemError,
"bad BUILD_CONST_KEY_MAP keys argument");
GOTO_ERROR(error); // Pop the keys and values.
}
assert(PyTuple_CheckExact(keys));
assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
@ -667,7 +663,7 @@
values = &stack_pointer[-oparg];
set = PySet_New(NULL);
if (set == NULL)
GOTO_ERROR(error);
goto error;
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
@ -808,7 +804,7 @@
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
GOTO_ERROR(error);
goto error;
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
@ -882,7 +878,7 @@
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
GOTO_ERROR(error);
goto error;
}
Py_DECREF(tp);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
@ -1213,11 +1209,11 @@
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
GOTO_ERROR(error);
goto error;
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
GOTO_ERROR(error);
goto error;
}
Py_SETREF(callargs, tuple);
}
@ -1229,7 +1225,7 @@
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
if (err) GOTO_ERROR(error);
if (err) goto error;
result = PyObject_Call(func, callargs, kwargs);
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
if (result == NULL) {
@ -1261,7 +1257,7 @@
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
GOTO_ERROR(error);
goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
@ -1342,7 +1338,7 @@
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
GOTO_ERROR(error);
goto error;
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@ -1407,7 +1403,7 @@
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
GOTO_ERROR(error);
goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
@ -1475,7 +1471,7 @@
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
GOTO_ERROR(error);
goto error;
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@ -2329,7 +2325,7 @@
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
GOTO_ERROR(error);
goto error;
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
@ -2341,7 +2337,13 @@
next_instr += 1;
INSTRUCTION_STATS(DELETE_FAST);
PyObject *v = GETLOCAL(oparg);
if (v == NULL) goto unbound_local_error;
if (v == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
SETLOCAL(oparg, NULL);
DISPATCH();
}
@ -2354,12 +2356,12 @@
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
GOTO_ERROR(error);
goto error;
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
GOTO_ERROR(error);
goto error;
}
DISPATCH();
}
@ -2374,7 +2376,7 @@
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
GOTO_ERROR(error);
goto error;
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
@ -2382,7 +2384,7 @@
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
GOTO_ERROR(error);
goto error;
}
DISPATCH();
}
@ -2523,7 +2525,7 @@
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
GOTO_ERROR(error);
goto error;
}
stack_pointer += -1;
DISPATCH();
@ -2610,7 +2612,7 @@
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
GOTO_ERROR(error);
goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
@ -2841,7 +2843,7 @@
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
GOTO_ERROR(error);
goto error;
}
} else {
if (type->tp_as_async != NULL){
@ -2850,7 +2852,7 @@
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
GOTO_ERROR(error);
goto error;
}
}
else {
@ -2858,7 +2860,7 @@
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
GOTO_ERROR(error);
goto error;
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
@ -2868,7 +2870,7 @@
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
GOTO_ERROR(error);
goto error;
} else {
Py_DECREF(next_iter);
}
@ -2956,7 +2958,7 @@
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
GOTO_ERROR(error);
goto error;
}
iter = iterable;
}
@ -2967,7 +2969,7 @@
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
GOTO_ERROR(error);
goto error;
}
Py_DECREF(iterable);
}
@ -3066,7 +3068,7 @@
if (PyGen_Check(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
GOTO_ERROR(error);
goto error;
}
PyErr_SetRaisedException(NULL);
}
@ -3087,7 +3089,7 @@
if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
GOTO_ERROR(error);
goto error;
}
PyErr_SetRaisedException(NULL);
}
@ -3113,7 +3115,7 @@
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
GOTO_ERROR(error);
goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
@ -3268,7 +3270,7 @@
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
GOTO_ERROR(error);
goto error;
}
next_instr = this_instr;
}
@ -3299,7 +3301,7 @@
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
if (err) GOTO_ERROR(error);
if (err) goto error;
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -3324,7 +3326,7 @@
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
if (err) GOTO_ERROR(error);
if (err) goto error;
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -3356,7 +3358,7 @@
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, retval);
if (err) GOTO_ERROR(error);
if (err) goto error;
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
@ -4138,7 +4140,13 @@
INSTRUCTION_STATS(LOAD_FAST_CHECK);
PyObject *value;
value = GETLOCAL(oparg);
if (value == NULL) goto unbound_local_error;
if (value == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
@ -4175,14 +4183,14 @@
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
GOTO_ERROR(error);
goto error;
}
if (!value) {
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
GOTO_ERROR(error);
goto error;
}
Py_INCREF(value);
}
@ -4200,21 +4208,21 @@
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
GOTO_ERROR(error);
goto error;
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
GOTO_ERROR(error);
goto error;
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
GOTO_ERROR(error);
goto error;
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
GOTO_ERROR(error);
goto error;
}
}
}
@ -4398,21 +4406,21 @@
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
GOTO_ERROR(error);
goto error;
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
GOTO_ERROR(error);
goto error;
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
GOTO_ERROR(error);
goto error;
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
GOTO_ERROR(error);
goto error;
}
}
}
@ -4574,7 +4582,7 @@
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
GOTO_ERROR(error);
goto error;
}
SETLOCAL(oparg, cell);
DISPATCH();
@ -4591,7 +4599,7 @@
PyFunction_New(codeobj, GLOBALS());
Py_DECREF(codeobj);
if (func_obj == NULL) {
GOTO_ERROR(error);
goto error;
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
@ -4910,7 +4918,7 @@
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
GOTO_ERROR(error);
goto error;
}
}
assert(exc && PyExceptionInstance_Check(exc));
@ -5017,7 +5025,7 @@
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
GOTO_ERROR(error);
goto error;
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -5126,7 +5134,7 @@
JUMPBY(oparg);
}
else {
GOTO_ERROR(error);
goto error;
}
}
Py_DECREF(v);

37
Python/jit.c
View file

@ -381,11 +381,13 @@ int
_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length)
{
// Loop once to find the total compiled size:
size_t code_size = 0;
size_t data_size = 0;
uint32_t instruction_starts[UOP_MAX_TRACE_LENGTH];
uint32_t code_size = 0;
uint32_t data_size = 0;
for (size_t i = 0; i < length; i++) {
_PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i];
const StencilGroup *group = &stencil_groups[instruction->opcode];
instruction_starts[i] = code_size;
code_size += group->code.body_size;
data_size += group->data.body_size;
}
@ -403,11 +405,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size
// Loop again to emit the code:
unsigned char *code = memory;
unsigned char *data = memory + code_size;
unsigned char *top = code;
if (trace[0].opcode == _START_EXECUTOR) {
// Don't want to execute this more than once:
top += stencil_groups[_START_EXECUTOR].code.body_size;
}
assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT);
for (size_t i = 0; i < length; i++) {
_PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i];
const StencilGroup *group = &stencil_groups[instruction->opcode];
@ -419,8 +417,29 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size
patches[HoleValue_EXECUTOR] = (uint64_t)executor;
patches[HoleValue_OPARG] = instruction->oparg;
patches[HoleValue_OPERAND] = instruction->operand;
patches[HoleValue_TARGET] = instruction->target;
patches[HoleValue_TOP] = (uint64_t)top;
switch (instruction->format) {
case UOP_FORMAT_TARGET:
patches[HoleValue_TARGET] = instruction->target;
break;
case UOP_FORMAT_EXIT:
assert(instruction->exit_index < executor->exit_count);
patches[HoleValue_EXIT_INDEX] = instruction->exit_index;
if (instruction->error_target < length) {
patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target];
}
break;
case UOP_FORMAT_JUMP:
assert(instruction->jump_target < length);
patches[HoleValue_JUMP_TARGET] = (uint64_t)memory + instruction_starts[instruction->jump_target];
if (instruction->error_target < length) {
patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target];
}
break;
default:
assert(0);
Py_FatalError("Illegal instruction format");
}
patches[HoleValue_TOP] = (uint64_t)memory + instruction_starts[1];
patches[HoleValue_ZERO] = 0;
emit(group, patches);
code += group->code.body_size;

383
Python/optimizer.c
View file

@ -154,13 +154,19 @@ PyUnstable_GetOptimizer(void)
}
static _PyExecutorObject *
make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies);
make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies);
static int
init_cold_exit_executor(_PyExecutorObject *executor, int oparg);
/* It is impossible for the number of exits to reach 1/4 of the total length,
* as the number of exits cannot reach 1/3 of the number of non-exits, due to
* the presence of CHECK_VALIDITY checks and instructions to produce the values
* being checked in exits. */
#define COLD_EXIT_COUNT (UOP_MAX_TRACE_LENGTH/4)
static int cold_exits_initialized = 0;
static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 };
static _PyExecutorObject COLD_EXITS[COLD_EXIT_COUNT] = { 0 };
static const _PyBloomFilter EMPTY_FILTER = { 0 };
@ -172,7 +178,7 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer)
}
else if (cold_exits_initialized == 0) {
cold_exits_initialized = 1;
for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) {
for (int i = 0; i < COLD_EXIT_COUNT; i++) {
if (init_cold_exit_executor(&COLD_EXITS[i], i)) {
return NULL;
}
@ -313,10 +319,33 @@ _PyUOpPrint(const _PyUOpInstruction *uop)
else {
printf("%s", name);
}
printf(" (%d, target=%d, operand=%#" PRIx64 ")",
uop->oparg,
uop->target,
(uint64_t)uop->operand);
switch(uop->format) {
case UOP_FORMAT_TARGET:
printf(" (%d, target=%d, operand=%#" PRIx64,
uop->oparg,
uop->target,
(uint64_t)uop->operand);
break;
case UOP_FORMAT_JUMP:
printf(" (%d, jump_target=%d, operand=%#" PRIx64,
uop->oparg,
uop->jump_target,
(uint64_t)uop->operand);
break;
case UOP_FORMAT_EXIT:
printf(" (%d, exit_index=%d, operand=%#" PRIx64,
uop->oparg,
uop->exit_index,
(uint64_t)uop->operand);
break;
default:
printf(" (%d, Unknown format)", uop->oparg);
}
if (_PyUop_Flags[uop->opcode] & HAS_ERROR_FLAG) {
printf(", error_target=%d", uop->error_target);
}
printf(")");
}
#endif
@ -432,28 +461,36 @@ BRANCH_TO_GUARD[4][2] = {
#endif
// Beware: Macro arg order differs from struct member order
static inline int
add_to_trace(
_PyUOpInstruction *trace,
int trace_length,
uint16_t opcode,
uint16_t oparg,
uint64_t operand,
uint32_t target)
{
trace[trace_length].opcode = opcode;
trace[trace_length].format = UOP_FORMAT_TARGET;
trace[trace_length].target = target;
trace[trace_length].oparg = oparg;
trace[trace_length].operand = operand;
return trace_length + 1;
}
#ifdef Py_DEBUG
#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \
assert(trace_length < max_length); \
trace[trace_length].opcode = (OPCODE); \
trace[trace_length].oparg = (OPARG); \
trace[trace_length].target = (TARGET); \
trace[trace_length].operand = (OPERAND); \
trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); \
if (lltrace >= 2) { \
printf("%4d ADD_TO_TRACE: ", trace_length); \
_PyUOpPrint(&trace[trace_length]); \
_PyUOpPrint(&trace[trace_length-1]); \
printf("\n"); \
} \
trace_length++;
}
#else
#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \
assert(trace_length < max_length); \
trace[trace_length].opcode = (OPCODE); \
trace[trace_length].oparg = (OPARG); \
trace[trace_length].target = (TARGET); \
trace[trace_length].operand = (OPERAND); \
trace_length++;
trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET));
#endif
#define INSTR_IP(INSTR, CODE) \
@ -476,8 +513,7 @@ BRANCH_TO_GUARD[4][2] = {
if (trace_stack_depth >= TRACE_STACK_SIZE) { \
DPRINTF(2, "Trace stack overflow\n"); \
OPT_STAT_INC(trace_stack_overflow); \
ADD_TO_TRACE(uop, oparg, operand, target); \
ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \
trace_length = 0; \
goto done; \
} \
assert(func == NULL || func->func_code == (PyObject *)code); \
@ -495,7 +531,7 @@ BRANCH_TO_GUARD[4][2] = {
assert(func == NULL || func->func_code == (PyObject *)code); \
instr = trace_stack[trace_stack_depth].instr;
/* Returns 1 on success,
/* Returns the length of the trace on success,
* 0 if it failed to produce a worthwhile trace,
* and -1 on an error.
*/
@ -515,7 +551,8 @@ translate_bytecode_to_trace(
_Py_BloomFilter_Add(dependencies, initial_code);
_Py_CODEUNIT *initial_instr = instr;
int trace_length = 0;
int max_length = buffer_size;
// Leave space for possible trailing _EXIT_TRACE
int max_length = buffer_size-2;
struct {
PyFunctionObject *func;
PyCodeObject *code;
@ -538,13 +575,16 @@ translate_bytecode_to_trace(
PyUnicode_AsUTF8(code->co_filename),
code->co_firstlineno,
2 * INSTR_IP(initial_instr, code));
ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code));
uint32_t target = 0;
top: // Jump here after _PUSH_FRAME or likely branches
for (;;) {
target = INSTR_IP(instr, code);
RESERVE_RAW(2, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE
RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP");
ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target);
// Need space for _DEOPT
max_length--;
uint32_t opcode = instr->op.code;
uint32_t oparg = instr->op.arg;
@ -582,13 +622,22 @@ translate_bytecode_to_trace(
continue;
}
else {
if (OPCODE_HAS_DEOPT(opcode)) {
if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) {
opcode = _PyOpcode_Deopt[opcode];
}
assert(!OPCODE_HAS_EXIT(opcode));
assert(!OPCODE_HAS_DEOPT(opcode));
}
}
if (OPCODE_HAS_EXIT(opcode)) {
// Make space for exit code
max_length--;
}
if (OPCODE_HAS_ERROR(opcode)) {
// Make space for error code
max_length--;
}
switch (opcode) {
case POP_JUMP_IF_NONE:
case POP_JUMP_IF_NOT_NONE:
@ -624,10 +673,10 @@ translate_bytecode_to_trace(
DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n",
instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code));
instr = target_instr;
ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code));
ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code));
goto top;
}
ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(target_instr, code));
ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code));
break;
}
@ -849,7 +898,9 @@ translate_bytecode_to_trace(
progress_needed ? "no progress" : "too short");
return 0;
}
ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target);
if (trace[trace_length-1].opcode != _JUMP_TO_TOP) {
ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target);
}
DPRINTF(1,
"Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n",
PyUnicode_AsUTF8(code->co_qualname),
@ -857,8 +908,8 @@ translate_bytecode_to_trace(
code->co_firstlineno,
2 * INSTR_IP(initial_instr, code),
trace_length);
OPT_HIST(trace_length + buffer_size - max_length, trace_length_hist);
return 1;
OPT_HIST(trace_length, trace_length_hist);
return trace_length;
}
#undef RESERVE
@ -871,43 +922,86 @@ translate_bytecode_to_trace(
#define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31)))
#define BIT_IS_SET(array, bit) (array[(bit)>>5] & (1<<((bit)&31)))
/* Count the number of used uops, and mark them in the bit vector `used`.
* This can be done in a single pass using simple reachability analysis,
* as there are no backward jumps.
* NOPs are excluded from the count.
/* Count the number of unused uops and exits
*/
static int
compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr)
count_exits(_PyUOpInstruction *buffer, int length)
{
int count = 0;
int exit_count = 0;
SET_BIT(used, 0);
for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) {
if (!BIT_IS_SET(used, i)) {
continue;
}
count++;
for (int i = 0; i < length; i++) {
int opcode = buffer[i].opcode;
if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) {
if (opcode == _SIDE_EXIT) {
exit_count++;
}
if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) {
continue;
}
/* All other micro-ops fall through, so i+1 is reachable */
SET_BIT(used, i+1);
assert(opcode <= MAX_UOP_ID);
if (_PyUop_Flags[opcode] & HAS_JUMP_FLAG) {
/* Mark target as reachable */
SET_BIT(used, buffer[i].oparg);
}
if (opcode == NOP) {
count--;
UNSET_BIT(used, i);
}
return exit_count;
}
static void make_exit(_PyUOpInstruction *inst, int opcode, int target)
{
inst->opcode = opcode;
inst->oparg = 0;
inst->format = UOP_FORMAT_TARGET;
inst->target = target;
}
/* Convert implicit exits, errors and deopts
* into explicit ones. */
static int
prepare_for_execution(_PyUOpInstruction *buffer, int length)
{
int32_t current_jump = -1;
int32_t current_jump_target = -1;
int32_t current_error = -1;
int32_t current_error_target = -1;
int32_t current_popped = -1;
/* Leaving in NOPs slows down the interpreter and messes up the stats */
_PyUOpInstruction *copy_to = &buffer[0];
for (int i = 0; i < length; i++) {
_PyUOpInstruction *inst = &buffer[i];
if (inst->opcode != _NOP) {
if (copy_to != inst) {
*copy_to = *inst;
}
copy_to++;
}
}
*exit_count_ptr = exit_count;
return count;
length = (int)(copy_to - buffer);
int next_spare = length;
for (int i = 0; i < length; i++) {
_PyUOpInstruction *inst = &buffer[i];
int opcode = inst->opcode;
int32_t target = (int32_t)uop_get_target(inst);
if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) {
if (target != current_jump_target) {
uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? _SIDE_EXIT : _DEOPT;
make_exit(&buffer[next_spare], exit_op, target);
current_jump_target = target;
current_jump = next_spare;
next_spare++;
}
buffer[i].jump_target = current_jump;
buffer[i].format = UOP_FORMAT_JUMP;
}
if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) {
int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ?
0 : _PyUop_num_popped(opcode, inst->oparg);
if (target != current_error_target || popped != current_popped) {
current_popped = popped;
current_error = next_spare;
current_error_target = target;
make_exit(&buffer[next_spare], _ERROR_POP_N, 0);
buffer[next_spare].oparg = popped;
next_spare++;
}
buffer[i].error_target = current_error;
if (buffer[i].format == UOP_FORMAT_TARGET) {
buffer[i].format = UOP_FORMAT_JUMP;
buffer[i].jump_target = 0;
}
}
}
return next_spare;
}
/* Executor side exits */
@ -926,61 +1020,118 @@ allocate_executor(int exit_count, int length)
return res;
}
#ifdef Py_DEBUG
#define CHECK(PRED) \
if (!(PRED)) { \
printf(#PRED " at %d\n", i); \
assert(0); \
}
static int
target_unused(int opcode)
{
return (_PyUop_Flags[opcode] & (HAS_ERROR_FLAG | HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) == 0;
}
static void
sanity_check(_PyExecutorObject *executor)
{
for (uint32_t i = 0; i < executor->exit_count; i++) {
_PyExitData *exit = &executor->exits[i];
CHECK(exit->target < (1 << 25));
}
bool ended = false;
uint32_t i = 0;
CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT);
for (; i < executor->code_size; i++) {
const _PyUOpInstruction *inst = &executor->trace[i];
uint16_t opcode = inst->opcode;
CHECK(opcode <= MAX_UOP_ID);
CHECK(_PyOpcode_uop_name[opcode] != NULL);
switch(inst->format) {
case UOP_FORMAT_TARGET:
CHECK(target_unused(opcode));
break;
case UOP_FORMAT_EXIT:
CHECK(opcode == _SIDE_EXIT);
CHECK(inst->exit_index < executor->exit_count);
break;
case UOP_FORMAT_JUMP:
CHECK(inst->jump_target < executor->code_size);
break;
case UOP_FORMAT_UNUSED:
CHECK(0);
break;
}
if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) {
CHECK(inst->format == UOP_FORMAT_JUMP);
CHECK(inst->error_target < executor->code_size);
}
if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE || opcode == _COLD_EXIT) {
ended = true;
i++;
break;
}
}
CHECK(ended);
for (; i < executor->code_size; i++) {
const _PyUOpInstruction *inst = &executor->trace[i];
uint16_t opcode = inst->opcode;
CHECK(
opcode == _DEOPT ||
opcode == _SIDE_EXIT ||
opcode == _ERROR_POP_N);
if (opcode == _SIDE_EXIT) {
CHECK(inst->format == UOP_FORMAT_EXIT);
}
}
}
#undef CHECK
#endif
/* Makes an executor from a buffer of uops.
* Account for the buffer having gaps and NOPs by computing a "used"
* bit vector and only copying the used uops. Here "used" means reachable
* and not a NOP.
*/
static _PyExecutorObject *
make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies)
make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies)
{
uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 };
int exit_count;
int length = compute_used(buffer, used, &exit_count);
length += 1; // For _START_EXECUTOR
int exit_count = count_exits(buffer, length);
_PyExecutorObject *executor = allocate_executor(exit_count, length);
if (executor == NULL) {
return NULL;
}
OPT_HIST(length, optimized_trace_length_hist);
/* Initialize exits */
assert(exit_count < COLD_EXIT_COUNT);
for (int i = 0; i < exit_count; i++) {
executor->exits[i].executor = &COLD_EXITS[i];
executor->exits[i].temperature = 0;
}
int next_exit = exit_count-1;
_PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length-1];
/* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */
for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) {
if (!BIT_IS_SET(used, i)) {
continue;
}
*dest = buffer[i];
_PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length];
assert(buffer[0].opcode == _START_EXECUTOR);
buffer[0].operand = (uint64_t)executor;
for (int i = length-1; i >= 0; i--) {
int opcode = buffer[i].opcode;
if (opcode == _POP_JUMP_IF_FALSE ||
opcode == _POP_JUMP_IF_TRUE)
{
/* The oparg of the target will already have been set to its new offset */
int oparg = dest->oparg;
dest->oparg = buffer[oparg].oparg;
}
if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) {
dest--;
*dest = buffer[i];
assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE);
if (opcode == _SIDE_EXIT) {
executor->exits[next_exit].target = buffer[i].target;
dest->exit_index = next_exit;
dest->format = UOP_FORMAT_EXIT;
next_exit--;
}
/* Set the oparg to be the destination offset,
* so that we can set the oparg of earlier jumps correctly. */
buffer[i].oparg = (uint16_t)(dest - executor->trace);
dest--;
}
assert(next_exit == -1);
assert(dest == executor->trace);
dest->opcode = _START_EXECUTOR;
assert(dest->opcode == _START_EXECUTOR);
dest->oparg = 0;
dest->target = 0;
dest->operand = (uintptr_t)executor;
_Py_ExecutorInit(executor, dependencies);
#ifdef Py_DEBUG
char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
@ -996,6 +1147,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende
printf("\n");
}
}
sanity_check(executor);
#endif
#ifdef _Py_JIT
executor->jit_code = NULL;
@ -1024,6 +1176,9 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg)
for (int i = 0; i < BLOOM_FILTER_WORDS; i++) {
assert(executor->vm_data.bloom.bits[i] == 0);
}
#ifdef Py_DEBUG
sanity_check(executor);
#endif
#ifdef _Py_JIT
executor->jit_code = NULL;
executor->jit_size = 0;
@ -1034,6 +1189,28 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg)
return 0;
}
#ifdef Py_STATS
/* Returns the effective trace length.
* Ignores NOPs and trailing exit and error handling.*/
int effective_trace_length(_PyUOpInstruction *buffer, int length)
{
int nop_count = 0;
for (int i = 0; i < length; i++) {
int opcode = buffer[i].opcode;
if (opcode == _NOP) {
nop_count++;
}
if (opcode == _EXIT_TRACE ||
opcode == _JUMP_TO_TOP ||
opcode == _COLD_EXIT) {
return i+1-nop_count;
}
}
Py_FatalError("No terminating instruction");
Py_UNREACHABLE();
}
#endif
static int
uop_optimize(
_PyOptimizerObject *self,
@ -1046,24 +1223,26 @@ uop_optimize(
_Py_BloomFilter_Init(&dependencies);
_PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH];
OPT_STAT_INC(attempts);
int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies);
if (err <= 0) {
int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies);
if (length <= 0) {
// Error or nothing translated
return err;
return length;
}
assert(length < UOP_MAX_TRACE_LENGTH);
OPT_STAT_INC(traces_created);
char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE");
if (env_var == NULL || *env_var == '\0' || *env_var > '0') {
err = _Py_uop_analyze_and_optimize(frame, buffer,
UOP_MAX_TRACE_LENGTH,
length = _Py_uop_analyze_and_optimize(frame, buffer,
length,
curr_stackentries, &dependencies);
if (err <= 0) {
return err;
if (length <= 0) {
return length;
}
}
assert(err == 1);
assert(length < UOP_MAX_TRACE_LENGTH);
assert(length >= 1);
/* Fix up */
for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) {
for (int pc = 0; pc < length; pc++) {
int opcode = buffer[pc].opcode;
int oparg = buffer[pc].oparg;
if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) {
@ -1078,10 +1257,14 @@ uop_optimize(
assert(_PyOpcode_uop_name[buffer[pc].opcode]);
assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0);
}
_PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies);
OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist);
length = prepare_for_execution(buffer, length);
assert(length <= UOP_MAX_TRACE_LENGTH);
_PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies);
if (executor == NULL) {
return -1;
}
assert(length <= UOP_MAX_TRACE_LENGTH);
*exec_ptr = executor;
return 1;
}
@ -1156,12 +1339,14 @@ counter_optimize(
return 0;
}
_Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg;
_PyUOpInstruction buffer[3] = {
_PyUOpInstruction buffer[5] = {
{ .opcode = _START_EXECUTOR },
{ .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self },
{ .opcode = _INTERNAL_INCREMENT_OPT_COUNTER },
{ .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) }
{ .opcode = _EXIT_TRACE, .jump_target = 4, .format=UOP_FORMAT_JUMP },
{ .opcode = _SIDE_EXIT, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET }
};
_PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER);
_PyExecutorObject *executor = make_executor_from_uops(buffer, 5, &EMPTY_FILTER);
if (executor == NULL) {
return -1;
}

50
Python/optimizer_analysis.c
View file

@ -387,9 +387,9 @@ optimize_uops(
ctx->curr_frame_depth++;
ctx->frame = frame;
for (_PyUOpInstruction *this_instr = trace;
this_instr < trace + trace_len && !op_is_end(this_instr->opcode);
this_instr++) {
_PyUOpInstruction *this_instr = NULL;
for (int i = 0; i < trace_len; i++) {
this_instr = &trace[i];
int oparg = this_instr->oparg;
opcode = this_instr->opcode;
@ -416,9 +416,8 @@ optimize_uops(
ctx->frame->stack_pointer = stack_pointer;
assert(STACK_LEVEL() >= 0);
}
_Py_uop_abstractcontext_fini(ctx);
return 1;
return trace_len;
out_of_space:
DPRINTF(3, "\n");
@ -447,11 +446,11 @@ optimize_uops(
/* Cannot optimize further, but there would be no benefit
* in retrying later */
_Py_uop_abstractcontext_fini(ctx);
return 1;
return trace_len;
}
static void
static int
remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
{
/* Remove _SET_IP and _CHECK_VALIDITY where possible.
@ -506,7 +505,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
}
case _JUMP_TO_TOP:
case _EXIT_TRACE:
return;
return pc + 1;
default:
{
bool needs_ip = false;
@ -530,6 +529,8 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
}
}
}
Py_FatalError("No terminating instruction");
Py_UNREACHABLE();
}
static void
@ -582,43 +583,36 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s
// 0 - failure, no error raised, just fall back to Tier 1
// -1 - failure, and raise error
// 1 - optimizer success
// > 0 - length of optimized trace
int
_Py_uop_analyze_and_optimize(
_PyInterpreterFrame *frame,
_PyUOpInstruction *buffer,
int buffer_size,
int length,
int curr_stacklen,
_PyBloomFilter *dependencies
)
{
OPT_STAT_INC(optimizer_attempts);
int err = remove_globals(frame, buffer, buffer_size, dependencies);
if (err == 0) {
goto not_ready;
}
if (err < 0) {
goto error;
int err = remove_globals(frame, buffer, length, dependencies);
if (err <= 0) {
return err;
}
peephole_opt(frame, buffer, buffer_size);
peephole_opt(frame, buffer, length);
err = optimize_uops(
length = optimize_uops(
_PyFrame_GetCode(frame), buffer,
buffer_size, curr_stacklen, dependencies);
length, curr_stacklen, dependencies);
if (err == 0) {
goto not_ready;
if (length <= 0) {
return length;
}
assert(err == 1);
remove_unneeded_uops(buffer, buffer_size);
length = remove_unneeded_uops(buffer, length);
assert(length > 0);
OPT_STAT_INC(optimizer_successes);
return 1;
not_ready:
return 0;
error:
return -1;
return length;
}

57
Python/optimizer_cases.c.h generated
View file

@ -769,14 +769,7 @@
break;
}
case _LOAD_NAME: {
_Py_UopsSymbol *v;
v = sym_new_not_null(ctx);
if (v == NULL) goto out_of_space;
stack_pointer[0] = v;
stack_pointer += 1;
break;
}
/* _LOAD_NAME is not a viable micro-op for tier 2 */
case _LOAD_GLOBAL: {
_Py_UopsSymbol *res;
@ -900,14 +893,7 @@
break;
}
case _BUILD_SET: {
_Py_UopsSymbol *set;
set = sym_new_not_null(ctx);
if (set == NULL) goto out_of_space;
stack_pointer[-oparg] = set;
stack_pointer += 1 - oparg;
break;
}
/* _BUILD_SET is not a viable micro-op for tier 2 */
case _BUILD_MAP: {
_Py_UopsSymbol *map;
@ -1408,31 +1394,9 @@
/* _FOR_ITER_GEN is not a viable micro-op for tier 2 */
case _BEFORE_ASYNC_WITH: {
_Py_UopsSymbol *exit;
_Py_UopsSymbol *res;
exit = sym_new_not_null(ctx);
if (exit == NULL) goto out_of_space;
res = sym_new_not_null(ctx);
if (res == NULL) goto out_of_space;
stack_pointer[-1] = exit;
stack_pointer[0] = res;
stack_pointer += 1;
break;
}
/* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 */
case _BEFORE_WITH: {
_Py_UopsSymbol *exit;
_Py_UopsSymbol *res;
exit = sym_new_not_null(ctx);
if (exit == NULL) goto out_of_space;
res = sym_new_not_null(ctx);
if (res == NULL) goto out_of_space;
stack_pointer[-1] = exit;
stack_pointer[0] = res;
stack_pointer += 1;
break;
}
/* _BEFORE_WITH is not a viable micro-op for tier 2 */
case _WITH_EXCEPT_START: {
_Py_UopsSymbol *res;
@ -2029,3 +1993,16 @@
break;
}
case _DEOPT: {
break;
}
case _SIDE_EXIT: {
break;
}
case _ERROR_POP_N: {
stack_pointer += -oparg;
break;
}

66
Tools/cases_generator/analyzer.py
View file

@ -8,7 +8,8 @@
@dataclass
class Properties:
escapes: bool
infallible: bool
error_with_pop: bool
error_without_pop: bool
deopts: bool
oparg: bool
jumps: bool
@ -37,7 +38,8 @@ def dump(self, indent: str) -> None:
def from_list(properties: list["Properties"]) -> "Properties":
return Properties(
escapes=any(p.escapes for p in properties),
infallible=all(p.infallible for p in properties),
error_with_pop=any(p.error_with_pop for p in properties),
error_without_pop=any(p.error_without_pop for p in properties),
deopts=any(p.deopts for p in properties),
oparg=any(p.oparg for p in properties),
jumps=any(p.jumps for p in properties),
@ -55,10 +57,16 @@ def from_list(properties: list["Properties"]) -> "Properties":
passthrough=all(p.passthrough for p in properties),
)
@property
def infallible(self) -> bool:
return not self.error_with_pop and not self.error_without_pop
SKIP_PROPERTIES = Properties(
escapes=False,
infallible=True,
error_with_pop=False,
error_without_pop=False,
deopts=False,
oparg=False,
jumps=False,
@ -157,20 +165,32 @@ def size(self) -> int:
self._size = sum(c.size for c in self.caches)
return self._size
def is_viable(self) -> bool:
def why_not_viable(self) -> str | None:
if self.name == "_SAVE_RETURN_OFFSET":
return True # Adjusts next_instr, but only in tier 1 code
if self.properties.needs_this:
return False
return None # Adjusts next_instr, but only in tier 1 code
if "INSTRUMENTED" in self.name:
return False
return "is instrumented"
if "replaced" in self.annotations:
return False
return "is replaced"
if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"):
return False
return "has tier 1 control flow"
if self.properties.needs_this:
return "uses the 'this_instr' variable"
if len([c for c in self.caches if c.name != "unused"]) > 1:
return False
return True
return "has unused cache entries"
if self.properties.error_with_pop and self.properties.error_without_pop:
return "has both popping and not-popping errors"
if self.properties.eval_breaker:
if self.properties.error_with_pop or self.properties.error_without_pop:
return "has error handling and eval-breaker check"
if self.properties.side_exit:
return "exits and eval-breaker check"
if self.properties.deopts:
return "deopts and eval-breaker check"
return None
def is_viable(self) -> bool:
return self.why_not_viable() is None
def is_super(self) -> bool:
for tkn in self.body:
@ -320,10 +340,17 @@ def tier_variable(node: parser.InstDef) -> int | None:
return int(token.text[-1])
return None
def is_infallible(op: parser.InstDef) -> bool:
return not (
def has_error_with_pop(op: parser.InstDef) -> bool:
return (
variable_used(op, "ERROR_IF")
or variable_used(op, "error")
or variable_used(op, "pop_1_error")
or variable_used(op, "exception_unwind")
or variable_used(op, "resume_with_error")
)
def has_error_without_pop(op: parser.InstDef) -> bool:
return (
variable_used(op, "ERROR_NO_POP")
or variable_used(op, "pop_1_error")
or variable_used(op, "exception_unwind")
or variable_used(op, "resume_with_error")
@ -507,12 +534,15 @@ def compute_properties(op: parser.InstDef) -> Properties:
tkn.column,
op.name,
)
infallible = is_infallible(op)
error_with_pop = has_error_with_pop(op)
error_without_pop = has_error_without_pop(op)
infallible = not error_with_pop and not error_without_pop
passthrough = stack_effect_only_peeks(op) and infallible
return Properties(
escapes=makes_escaping_api_call(op),
infallible=infallible,
deopts=deopts_if or exits_if,
error_with_pop=error_with_pop,
error_without_pop=error_without_pop,
deopts=deopts_if,
side_exit=exits_if,
oparg=variable_used(op, "oparg"),
jumps=variable_used(op, "JUMPBY"),

17
Tools/cases_generator/generators_common.py
View file

@ -99,6 +99,20 @@ def replace_error(
out.emit(close)
def replace_error_no_pop(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction | None,
) -> None:
next(tkn_iter) # LPAREN
next(tkn_iter) # RPAREN
next(tkn_iter) # Semi colon
out.emit_at("goto error;", tkn)
def replace_decrefs(
out: CWriter,
tkn: Token,
@ -160,6 +174,7 @@ def replace_check_eval_breaker(
"EXIT_IF": replace_deopt,
"DEOPT_IF": replace_deopt,
"ERROR_IF": replace_error,
"ERROR_NO_POP": replace_error_no_pop,
"DECREF_INPUTS": replace_decrefs,
"CHECK_EVAL_BREAKER": replace_check_eval_breaker,
"SYNC_SP": replace_sync_sp,
@ -213,6 +228,8 @@ def cflags(p: Properties) -> str:
flags.append("HAS_EXIT_FLAG")
if not p.infallible:
flags.append("HAS_ERROR_FLAG")
if p.error_without_pop:
flags.append("HAS_ERROR_NO_POP_FLAG")
if p.escapes:
flags.append("HAS_ESCAPES_FLAG")
if p.pure:

1
Tools/cases_generator/opcode_metadata_generator.py
View file

@ -54,6 +54,7 @@
"PURE",
"PASSTHROUGH",
"OPARG_AND_1",
"ERROR_NO_POP",
]

38
Tools/cases_generator/tier2_generator.py
View file

@ -72,22 +72,22 @@ def tier2_replace_error(
label = next(tkn_iter).text
next(tkn_iter) # RPAREN
next(tkn_iter) # Semi colon
out.emit(") ")
c_offset = stack.peek_offset.to_c()
try:
offset = -int(c_offset)
close = ";\n"
except ValueError:
offset = None
out.emit(f"{{ stack_pointer += {c_offset}; ")
close = "; }\n"
out.emit("goto ")
if offset:
out.emit(f"pop_{offset}_")
out.emit(label + "_tier_two")
out.emit(close)
out.emit(") JUMP_TO_ERROR();\n")
def tier2_replace_error_no_pop(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction | None,
) -> None:
next(tkn_iter) # LPAREN
next(tkn_iter) # RPAREN
next(tkn_iter) # Semi colon
out.emit_at("JUMP_TO_ERROR();", tkn)
def tier2_replace_deopt(
out: CWriter,
tkn: Token,
@ -100,7 +100,7 @@ def tier2_replace_deopt(
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "RPAREN")
next(tkn_iter) # Semi colon
out.emit(") goto deoptimize;\n")
out.emit(") JUMP_TO_JUMP_TARGET();\n")
def tier2_replace_exit_if(
@ -115,7 +115,7 @@ def tier2_replace_exit_if(
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "RPAREN")
next(tkn_iter) # Semi colon
out.emit(") goto side_exit;\n")
out.emit(") JUMP_TO_JUMP_TARGET();\n")
def tier2_replace_oparg(
@ -141,6 +141,7 @@ def tier2_replace_oparg(
TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy()
TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error
TIER2_REPLACEMENT_FUNCTIONS["ERROR_NO_POP"] = tier2_replace_error_no_pop
TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt
TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg
TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if
@ -201,8 +202,9 @@ def generate_tier2(
continue
if uop.is_super():
continue
if not uop.is_viable():
out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n")
why_not_viable = uop.why_not_viable()
if why_not_viable is not None:
out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n")
continue
out.emit(f"case {uop.name}: {{\n")
declare_variables(uop, out)

17
Tools/cases_generator/uop_metadata_generator.py
View file

@ -15,10 +15,10 @@
write_header,
cflags,
)
from stack import Stack
from cwriter import CWriter
from typing import TextIO
DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h"
@ -26,6 +26,7 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None:
out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n")
out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n")
out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n")
out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n")
out.emit("#ifdef NEED_OPCODE_METADATA\n")
out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n")
for uop in analysis.uops.values():
@ -44,6 +45,20 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None:
if uop.is_viable() and uop.properties.tier != 1:
out.emit(f'[{uop.name}] = "{uop.name}",\n')
out.emit("};\n")
out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n")
out.emit("switch(opcode) {\n")
for uop in analysis.uops.values():
if uop.is_viable() and uop.properties.tier != 1:
stack = Stack()
for var in reversed(uop.stack.inputs):
stack.pop(var)
popped = (-stack.base_offset).to_c()
out.emit(f"case {uop.name}:\n")
out.emit(f" return {popped};\n")
out.emit("default:\n")
out.emit(" return -1;\n")
out.emit("}\n")
out.emit("}\n\n")
out.emit("#endif // NEED_OPCODE_METADATA\n\n")

6
Tools/jit/_stencils.py
View file

@ -31,6 +31,12 @@ class HoleValue(enum.Enum):
OPERAND = enum.auto()
# The current uop's target (exposed as _JIT_TARGET):
TARGET = enum.auto()
# The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET):
JUMP_TARGET = enum.auto()
# The base address of the machine code for the error jump target (exposed as _JIT_ERROR_TARGET):
ERROR_TARGET = enum.auto()
# The index of the exit to be jumped through (exposed as _JIT_EXIT_INDEX):
EXIT_INDEX = enum.auto()
# The base address of the machine code for the first uop (exposed as _JIT_TOP):
TOP = enum.auto()
# A hardcoded value of zero (used for symbol lookups):

31
Tools/jit/template.c
View file

@ -64,9 +64,17 @@ do { \
TYPE NAME = (TYPE)(uint64_t)&ALIAS;
#define PATCH_JUMP(ALIAS) \
do { \
PyAPI_DATA(void) ALIAS; \
__attribute__((musttail)) \
return ((jit_func)&ALIAS)(frame, stack_pointer, tstate);
return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \
} while (0)
#undef JUMP_TO_JUMP_TARGET
#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET)
#undef JUMP_TO_ERROR
#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET)
_Py_CODEUNIT *
_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate)
@ -79,6 +87,7 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX)
// The actual instruction definitions (only one will be used):
if (opcode == _JUMP_TO_TOP) {
CHECK_EVAL_BREAKER();
@ -91,28 +100,16 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *
}
PATCH_JUMP(_JIT_CONTINUE);
// Labels that the instruction implementations expect to exist:
unbound_local_error_tier_two:
_PyEval_FormatExcCheckArg(
tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg));
goto error_tier_two;
pop_4_error_tier_two:
STACK_SHRINK(1);
pop_3_error_tier_two:
STACK_SHRINK(1);
pop_2_error_tier_two:
STACK_SHRINK(1);
pop_1_error_tier_two:
STACK_SHRINK(1);
error_tier_two:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(NULL);
deoptimize:
exit_to_tier1:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target);
side_exit:
exit_to_trace:
{
_PyExitData *exit = &current_executor->exits[_target];
_PyExitData *exit = &current_executor->exits[_exit_index];
Py_INCREF(exit->executor);
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_TWO(exit->executor);