[3.12] GH-108390: Prevent non-local events being set with sys.monitoring.set_local_events() (GH-108420) (#108899)

* GH-108390: Prevent non-local events being set with `sys.monitoring.set_local_events()` (GH-108420)

* Restore generated objects

* Restore size of monitoring arrays in code object for 3.12 ABI compatibility.

* Update ABI file
This commit is contained in:
Mark Shannon 2023-09-05 12:29:38 +01:00 committed by GitHub
parent 5121faabd1
commit 7ee021f999
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 3101 additions and 3001 deletions

File diff suppressed because it is too large Load diff

View file

@ -8,16 +8,23 @@
extern "C" {
#endif
/* Count of all local monitoring events */
#define _PY_MONITORING_LOCAL_EVENTS 10
/* Count of all "real" monitoring events (not derived from other events) */
#define _PY_MONITORING_UNGROUPED_EVENTS 15
/* Count of all monitoring events */
#define _PY_MONITORING_EVENTS 17
/* Table of which tools are active for each monitored event. */
typedef struct _Py_Monitors {
/* Tables of which tools are active for each monitored event. */
/* For 3.12 ABI compatibility this is over sized */
typedef struct _Py_LocalMonitors {
/* Only _PY_MONITORING_LOCAL_EVENTS of these are used */
uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS];
} _Py_Monitors;
} _Py_LocalMonitors;
typedef struct _Py_GlobalMonitors {
uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS];
} _Py_GlobalMonitors;
/* Each instruction in a code object is a fixed-width value,
* currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG
@ -81,9 +88,9 @@ typedef struct {
*/
typedef struct {
/* Monitoring specific to this code object */
_Py_Monitors local_monitors;
_Py_LocalMonitors local_monitors;
/* Monitoring that is active on this code object */
_Py_Monitors active_monitors;
_Py_LocalMonitors active_monitors;
/* The tools that are to be notified for events for the matching code unit */
uint8_t *tools;
/* Information to support line events */

View file

@ -29,7 +29,7 @@ extern "C" {
#define PY_MONITORING_EVENT_STOP_ITERATION 9
#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \
((ev) <= PY_MONITORING_EVENT_STOP_ITERATION)
((ev) < _PY_MONITORING_LOCAL_EVENTS)
/* Other events, mainly exceptions */

View file

@ -182,7 +182,7 @@ struct _is {
struct callable_cache callable_cache;
PyCodeObject *interpreter_trampoline;
_Py_Monitors monitors;
_Py_GlobalMonitors monitors;
bool f_opcode_trace_set;
bool sys_profile_initialized;
bool sys_trace_initialized;

View file

@ -1218,9 +1218,11 @@ def test_instruction_then_line(self):
self.check_events(self.func2,
recorders = recorders, must_include = self.MUST_INCLUDE_CI)
LOCAL_RECORDERS = CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder
class TestLocalEvents(MonitoringTestBase, unittest.TestCase):
def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)):
def check_events(self, func, expected, tool=TEST_TOOL, recorders=()):
try:
self.assertEqual(sys.monitoring._all_events(), {})
event_list = []
@ -1248,7 +1250,7 @@ def func1():
line2 = 2
line3 = 3
self.check_events(func1, recorders = MANY_RECORDERS, expected = [
self.check_events(func1, recorders = LOCAL_RECORDERS, expected = [
('line', 'func1', 1),
('line', 'func1', 2),
('line', 'func1', 3)])
@ -1260,7 +1262,7 @@ def func2():
[].append(2)
line3 = 3
self.check_events(func2, recorders = MANY_RECORDERS, expected = [
self.check_events(func2, recorders = LOCAL_RECORDERS, expected = [
('line', 'func2', 1),
('line', 'func2', 2),
('call', 'append', [2]),
@ -1277,15 +1279,17 @@ def func3():
line = 5
line = 6
self.check_events(func3, recorders = MANY_RECORDERS, expected = [
self.check_events(func3, recorders = LOCAL_RECORDERS, expected = [
('line', 'func3', 1),
('line', 'func3', 2),
('line', 'func3', 3),
('raise', KeyError),
('line', 'func3', 4),
('line', 'func3', 5),
('line', 'func3', 6)])
def test_set_non_local_event(self):
with self.assertRaises(ValueError):
sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.RAISE)
def line_from_offset(code, offset):
for start, end, line in code.co_lines():
@ -1698,3 +1702,19 @@ def run():
self.assertEqual(caught, "inner")
finally:
sys.monitoring.set_events(TEST_TOOL, 0)
def test_108390(self):
class Foo:
def __init__(self, set_event):
if set_event:
sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME)
def make_foo_optimized_then_set_event():
for i in range(100):
Foo(i == 99)
try:
make_foo_optimized_then_set_event()
finally:
sys.monitoring.set_events(TEST_TOOL, 0)

View file

@ -0,0 +1,4 @@
Raise an exception when setting a non-local event (``RAISE``, ``EXCEPTION_HANDLED``,
etc.) in ``sys.monitoring.set_local_events``.
Fixes crash when tracing in recursive calls to Python classes.

View file

@ -2021,28 +2021,30 @@ do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame,
return err;
}
static inline int
no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event)
static inline bool
no_tools_for_global_event(PyThreadState *tstate, int event)
{
return tstate->interp->monitors.tools[event] == 0;
}
static inline bool
no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event)
{
assert(event < _PY_MONITORING_LOCAL_EVENTS);
_PyCoMonitoringData *data = frame->f_code->_co_monitoring;
if (data) {
if (data->active_monitors.tools[event] == 0) {
return 1;
}
return data->active_monitors.tools[event] == 0;
}
else {
if (tstate->interp->monitors.tools[event] == 0) {
return 1;
}
return no_tools_for_global_event(tstate, event);
}
return 0;
}
static void
monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame,
_Py_CODEUNIT *instr)
{
if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) {
if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) {
return;
}
do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE);
@ -2052,7 +2054,7 @@ static void
monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame,
_Py_CODEUNIT *instr)
{
if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) {
if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) {
return;
}
do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
@ -2062,7 +2064,7 @@ static int
monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame,
_Py_CODEUNIT *instr)
{
if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
return 0;
}
return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION);
@ -2073,7 +2075,7 @@ monitor_unwind(PyThreadState *tstate,
_PyInterpreterFrame *frame,
_Py_CODEUNIT *instr)
{
if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) {
if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) {
return;
}
do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND);
@ -2085,7 +2087,7 @@ monitor_handled(PyThreadState *tstate,
_PyInterpreterFrame *frame,
_Py_CODEUNIT *instr, PyObject *exc)
{
if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
return 0;
}
return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc);
@ -2096,7 +2098,7 @@ monitor_throw(PyThreadState *tstate,
_PyInterpreterFrame *frame,
_Py_CODEUNIT *instr)
{
if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) {
if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) {
return;
}
do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW);

View file

@ -1,5 +1,6 @@
#include "Python.h"
#include "pycore_call.h"
#include "pycore_frame.h"
@ -136,9 +137,9 @@ is_instrumented(int opcode)
#ifndef NDEBUG
static inline bool
monitors_equals(_Py_Monitors a, _Py_Monitors b)
monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
if (a.tools[i] != b.tools[i]) {
return false;
}
@ -147,42 +148,47 @@ monitors_equals(_Py_Monitors a, _Py_Monitors b)
}
#endif
static inline _Py_Monitors
monitors_sub(_Py_Monitors a, _Py_Monitors b)
static inline _Py_LocalMonitors
monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
_Py_Monitors res;
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
res.tools[i] = a.tools[i] & ~b.tools[i];
}
return res;
}
#ifndef NDEBUG
static inline _Py_Monitors
monitors_and(_Py_Monitors a, _Py_Monitors b)
static inline _Py_LocalMonitors
monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
_Py_Monitors res;
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
res.tools[i] = a.tools[i] & b.tools[i];
}
return res;
}
#endif
static inline _Py_Monitors
monitors_or(_Py_Monitors a, _Py_Monitors b)
/* The union of the *local* events in a and b.
* Global events like RAISE are ignored.
* Used for instrumentation, as only local
* events get instrumented.
*/
static inline _Py_LocalMonitors
local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b)
{
_Py_Monitors res;
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
res.tools[i] = a.tools[i] | b.tools[i];
}
return res;
}
static inline bool
monitors_are_empty(_Py_Monitors m)
monitors_are_empty(_Py_LocalMonitors m)
{
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
if (m.tools[i]) {
return false;
}
@ -191,9 +197,9 @@ monitors_are_empty(_Py_Monitors m)
}
static inline bool
multiple_tools(_Py_Monitors *m)
multiple_tools(_Py_LocalMonitors *m)
{
for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) {
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
if (_Py_popcount32(m->tools[i]) > 1) {
return true;
}
@ -202,7 +208,19 @@ multiple_tools(_Py_Monitors *m)
}
static inline _PyMonitoringEventSet
get_events(_Py_Monitors *m, int tool_id)
get_local_events(_Py_LocalMonitors *m, int tool_id)
{
_PyMonitoringEventSet result = 0;
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
if ((m->tools[e] >> tool_id) & 1) {
result |= (1 << e);
}
}
return result;
}
static inline _PyMonitoringEventSet
get_events(_Py_GlobalMonitors *m, int tool_id)
{
_PyMonitoringEventSet result = 0;
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
@ -449,10 +467,10 @@ sanity_check_instrumentation(PyCodeObject *code)
if (data == NULL) {
return;
}
_Py_Monitors active_monitors = PyInterpreterState_Get()->monitors;
_Py_GlobalMonitors active_monitors = _PyInterpreterState_GET()->monitors;
if (code->_co_monitoring) {
_Py_Monitors local_monitors = code->_co_monitoring->local_monitors;
active_monitors = monitors_or(active_monitors, local_monitors);
active_monitors = local_union(active_monitors, local_monitors);
}
assert(monitors_equals(
code->_co_monitoring->active_monitors,
@ -879,7 +897,7 @@ is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp)
static bool
instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)
{
_Py_Monitors expected = monitors_or(
_Py_LocalMonitors expected = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
return monitors_equals(code->_co_monitoring->active_monitors, expected);
@ -887,26 +905,29 @@ instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)
#endif
static inline uint8_t
get_tools_for_instruction(PyCodeObject * code, int i, int event)
get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, int event)
{
uint8_t tools;
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code));
_PyCoMonitoringData *monitoring = code->_co_monitoring;
if (event >= _PY_MONITORING_UNGROUPED_EVENTS) {
assert(event == PY_MONITORING_EVENT_C_RAISE ||
event == PY_MONITORING_EVENT_C_RETURN);
event = PY_MONITORING_EVENT_CALL;
}
if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event) && monitoring->tools) {
tools = monitoring->tools[i];
if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
CHECK(is_version_up_to_date(code, interp));
CHECK(instrumentation_cross_checks(interp, code));
if (code->_co_monitoring->tools) {
tools = code->_co_monitoring->tools[i];
}
else {
tools = code->_co_monitoring->active_monitors.tools[event];
}
}
else {
tools = code->_co_monitoring->active_monitors.tools[event];
tools = interp->monitors.tools[event];
}
CHECK(tools_is_subset_for_event(code, event, tools));
CHECK((tools & code->_co_monitoring->active_monitors.tools[event]) == tools);
return tools;
}
@ -956,11 +977,11 @@ call_instrumentation_vector(
}
assert(args[2] == NULL);
args[2] = offset_obj;
uint8_t tools = get_tools_for_instruction(code, offset, event);
PyInterpreterState *interp = tstate->interp;
uint8_t tools = get_tools_for_instruction(code, interp, offset, event);
Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
PyObject **callargs = &args[1];
int err = 0;
PyInterpreterState *interp = tstate->interp;
while (tools) {
int tool = most_significant_bit(tools);
assert(tool >= 0 && tool < 8);
@ -1397,7 +1418,7 @@ initialize_lines(PyCodeObject *code)
}
static void
initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events)
initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events)
{
uint8_t *line_tools = code->_co_monitoring->line_tools;
assert(line_tools != NULL);
@ -1417,8 +1438,8 @@ allocate_instrumentation_data(PyCodeObject *code)
PyErr_NoMemory();
return -1;
}
code->_co_monitoring->local_monitors = (_Py_Monitors){ 0 };
code->_co_monitoring->active_monitors = (_Py_Monitors){ 0 };
code->_co_monitoring->local_monitors = (_Py_LocalMonitors){ 0 };
code->_co_monitoring->active_monitors = (_Py_LocalMonitors){ 0 };
code->_co_monitoring->tools = NULL;
code->_co_monitoring->lines = NULL;
code->_co_monitoring->line_tools = NULL;
@ -1435,7 +1456,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
if (allocate_instrumentation_data(code)) {
return -1;
}
_Py_Monitors all_events = monitors_or(
_Py_LocalMonitors all_events = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
bool multitools = multiple_tools(&all_events);
@ -1518,14 +1539,23 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
return 0;
}
int code_len = (int)Py_SIZE(code);
/* code->_co_firsttraceable >= code_len indicates
* that no instrumentation can be inserted.
* Exit early to avoid creating instrumentation
* data for potential statically allocated code
* objects.
* See https://github.com/python/cpython/issues/108390 */
if (code->_co_firsttraceable >= code_len) {
return 0;
}
if (update_instrumentation_data(code, interp)) {
return -1;
}
_Py_Monitors active_events = monitors_or(
_Py_LocalMonitors active_events = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
_Py_Monitors new_events;
_Py_Monitors removed_events;
_Py_LocalMonitors new_events;
_Py_LocalMonitors removed_events;
bool restarted = interp->last_restart_version > code->_co_instrumentation_version;
if (restarted) {
@ -1669,10 +1699,22 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) {
}
static void
set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events)
set_events(_Py_GlobalMonitors *m, int tool_id, _PyMonitoringEventSet events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
uint8_t *tools = &m->tools[e];
int active = (events >> e) & 1;
*tools &= ~(1 << tool_id);
*tools |= (active << tool_id);
}
}
static void
set_local_events(_Py_LocalMonitors *m, int tool_id, _PyMonitoringEventSet events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
uint8_t *tools = &m->tools[e];
int val = (events >> e) & 1;
*tools &= ~(1 << tool_id);
@ -1715,19 +1757,23 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
PyInterpreterState *interp = _PyInterpreterState_Get();
assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS));
assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS));
if (code->_co_firsttraceable >= Py_SIZE(code)) {
PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name);
return -1;
}
if (check_tool(interp, tool_id)) {
return -1;
}
if (allocate_instrumentation_data(code)) {
return -1;
}
_Py_Monitors *local = &code->_co_monitoring->local_monitors;
uint32_t existing_events = get_events(local, tool_id);
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
uint32_t existing_events = get_local_events(local, tool_id);
if (existing_events == events) {
return 0;
}
set_events(local, tool_id, events);
set_local_events(local, tool_id, events);
if (is_version_up_to_date(code, interp)) {
/* Force instrumentation update */
code->_co_instrumentation_version = UINT64_MAX;
@ -1886,7 +1932,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id)
if (check_valid_tool(tool_id)) {
return -1;
}
_Py_Monitors *m = &_PyInterpreterState_Get()->monitors;
_Py_GlobalMonitors *m = &_PyInterpreterState_GET()->monitors;
_PyMonitoringEventSet event_set = get_events(m, tool_id);
return event_set;
}
@ -1949,7 +1995,7 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id,
_PyMonitoringEventSet event_set = 0;
_PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring;
if (data != NULL) {
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
if ((data->local_monitors.tools[e] >> tool_id) & 1) {
event_set |= (1 << e);
}
@ -1983,15 +2029,16 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id,
if (check_valid_tool(tool_id)) {
return NULL;
}
if (event_set < 0 || event_set >= (1 << _PY_MONITORING_EVENTS)) {
PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set);
return NULL;
}
if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) {
PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently");
return NULL;
}
event_set &= ~C_RETURN_EVENTS;
if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) {
PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set);
return NULL;
}
if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) {
return NULL;
}