2015-01-28 16:08:07 +00:00
|
|
|
/*-
|
|
|
|
* Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* This software was developed by SRI International and the University of
|
|
|
|
* Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
|
|
|
|
* ("CTSRD"), as part of the DARPA CRASH research programme.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/systm.h>
|
|
|
|
#include <sys/pmc.h>
|
|
|
|
#include <sys/pmckern.h>
|
|
|
|
|
|
|
|
#include <machine/pmc_mdep.h>
|
|
|
|
#include <machine/cpu.h>
|
|
|
|
|
|
|
|
static int armv7_npmcs;
|
|
|
|
|
|
|
|
struct armv7_event_code_map {
|
|
|
|
enum pmc_event pe_ev;
|
|
|
|
uint8_t pe_code;
|
|
|
|
};
|
|
|
|
|
2017-06-13 18:51:23 +00:00
|
|
|
#define PMC_EV_CPU_CYCLES 0xFF
|
|
|
|
|
2015-01-28 16:08:07 +00:00
|
|
|
/*
|
|
|
|
* Per-processor information.
|
|
|
|
*/
|
|
|
|
struct armv7_cpu {
|
|
|
|
struct pmc_hw *pc_armv7pmcs;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct armv7_cpu **armv7_pcpu;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Interrupt Enable Set Register
|
|
|
|
*/
|
|
|
|
static __inline void
|
|
|
|
armv7_interrupt_enable(uint32_t pmc)
|
|
|
|
{
|
|
|
|
uint32_t reg;
|
|
|
|
|
|
|
|
reg = (1 << pmc);
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pminten_set(reg);
|
2015-01-28 16:08:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Interrupt Clear Set Register
|
|
|
|
*/
|
|
|
|
static __inline void
|
|
|
|
armv7_interrupt_disable(uint32_t pmc)
|
|
|
|
{
|
|
|
|
uint32_t reg;
|
|
|
|
|
|
|
|
reg = (1 << pmc);
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pminten_clr(reg);
|
2015-01-28 16:08:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Counter Set Enable Register
|
|
|
|
*/
|
|
|
|
static __inline void
|
|
|
|
armv7_counter_enable(unsigned int pmc)
|
|
|
|
{
|
|
|
|
uint32_t reg;
|
|
|
|
|
|
|
|
reg = (1 << pmc);
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmcnten_set(reg);
|
2015-01-28 16:08:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Counter Clear Enable Register
|
|
|
|
*/
|
|
|
|
static __inline void
|
|
|
|
armv7_counter_disable(unsigned int pmc)
|
|
|
|
{
|
|
|
|
uint32_t reg;
|
|
|
|
|
|
|
|
reg = (1 << pmc);
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmcnten_clr(reg);
|
2015-01-28 16:08:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Performance Count Register N
|
|
|
|
*/
|
|
|
|
static uint32_t
|
2021-01-04 13:50:17 +00:00
|
|
|
armv7_pmcn_read(unsigned int pmc, uint32_t evsel)
|
2015-01-28 16:08:07 +00:00
|
|
|
{
|
|
|
|
|
2021-01-04 13:50:17 +00:00
|
|
|
if (evsel == PMC_EV_CPU_CYCLES) {
|
|
|
|
return ((uint32_t)cp15_pmccntr_get());
|
|
|
|
}
|
|
|
|
|
2015-03-28 18:57:13 +00:00
|
|
|
KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc));
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmselr_set(pmc);
|
|
|
|
return (cp15_pmxevcntr_get());
|
2015-01-28 16:08:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static uint32_t
|
|
|
|
armv7_pmcn_write(unsigned int pmc, uint32_t reg)
|
|
|
|
{
|
|
|
|
|
2015-03-28 18:57:13 +00:00
|
|
|
KASSERT(pmc < armv7_npmcs, ("%s: illegal PMC number %d", __func__, pmc));
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmselr_set(pmc);
|
|
|
|
cp15_pmxevcntr_set(reg);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
return (reg);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
armv7_allocate_pmc(int cpu, int ri, struct pmc *pm,
|
|
|
|
const struct pmc_op_pmcallocate *a)
|
|
|
|
{
|
|
|
|
enum pmc_event pe;
|
2015-06-10 12:42:30 +00:00
|
|
|
uint32_t config;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d] illegal CPU value %d", __LINE__, cpu));
|
|
|
|
KASSERT(ri >= 0 && ri < armv7_npmcs,
|
|
|
|
("[armv7,%d] illegal row index %d", __LINE__, ri));
|
|
|
|
|
|
|
|
if (a->pm_class != PMC_CLASS_ARMV7)
|
|
|
|
return (EINVAL);
|
|
|
|
pe = a->pm_ev;
|
|
|
|
|
2015-06-10 12:42:30 +00:00
|
|
|
config = (pe & EVENT_ID_MASK);
|
2015-01-28 16:08:07 +00:00
|
|
|
pm->pm_md.pm_armv7.pm_armv7_evsel = config;
|
|
|
|
|
2015-06-10 12:42:30 +00:00
|
|
|
PMCDBG2(MDP, ALL, 2, "armv7-allocate ri=%d -> config=0x%x", ri, config);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static int
|
2023-05-05 22:00:02 +00:00
|
|
|
armv7_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
|
2015-01-28 16:08:07 +00:00
|
|
|
{
|
|
|
|
pmc_value_t tmp;
|
2021-01-04 13:50:17 +00:00
|
|
|
register_t s;
|
|
|
|
u_int reg;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d] illegal CPU value %d", __LINE__, cpu));
|
|
|
|
KASSERT(ri >= 0 && ri < armv7_npmcs,
|
|
|
|
("[armv7,%d] illegal row index %d", __LINE__, ri));
|
|
|
|
|
2021-01-04 13:50:17 +00:00
|
|
|
s = intr_disable();
|
|
|
|
tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel);
|
|
|
|
|
|
|
|
/* Check if counter has overflowed */
|
2017-06-13 18:51:23 +00:00
|
|
|
if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES)
|
2021-01-04 13:50:17 +00:00
|
|
|
reg = (1u << 31);
|
2015-01-28 16:08:07 +00:00
|
|
|
else
|
2021-01-04 13:50:17 +00:00
|
|
|
reg = (1u << ri);
|
|
|
|
|
|
|
|
if ((cp15_pmovsr_get() & reg) != 0) {
|
|
|
|
/* Clear Overflow Flag */
|
|
|
|
cp15_pmovsr_set(reg);
|
hwpmc: Fix amd/arm64/armv7/uncore sampling overflow race
If a counter more than overflows just as we read it on switch out then,
if using sampling mode, we will negate this small value to give a huge
reload count, and if we later switch back in that context we will
validate that value against pm_reloadcount and panic an INVARIANTS
kernel with:
panic: [pmc,1470] pmcval outside of expected range cpu=2 ri=16 pmcval=fffff292 pm_reloadcount=10000
or similar. Presumably in a non-INVARIANTS kernel we will instead just
use the provided value as the reload count, which would lead to the
overflow not happing for a very long time (e.g. 78 minutes for a 48-bit
counter incrementing at an averate rate of 1GHz).
Instead, clamp the reload count to 0 (which corresponds precisely to the
value we would compute if it had just overflowed and no more), which
will result in hwpmc using the full original reload count again. This is
the approach used by core for Intel (for both fixed and programmable
counters).
As part of this, armv7 and arm64 are made conceptually simpler; rather
than skipping modifying the overflow count for sampling mode counters so
it's always kept as ~0, those special cases are removed so it's always
applicable and the concatentation of it and the hardware counter can
always be viewed as a 64-bit counter, which also makes them look more
like other architectures.
Whilst here, fix an instance of UB (shifting a 1 into the sign bit) for
amd in its sign-extension code.
Reviewed by: andrew, mhorne, kib
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D33654
2022-01-10 14:30:05 +00:00
|
|
|
pm->pm_pcpu_state[cpu].pps_overflowcnt++;
|
2021-01-04 13:50:17 +00:00
|
|
|
|
|
|
|
/* Reread counter in case we raced. */
|
|
|
|
tmp = armv7_pmcn_read(ri, pm->pm_md.pm_armv7.pm_armv7_evsel);
|
|
|
|
}
|
2021-01-04 13:25:45 +00:00
|
|
|
tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt;
|
2021-01-04 13:50:17 +00:00
|
|
|
intr_restore(s);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2015-06-10 12:42:30 +00:00
|
|
|
PMCDBG2(MDP, REA, 2, "armv7-read id=%d -> %jd", ri, tmp);
|
hwpmc: Fix amd/arm64/armv7/uncore sampling overflow race
If a counter more than overflows just as we read it on switch out then,
if using sampling mode, we will negate this small value to give a huge
reload count, and if we later switch back in that context we will
validate that value against pm_reloadcount and panic an INVARIANTS
kernel with:
panic: [pmc,1470] pmcval outside of expected range cpu=2 ri=16 pmcval=fffff292 pm_reloadcount=10000
or similar. Presumably in a non-INVARIANTS kernel we will instead just
use the provided value as the reload count, which would lead to the
overflow not happing for a very long time (e.g. 78 minutes for a 48-bit
counter incrementing at an averate rate of 1GHz).
Instead, clamp the reload count to 0 (which corresponds precisely to the
value we would compute if it had just overflowed and no more), which
will result in hwpmc using the full original reload count again. This is
the approach used by core for Intel (for both fixed and programmable
counters).
As part of this, armv7 and arm64 are made conceptually simpler; rather
than skipping modifying the overflow count for sampling mode counters so
it's always kept as ~0, those special cases are removed so it's always
applicable and the concatentation of it and the hardware counter can
always be viewed as a 64-bit counter, which also makes them look more
like other architectures.
Whilst here, fix an instance of UB (shifting a 1 into the sign bit) for
amd in its sign-extension code.
Reviewed by: andrew, mhorne, kib
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D33654
2022-01-10 14:30:05 +00:00
|
|
|
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
|
|
|
|
/*
|
|
|
|
* Clamp value to 0 if the counter just overflowed,
|
|
|
|
* otherwise the returned reload count would wrap to a
|
|
|
|
* huge value.
|
|
|
|
*/
|
|
|
|
if ((tmp & (1ull << 63)) == 0)
|
|
|
|
tmp = 0;
|
|
|
|
else
|
|
|
|
tmp = ARMV7_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
|
|
|
|
}
|
|
|
|
*v = tmp;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2023-05-05 22:00:02 +00:00
|
|
|
armv7_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
|
2015-01-28 16:08:07 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d] illegal CPU value %d", __LINE__, cpu));
|
|
|
|
KASSERT(ri >= 0 && ri < armv7_npmcs,
|
|
|
|
("[armv7,%d] illegal row-index %d", __LINE__, ri));
|
|
|
|
|
|
|
|
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
|
|
|
|
v = ARMV7_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
|
|
|
|
|
2015-06-10 12:42:30 +00:00
|
|
|
PMCDBG3(MDP, WRI, 1, "armv7-write cpu=%d ri=%d v=%jx", cpu, ri, v);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2021-01-04 13:25:45 +00:00
|
|
|
pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32;
|
2017-06-13 18:51:23 +00:00
|
|
|
if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES)
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmccntr_set(v);
|
2015-01-28 16:08:07 +00:00
|
|
|
else
|
|
|
|
armv7_pmcn_write(ri, v);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
armv7_config_pmc(int cpu, int ri, struct pmc *pm)
|
|
|
|
{
|
|
|
|
struct pmc_hw *phw;
|
|
|
|
|
2015-06-10 12:42:30 +00:00
|
|
|
PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d] illegal CPU value %d", __LINE__, cpu));
|
|
|
|
KASSERT(ri >= 0 && ri < armv7_npmcs,
|
|
|
|
("[armv7,%d] illegal row-index %d", __LINE__, ri));
|
|
|
|
|
|
|
|
phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri];
|
|
|
|
|
|
|
|
KASSERT(pm == NULL || phw->phw_pmc == NULL,
|
|
|
|
("[armv7,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
|
|
|
|
__LINE__, pm, phw->phw_pmc));
|
|
|
|
|
|
|
|
phw->phw_pmc = pm;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2023-05-05 22:00:02 +00:00
|
|
|
armv7_start_pmc(int cpu, int ri, struct pmc *pm)
|
2015-01-28 16:08:07 +00:00
|
|
|
{
|
|
|
|
uint32_t config;
|
|
|
|
|
|
|
|
config = pm->pm_md.pm_armv7.pm_armv7_evsel;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Configure the event selection.
|
|
|
|
*/
|
2017-06-13 18:51:23 +00:00
|
|
|
if (config != PMC_EV_CPU_CYCLES) {
|
|
|
|
cp15_pmselr_set(ri);
|
|
|
|
cp15_pmxevtyper_set(config);
|
|
|
|
} else
|
|
|
|
ri = 31;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable the PMC.
|
|
|
|
*/
|
|
|
|
armv7_interrupt_enable(ri);
|
|
|
|
armv7_counter_enable(ri);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2023-05-05 22:00:02 +00:00
|
|
|
armv7_stop_pmc(int cpu, int ri, struct pmc *pm)
|
2015-01-28 16:08:07 +00:00
|
|
|
{
|
2017-06-13 18:51:23 +00:00
|
|
|
uint32_t config;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2017-06-13 18:51:23 +00:00
|
|
|
config = pm->pm_md.pm_armv7.pm_armv7_evsel;
|
|
|
|
if (config == PMC_EV_CPU_CYCLES)
|
|
|
|
ri = 31;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable the PMCs.
|
|
|
|
*/
|
|
|
|
armv7_counter_disable(ri);
|
|
|
|
armv7_interrupt_disable(ri);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
armv7_release_pmc(int cpu, int ri, struct pmc *pmc)
|
|
|
|
{
|
2022-04-13 23:08:22 +00:00
|
|
|
struct pmc_hw *phw __diagused;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d] illegal CPU value %d", __LINE__, cpu));
|
|
|
|
KASSERT(ri >= 0 && ri < armv7_npmcs,
|
|
|
|
("[armv7,%d] illegal row-index %d", __LINE__, ri));
|
|
|
|
|
|
|
|
phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri];
|
|
|
|
KASSERT(phw->phw_pmc == NULL,
|
|
|
|
("[armv7,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
hwpmc: simplify calling convention for hwpmc interrupt handling
pmc_process_interrupt takes 5 arguments when only 3 are needed.
cpu is always available in curcpu and inuserspace can always be
derived from the passed trapframe.
While facially a reasonable cleanup this change was motivated
by the need to workaround a compiler bug.
core2_intr(cpu, tf) ->
pmc_process_interrupt(cpu, ring, pmc, tf, inuserspace) ->
pmc_add_sample(cpu, ring, pm, tf, inuserspace)
In the process of optimizing the tail call the tf pointer was getting
clobbered:
(kgdb) up
at /storage/mmacy/devel/freebsd/sys/dev/hwpmc/hwpmc_mod.c:4709
4709 pmc_save_kernel_callchain(ps->ps_pc,
(kgdb) up
1205 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
resulting in a crash in pmc_save_kernel_callchain.
2018-06-08 04:58:03 +00:00
|
|
|
armv7_intr(struct trapframe *tf)
|
2015-01-28 16:08:07 +00:00
|
|
|
{
|
|
|
|
int retval, ri;
|
|
|
|
struct pmc *pm;
|
|
|
|
int error;
|
hwpmc: simplify calling convention for hwpmc interrupt handling
pmc_process_interrupt takes 5 arguments when only 3 are needed.
cpu is always available in curcpu and inuserspace can always be
derived from the passed trapframe.
While facially a reasonable cleanup this change was motivated
by the need to workaround a compiler bug.
core2_intr(cpu, tf) ->
pmc_process_interrupt(cpu, ring, pmc, tf, inuserspace) ->
pmc_add_sample(cpu, ring, pm, tf, inuserspace)
In the process of optimizing the tail call the tf pointer was getting
clobbered:
(kgdb) up
at /storage/mmacy/devel/freebsd/sys/dev/hwpmc/hwpmc_mod.c:4709
4709 pmc_save_kernel_callchain(ps->ps_pc,
(kgdb) up
1205 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
resulting in a crash in pmc_save_kernel_callchain.
2018-06-08 04:58:03 +00:00
|
|
|
int reg, cpu;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2019-06-06 15:18:23 +00:00
|
|
|
cpu = curcpu;
|
2015-01-28 16:08:07 +00:00
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d] CPU %d out of range", __LINE__, cpu));
|
|
|
|
|
|
|
|
retval = 0;
|
|
|
|
|
|
|
|
for (ri = 0; ri < armv7_npmcs; ri++) {
|
|
|
|
pm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc;
|
|
|
|
if (pm == NULL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* Check if counter has overflowed */
|
2017-06-13 18:51:23 +00:00
|
|
|
if (pm->pm_md.pm_armv7.pm_armv7_evsel == PMC_EV_CPU_CYCLES)
|
2021-01-04 13:50:17 +00:00
|
|
|
reg = (1u << 31);
|
2015-01-28 16:08:07 +00:00
|
|
|
else
|
2021-01-04 13:50:17 +00:00
|
|
|
reg = (1u << ri);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2015-03-28 18:57:13 +00:00
|
|
|
if ((cp15_pmovsr_get() & reg) == 0) {
|
2015-01-28 16:08:07 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Clear Overflow Flag */
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmovsr_set(reg);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
retval = 1; /* Found an interrupting PMC. */
|
2017-06-13 18:51:23 +00:00
|
|
|
|
hwpmc: Fix amd/arm64/armv7/uncore sampling overflow race
If a counter more than overflows just as we read it on switch out then,
if using sampling mode, we will negate this small value to give a huge
reload count, and if we later switch back in that context we will
validate that value against pm_reloadcount and panic an INVARIANTS
kernel with:
panic: [pmc,1470] pmcval outside of expected range cpu=2 ri=16 pmcval=fffff292 pm_reloadcount=10000
or similar. Presumably in a non-INVARIANTS kernel we will instead just
use the provided value as the reload count, which would lead to the
overflow not happing for a very long time (e.g. 78 minutes for a 48-bit
counter incrementing at an averate rate of 1GHz).
Instead, clamp the reload count to 0 (which corresponds precisely to the
value we would compute if it had just overflowed and no more), which
will result in hwpmc using the full original reload count again. This is
the approach used by core for Intel (for both fixed and programmable
counters).
As part of this, armv7 and arm64 are made conceptually simpler; rather
than skipping modifying the overflow count for sampling mode counters so
it's always kept as ~0, those special cases are removed so it's always
applicable and the concatentation of it and the hardware counter can
always be viewed as a 64-bit counter, which also makes them look more
like other architectures.
Whilst here, fix an instance of UB (shifting a 1 into the sign bit) for
amd in its sign-extension code.
Reviewed by: andrew, mhorne, kib
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D33654
2022-01-10 14:30:05 +00:00
|
|
|
pm->pm_pcpu_state[cpu].pps_overflowcnt += 1;
|
|
|
|
|
|
|
|
if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
|
2017-06-13 18:51:23 +00:00
|
|
|
continue;
|
hwpmc: Fix amd/arm64/armv7/uncore sampling overflow race
If a counter more than overflows just as we read it on switch out then,
if using sampling mode, we will negate this small value to give a huge
reload count, and if we later switch back in that context we will
validate that value against pm_reloadcount and panic an INVARIANTS
kernel with:
panic: [pmc,1470] pmcval outside of expected range cpu=2 ri=16 pmcval=fffff292 pm_reloadcount=10000
or similar. Presumably in a non-INVARIANTS kernel we will instead just
use the provided value as the reload count, which would lead to the
overflow not happing for a very long time (e.g. 78 minutes for a 48-bit
counter incrementing at an averate rate of 1GHz).
Instead, clamp the reload count to 0 (which corresponds precisely to the
value we would compute if it had just overflowed and no more), which
will result in hwpmc using the full original reload count again. This is
the approach used by core for Intel (for both fixed and programmable
counters).
As part of this, armv7 and arm64 are made conceptually simpler; rather
than skipping modifying the overflow count for sampling mode counters so
it's always kept as ~0, those special cases are removed so it's always
applicable and the concatentation of it and the hardware counter can
always be viewed as a 64-bit counter, which also makes them look more
like other architectures.
Whilst here, fix an instance of UB (shifting a 1 into the sign bit) for
amd in its sign-extension code.
Reviewed by: andrew, mhorne, kib
MFC after: 1 week
Differential Revision: https://reviews.freebsd.org/D33654
2022-01-10 14:30:05 +00:00
|
|
|
|
2015-01-28 16:08:07 +00:00
|
|
|
if (pm->pm_state != PMC_STATE_RUNNING)
|
|
|
|
continue;
|
|
|
|
|
hwpmc: simplify calling convention for hwpmc interrupt handling
pmc_process_interrupt takes 5 arguments when only 3 are needed.
cpu is always available in curcpu and inuserspace can always be
derived from the passed trapframe.
While facially a reasonable cleanup this change was motivated
by the need to workaround a compiler bug.
core2_intr(cpu, tf) ->
pmc_process_interrupt(cpu, ring, pmc, tf, inuserspace) ->
pmc_add_sample(cpu, ring, pm, tf, inuserspace)
In the process of optimizing the tail call the tf pointer was getting
clobbered:
(kgdb) up
at /storage/mmacy/devel/freebsd/sys/dev/hwpmc/hwpmc_mod.c:4709
4709 pmc_save_kernel_callchain(ps->ps_pc,
(kgdb) up
1205 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
resulting in a crash in pmc_save_kernel_callchain.
2018-06-08 04:58:03 +00:00
|
|
|
error = pmc_process_interrupt(PMC_HR, pm, tf);
|
2015-01-28 16:08:07 +00:00
|
|
|
if (error)
|
2023-05-05 22:00:02 +00:00
|
|
|
armv7_stop_pmc(cpu, ri, pm);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
/* Reload sampling count */
|
2023-05-05 22:00:02 +00:00
|
|
|
armv7_write_pmc(cpu, ri, pm, pm->pm_sc.pm_reloadcount);
|
2015-01-28 16:08:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return (retval);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
armv7_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
|
|
|
|
{
|
|
|
|
struct pmc_hw *phw;
|
|
|
|
|
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d], illegal CPU %d", __LINE__, cpu));
|
|
|
|
KASSERT(ri >= 0 && ri < armv7_npmcs,
|
|
|
|
("[armv7,%d] row-index %d out of range", __LINE__, ri));
|
|
|
|
|
|
|
|
phw = &armv7_pcpu[cpu]->pc_armv7pmcs[ri];
|
2023-05-05 21:59:15 +00:00
|
|
|
|
|
|
|
snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV7-%d", ri);
|
2015-01-28 16:08:07 +00:00
|
|
|
pi->pm_class = PMC_CLASS_ARMV7;
|
2023-05-05 21:59:15 +00:00
|
|
|
|
2015-01-28 16:08:07 +00:00
|
|
|
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
|
|
|
|
pi->pm_enabled = TRUE;
|
|
|
|
*ppmc = phw->phw_pmc;
|
|
|
|
} else {
|
|
|
|
pi->pm_enabled = FALSE;
|
|
|
|
*ppmc = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
armv7_get_config(int cpu, int ri, struct pmc **ppm)
|
|
|
|
{
|
|
|
|
|
|
|
|
*ppm = armv7_pcpu[cpu]->pc_armv7pmcs[ri].phw_pmc;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
armv7_pcpu_init(struct pmc_mdep *md, int cpu)
|
|
|
|
{
|
|
|
|
struct armv7_cpu *pac;
|
|
|
|
struct pmc_hw *phw;
|
|
|
|
struct pmc_cpu *pc;
|
|
|
|
uint32_t pmnc;
|
|
|
|
int first_ri;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
|
|
|
|
("[armv7,%d] wrong cpu number %d", __LINE__, cpu));
|
2023-08-03 14:10:26 +00:00
|
|
|
PMCDBG0(MDP, INI, 1, "armv7-pcpu-init");
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
armv7_pcpu[cpu] = pac = malloc(sizeof(struct armv7_cpu), M_PMC,
|
|
|
|
M_WAITOK|M_ZERO);
|
|
|
|
|
|
|
|
pac->pc_armv7pmcs = malloc(sizeof(struct pmc_hw) * armv7_npmcs,
|
|
|
|
M_PMC, M_WAITOK|M_ZERO);
|
|
|
|
pc = pmc_pcpu[cpu];
|
|
|
|
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7].pcd_ri;
|
|
|
|
KASSERT(pc != NULL, ("[armv7,%d] NULL per-cpu pointer", __LINE__));
|
|
|
|
|
|
|
|
for (i = 0, phw = pac->pc_armv7pmcs; i < armv7_npmcs; i++, phw++) {
|
|
|
|
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
|
|
|
|
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
|
|
|
|
phw->phw_pmc = NULL;
|
|
|
|
pc->pc_hwpmcs[i + first_ri] = phw;
|
|
|
|
}
|
|
|
|
|
2017-06-13 18:51:23 +00:00
|
|
|
pmnc = 0xffffffff;
|
|
|
|
cp15_pmcnten_clr(pmnc);
|
|
|
|
cp15_pminten_clr(pmnc);
|
|
|
|
cp15_pmovsr_set(pmnc);
|
|
|
|
|
2015-01-28 16:08:07 +00:00
|
|
|
/* Enable unit */
|
2015-03-28 18:57:13 +00:00
|
|
|
pmnc = cp15_pmcr_get();
|
2015-01-28 16:08:07 +00:00
|
|
|
pmnc |= ARMV7_PMNC_ENABLE;
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmcr_set(pmnc);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
armv7_pcpu_fini(struct pmc_mdep *md, int cpu)
|
|
|
|
{
|
|
|
|
uint32_t pmnc;
|
|
|
|
|
2023-08-03 14:10:26 +00:00
|
|
|
PMCDBG0(MDP, INI, 1, "armv7-pcpu-fini");
|
|
|
|
|
2015-03-28 18:57:13 +00:00
|
|
|
pmnc = cp15_pmcr_get();
|
2015-01-28 16:08:07 +00:00
|
|
|
pmnc &= ~ARMV7_PMNC_ENABLE;
|
2015-03-28 18:57:13 +00:00
|
|
|
cp15_pmcr_set(pmnc);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2017-06-13 18:51:23 +00:00
|
|
|
pmnc = 0xffffffff;
|
|
|
|
cp15_pmcnten_clr(pmnc);
|
|
|
|
cp15_pminten_clr(pmnc);
|
|
|
|
cp15_pmovsr_set(pmnc);
|
|
|
|
|
2023-08-03 14:10:26 +00:00
|
|
|
free(armv7_pcpu[cpu]->pc_armv7pmcs, M_PMC);
|
|
|
|
free(armv7_pcpu[cpu], M_PMC);
|
|
|
|
armv7_pcpu[cpu] = NULL;
|
|
|
|
|
2015-01-28 16:08:07 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct pmc_mdep *
|
2022-08-11 12:02:33 +00:00
|
|
|
pmc_armv7_initialize(void)
|
2015-01-28 16:08:07 +00:00
|
|
|
{
|
|
|
|
struct pmc_mdep *pmc_mdep;
|
|
|
|
struct pmc_classdep *pcd;
|
2015-06-10 12:42:30 +00:00
|
|
|
int idcode;
|
2015-01-28 16:08:07 +00:00
|
|
|
int reg;
|
|
|
|
|
2015-03-28 18:57:13 +00:00
|
|
|
reg = cp15_pmcr_get();
|
2015-01-28 16:08:07 +00:00
|
|
|
armv7_npmcs = (reg >> ARMV7_PMNC_N_SHIFT) & \
|
|
|
|
ARMV7_PMNC_N_MASK;
|
2015-06-10 12:42:30 +00:00
|
|
|
idcode = (reg & ARMV7_IDCODE_MASK) >> ARMV7_IDCODE_SHIFT;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2015-06-10 12:42:30 +00:00
|
|
|
PMCDBG1(MDP, INI, 1, "armv7-init npmcs=%d", armv7_npmcs);
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate space for pointers to PMC HW descriptors and for
|
|
|
|
* the MDEP structure used by MI code.
|
|
|
|
*/
|
|
|
|
armv7_pcpu = malloc(sizeof(struct armv7_cpu *) * pmc_cpu_max(),
|
|
|
|
M_PMC, M_WAITOK | M_ZERO);
|
|
|
|
|
|
|
|
/* Just one class */
|
|
|
|
pmc_mdep = pmc_mdep_alloc(1);
|
2015-06-10 12:42:30 +00:00
|
|
|
|
|
|
|
switch (idcode) {
|
|
|
|
case ARMV7_IDCODE_CORTEX_A9:
|
|
|
|
pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A9;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
case ARMV7_IDCODE_CORTEX_A8:
|
|
|
|
/*
|
|
|
|
* On A8 we implemented common events only,
|
|
|
|
* so use it for the rest of machines.
|
|
|
|
*/
|
|
|
|
pmc_mdep->pmd_cputype = PMC_CPU_ARMV7_CORTEX_A8;
|
|
|
|
break;
|
|
|
|
}
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV7];
|
|
|
|
pcd->pcd_caps = ARMV7_PMC_CAPS;
|
|
|
|
pcd->pcd_class = PMC_CLASS_ARMV7;
|
|
|
|
pcd->pcd_num = armv7_npmcs;
|
|
|
|
pcd->pcd_ri = pmc_mdep->pmd_npmc;
|
|
|
|
pcd->pcd_width = 32;
|
|
|
|
|
|
|
|
pcd->pcd_allocate_pmc = armv7_allocate_pmc;
|
|
|
|
pcd->pcd_config_pmc = armv7_config_pmc;
|
|
|
|
pcd->pcd_pcpu_fini = armv7_pcpu_fini;
|
|
|
|
pcd->pcd_pcpu_init = armv7_pcpu_init;
|
|
|
|
pcd->pcd_describe = armv7_describe;
|
|
|
|
pcd->pcd_get_config = armv7_get_config;
|
|
|
|
pcd->pcd_read_pmc = armv7_read_pmc;
|
|
|
|
pcd->pcd_release_pmc = armv7_release_pmc;
|
|
|
|
pcd->pcd_start_pmc = armv7_start_pmc;
|
|
|
|
pcd->pcd_stop_pmc = armv7_stop_pmc;
|
|
|
|
pcd->pcd_write_pmc = armv7_write_pmc;
|
|
|
|
|
2023-05-05 21:59:41 +00:00
|
|
|
pmc_mdep->pmd_intr = armv7_intr;
|
|
|
|
pmc_mdep->pmd_npmc += armv7_npmcs;
|
2015-01-28 16:08:07 +00:00
|
|
|
|
|
|
|
return (pmc_mdep);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
pmc_armv7_finalize(struct pmc_mdep *md)
|
|
|
|
{
|
2023-08-03 14:10:26 +00:00
|
|
|
PMCDBG0(MDP, INI, 1, "armv7-finalize");
|
2015-01-28 16:08:07 +00:00
|
|
|
|
2023-10-18 17:50:49 +00:00
|
|
|
for (int i = 0; i < pmc_cpu_max(); i++)
|
|
|
|
KASSERT(armv7_pcpu[i] == NULL,
|
|
|
|
("[armv7,%d] non-null pcpu cpu %d", __LINE__, i));
|
|
|
|
|
2023-08-03 14:10:26 +00:00
|
|
|
free(armv7_pcpu, M_PMC);
|
2015-01-28 16:08:07 +00:00
|
|
|
}
|