mirror of
https://github.com/freebsd/freebsd-src
synced 2024-10-15 12:54:27 +00:00
736 lines
19 KiB
C
736 lines
19 KiB
C
|
/*-
|
||
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
||
|
|
*
|
||
|
|
* Copyright (c) 2021 Andrew Turner
|
||
|
|
*
|
||
|
|
* This work was supported by Innovate UK project 105694, "Digital Security
|
||
|
|
* by Design (DSbD) Technology Platform Prototype".
|
||
|
|
*
|
||
|
|
* Redistribution and use in source and binary forms, with or without
|
||
|
|
* modification, are permitted provided that the following conditions
|
||
|
|
* are met:
|
||
|
|
* 1. Redistributions of source code must retain the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer.
|
||
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer in the
|
||
|
|
* documentation and/or other materials provided with the distribution.
|
||
|
|
*
|
||
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||
|
|
* SUCH DAMAGE.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <sys/cdefs.h>
|
||
|
|
#include <sys/types.h>
|
||
|
|
#include <sys/proc.h>
|
||
|
|
|
||
|
|
#include <machine/armreg.h>
|
||
|
|
|
||
|
|
#include "arm64.h"
|
||
|
|
#include "hyp.h"
|
||
|
|
|
||
|
|
struct hypctx;
|
||
|
|
|
||
|
|
uint64_t vmm_hyp_enter(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t,
|
||
|
|
uint64_t, uint64_t, uint64_t);
|
||
|
|
uint64_t vmm_enter_guest(struct hypctx *);
|
||
|
|
|
||
|
|
static void
|
||
|
|
vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
|
||
|
|
{
|
||
|
|
uint64_t dfr0;
|
||
|
|
|
||
|
|
/* Store the guest VFP registers */
|
||
|
|
if (guest) {
|
||
|
|
/* Store the timer registers */
|
||
|
|
hypctx->vtimer_cpu.cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
|
||
|
|
hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 =
|
||
|
|
READ_SPECIALREG(cntv_cval_el0);
|
||
|
|
hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 =
|
||
|
|
READ_SPECIALREG(cntv_ctl_el0);
|
||
|
|
|
||
|
|
/* Store the GICv3 registers */
|
||
|
|
hypctx->vgic_v3_regs.ich_eisr_el2 =
|
||
|
|
READ_SPECIALREG(ich_eisr_el2);
|
||
|
|
hypctx->vgic_v3_regs.ich_elrsr_el2 =
|
||
|
|
READ_SPECIALREG(ich_elrsr_el2);
|
||
|
|
hypctx->vgic_v3_regs.ich_hcr_el2 =
|
||
|
|
READ_SPECIALREG(ich_hcr_el2);
|
||
|
|
hypctx->vgic_v3_regs.ich_misr_el2 =
|
||
|
|
READ_SPECIALREG(ich_misr_el2);
|
||
|
|
hypctx->vgic_v3_regs.ich_vmcr_el2 =
|
||
|
|
READ_SPECIALREG(ich_vmcr_el2);
|
||
|
|
switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
|
||
|
|
#define STORE_LR(x) \
|
||
|
|
case x: \
|
||
|
|
hypctx->vgic_v3_regs.ich_lr_el2[x] = \
|
||
|
|
READ_SPECIALREG(ich_lr ## x ##_el2)
|
||
|
|
STORE_LR(15);
|
||
|
|
STORE_LR(14);
|
||
|
|
STORE_LR(13);
|
||
|
|
STORE_LR(12);
|
||
|
|
STORE_LR(11);
|
||
|
|
STORE_LR(10);
|
||
|
|
STORE_LR(9);
|
||
|
|
STORE_LR(8);
|
||
|
|
STORE_LR(7);
|
||
|
|
STORE_LR(6);
|
||
|
|
STORE_LR(5);
|
||
|
|
STORE_LR(4);
|
||
|
|
STORE_LR(3);
|
||
|
|
STORE_LR(2);
|
||
|
|
STORE_LR(1);
|
||
|
|
default:
|
||
|
|
STORE_LR(0);
|
||
|
|
#undef STORE_LR
|
||
|
|
}
|
||
|
|
|
||
|
|
switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
|
||
|
|
#define STORE_APR(x) \
|
||
|
|
case x: \
|
||
|
|
hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
|
||
|
|
READ_SPECIALREG(ich_ap0r ## x ##_el2); \
|
||
|
|
hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
|
||
|
|
READ_SPECIALREG(ich_ap1r ## x ##_el2)
|
||
|
|
STORE_APR(3);
|
||
|
|
STORE_APR(2);
|
||
|
|
STORE_APR(1);
|
||
|
|
default:
|
||
|
|
STORE_APR(0);
|
||
|
|
#undef STORE_APR
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
|
||
|
|
switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
|
||
|
|
#define STORE_DBG_BRP(x) \
|
||
|
|
case x: \
|
||
|
|
hypctx->dbgbcr_el1[x] = \
|
||
|
|
READ_SPECIALREG(dbgbcr ## x ## _el1); \
|
||
|
|
hypctx->dbgbvr_el1[x] = \
|
||
|
|
READ_SPECIALREG(dbgbvr ## x ## _el1)
|
||
|
|
STORE_DBG_BRP(15);
|
||
|
|
STORE_DBG_BRP(14);
|
||
|
|
STORE_DBG_BRP(13);
|
||
|
|
STORE_DBG_BRP(12);
|
||
|
|
STORE_DBG_BRP(11);
|
||
|
|
STORE_DBG_BRP(10);
|
||
|
|
STORE_DBG_BRP(9);
|
||
|
|
STORE_DBG_BRP(8);
|
||
|
|
STORE_DBG_BRP(7);
|
||
|
|
STORE_DBG_BRP(6);
|
||
|
|
STORE_DBG_BRP(5);
|
||
|
|
STORE_DBG_BRP(4);
|
||
|
|
STORE_DBG_BRP(3);
|
||
|
|
STORE_DBG_BRP(2);
|
||
|
|
STORE_DBG_BRP(1);
|
||
|
|
default:
|
||
|
|
STORE_DBG_BRP(0);
|
||
|
|
#undef STORE_DBG_BRP
|
||
|
|
}
|
||
|
|
|
||
|
|
switch (ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
|
||
|
|
#define STORE_DBG_WRP(x) \
|
||
|
|
case x: \
|
||
|
|
hypctx->dbgwcr_el1[x] = \
|
||
|
|
READ_SPECIALREG(dbgwcr ## x ## _el1); \
|
||
|
|
hypctx->dbgwvr_el1[x] = \
|
||
|
|
READ_SPECIALREG(dbgwvr ## x ## _el1)
|
||
|
|
STORE_DBG_WRP(15);
|
||
|
|
STORE_DBG_WRP(14);
|
||
|
|
STORE_DBG_WRP(13);
|
||
|
|
STORE_DBG_WRP(12);
|
||
|
|
STORE_DBG_WRP(11);
|
||
|
|
STORE_DBG_WRP(10);
|
||
|
|
STORE_DBG_WRP(9);
|
||
|
|
STORE_DBG_WRP(8);
|
||
|
|
STORE_DBG_WRP(7);
|
||
|
|
STORE_DBG_WRP(6);
|
||
|
|
STORE_DBG_WRP(5);
|
||
|
|
STORE_DBG_WRP(4);
|
||
|
|
STORE_DBG_WRP(3);
|
||
|
|
STORE_DBG_WRP(2);
|
||
|
|
STORE_DBG_WRP(1);
|
||
|
|
default:
|
||
|
|
STORE_DBG_WRP(0);
|
||
|
|
#undef STORE_DBG_WRP
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Store the PMU registers */
|
||
|
|
hypctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0);
|
||
|
|
hypctx->pmccntr_el0 = READ_SPECIALREG(pmccntr_el0);
|
||
|
|
hypctx->pmccfiltr_el0 = READ_SPECIALREG(pmccfiltr_el0);
|
||
|
|
hypctx->pmcntenset_el0 = READ_SPECIALREG(pmcntenset_el0);
|
||
|
|
hypctx->pmintenset_el1 = READ_SPECIALREG(pmintenset_el1);
|
||
|
|
hypctx->pmovsset_el0 = READ_SPECIALREG(pmovsset_el0);
|
||
|
|
hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0);
|
||
|
|
switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
|
||
|
|
#define STORE_PMU(x) \
|
||
|
|
case (x + 1): \
|
||
|
|
hypctx->pmevcntr_el0[x] = \
|
||
|
|
READ_SPECIALREG(pmevcntr ## x ## _el0); \
|
||
|
|
hypctx->pmevtyper_el0[x] = \
|
||
|
|
READ_SPECIALREG(pmevtyper ## x ## _el0)
|
||
|
|
STORE_PMU(30);
|
||
|
|
STORE_PMU(29);
|
||
|
|
STORE_PMU(28);
|
||
|
|
STORE_PMU(27);
|
||
|
|
STORE_PMU(26);
|
||
|
|
STORE_PMU(25);
|
||
|
|
STORE_PMU(24);
|
||
|
|
STORE_PMU(23);
|
||
|
|
STORE_PMU(22);
|
||
|
|
STORE_PMU(21);
|
||
|
|
STORE_PMU(20);
|
||
|
|
STORE_PMU(19);
|
||
|
|
STORE_PMU(18);
|
||
|
|
STORE_PMU(17);
|
||
|
|
STORE_PMU(16);
|
||
|
|
STORE_PMU(15);
|
||
|
|
STORE_PMU(14);
|
||
|
|
STORE_PMU(13);
|
||
|
|
STORE_PMU(12);
|
||
|
|
STORE_PMU(11);
|
||
|
|
STORE_PMU(10);
|
||
|
|
STORE_PMU(9);
|
||
|
|
STORE_PMU(8);
|
||
|
|
STORE_PMU(7);
|
||
|
|
STORE_PMU(6);
|
||
|
|
STORE_PMU(5);
|
||
|
|
STORE_PMU(4);
|
||
|
|
STORE_PMU(3);
|
||
|
|
STORE_PMU(2);
|
||
|
|
STORE_PMU(1);
|
||
|
|
STORE_PMU(0);
|
||
|
|
default: /* N == 0 when only PMCCNTR_EL0 is available */
|
||
|
|
break;
|
||
|
|
#undef STORE_PMU
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Store the special to from the trapframe */
|
||
|
|
hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1);
|
||
|
|
hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2);
|
||
|
|
hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2);
|
||
|
|
if (guest) {
|
||
|
|
hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2);
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Store the guest special registers */
|
||
|
|
hypctx->elr_el1 = READ_SPECIALREG(elr_el1);
|
||
|
|
hypctx->sp_el0 = READ_SPECIALREG(sp_el0);
|
||
|
|
hypctx->tpidr_el0 = READ_SPECIALREG(tpidr_el0);
|
||
|
|
hypctx->tpidrro_el0 = READ_SPECIALREG(tpidrro_el0);
|
||
|
|
hypctx->tpidr_el1 = READ_SPECIALREG(tpidr_el1);
|
||
|
|
hypctx->vbar_el1 = READ_SPECIALREG(vbar_el1);
|
||
|
|
|
||
|
|
hypctx->actlr_el1 = READ_SPECIALREG(actlr_el1);
|
||
|
|
hypctx->afsr0_el1 = READ_SPECIALREG(afsr0_el1);
|
||
|
|
hypctx->afsr1_el1 = READ_SPECIALREG(afsr1_el1);
|
||
|
|
hypctx->amair_el1 = READ_SPECIALREG(amair_el1);
|
||
|
|
hypctx->contextidr_el1 = READ_SPECIALREG(contextidr_el1);
|
||
|
|
hypctx->cpacr_el1 = READ_SPECIALREG(cpacr_el1);
|
||
|
|
hypctx->csselr_el1 = READ_SPECIALREG(csselr_el1);
|
||
|
|
hypctx->esr_el1 = READ_SPECIALREG(esr_el1);
|
||
|
|
hypctx->far_el1 = READ_SPECIALREG(far_el1);
|
||
|
|
hypctx->mair_el1 = READ_SPECIALREG(mair_el1);
|
||
|
|
hypctx->mdccint_el1 = READ_SPECIALREG(mdccint_el1);
|
||
|
|
hypctx->mdscr_el1 = READ_SPECIALREG(mdscr_el1);
|
||
|
|
hypctx->par_el1 = READ_SPECIALREG(par_el1);
|
||
|
|
hypctx->sctlr_el1 = READ_SPECIALREG(sctlr_el1);
|
||
|
|
hypctx->spsr_el1 = READ_SPECIALREG(spsr_el1);
|
||
|
|
hypctx->tcr_el1 = READ_SPECIALREG(tcr_el1);
|
||
|
|
/* TODO: Support when this is not res0 */
|
||
|
|
hypctx->tcr2_el1 = 0;
|
||
|
|
hypctx->ttbr0_el1 = READ_SPECIALREG(ttbr0_el1);
|
||
|
|
hypctx->ttbr1_el1 = READ_SPECIALREG(ttbr1_el1);
|
||
|
|
|
||
|
|
hypctx->cptr_el2 = READ_SPECIALREG(cptr_el2);
|
||
|
|
hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2);
|
||
|
|
hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2);
|
||
|
|
hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void
|
||
|
|
vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
|
||
|
|
{
|
||
|
|
uint64_t dfr0;
|
||
|
|
|
||
|
|
/* Restore the special registers */
|
||
|
|
WRITE_SPECIALREG(elr_el1, hypctx->elr_el1);
|
||
|
|
WRITE_SPECIALREG(sp_el0, hypctx->sp_el0);
|
||
|
|
WRITE_SPECIALREG(tpidr_el0, hypctx->tpidr_el0);
|
||
|
|
WRITE_SPECIALREG(tpidrro_el0, hypctx->tpidrro_el0);
|
||
|
|
WRITE_SPECIALREG(tpidr_el1, hypctx->tpidr_el1);
|
||
|
|
WRITE_SPECIALREG(vbar_el1, hypctx->vbar_el1);
|
||
|
|
|
||
|
|
WRITE_SPECIALREG(actlr_el1, hypctx->actlr_el1);
|
||
|
|
WRITE_SPECIALREG(afsr0_el1, hypctx->afsr0_el1);
|
||
|
|
WRITE_SPECIALREG(afsr1_el1, hypctx->afsr1_el1);
|
||
|
|
WRITE_SPECIALREG(amair_el1, hypctx->amair_el1);
|
||
|
|
WRITE_SPECIALREG(contextidr_el1, hypctx->contextidr_el1);
|
||
|
|
WRITE_SPECIALREG(cpacr_el1, hypctx->cpacr_el1);
|
||
|
|
WRITE_SPECIALREG(csselr_el1, hypctx->csselr_el1);
|
||
|
|
WRITE_SPECIALREG(esr_el1, hypctx->esr_el1);
|
||
|
|
WRITE_SPECIALREG(far_el1, hypctx->far_el1);
|
||
|
|
WRITE_SPECIALREG(mdccint_el1, hypctx->mdccint_el1);
|
||
|
|
WRITE_SPECIALREG(mdscr_el1, hypctx->mdscr_el1);
|
||
|
|
WRITE_SPECIALREG(mair_el1, hypctx->mair_el1);
|
||
|
|
WRITE_SPECIALREG(par_el1, hypctx->par_el1);
|
||
|
|
WRITE_SPECIALREG(sctlr_el1, hypctx->sctlr_el1);
|
||
|
|
WRITE_SPECIALREG(tcr_el1, hypctx->tcr_el1);
|
||
|
|
/* TODO: tcr2_el1 */
|
||
|
|
WRITE_SPECIALREG(ttbr0_el1, hypctx->ttbr0_el1);
|
||
|
|
WRITE_SPECIALREG(ttbr1_el1, hypctx->ttbr1_el1);
|
||
|
|
WRITE_SPECIALREG(spsr_el1, hypctx->spsr_el1);
|
||
|
|
|
||
|
|
WRITE_SPECIALREG(cptr_el2, hypctx->cptr_el2);
|
||
|
|
WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2);
|
||
|
|
WRITE_SPECIALREG(vpidr_el2, hypctx->vpidr_el2);
|
||
|
|
WRITE_SPECIALREG(vmpidr_el2, hypctx->vmpidr_el2);
|
||
|
|
|
||
|
|
/* Load the special regs from the trapframe */
|
||
|
|
WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp);
|
||
|
|
WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr);
|
||
|
|
WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr);
|
||
|
|
|
||
|
|
/* Restore the PMU registers */
|
||
|
|
WRITE_SPECIALREG(pmcr_el0, hypctx->pmcr_el0);
|
||
|
|
WRITE_SPECIALREG(pmccntr_el0, hypctx->pmccntr_el0);
|
||
|
|
WRITE_SPECIALREG(pmccfiltr_el0, hypctx->pmccfiltr_el0);
|
||
|
|
/* Clear all events/interrupts then enable them */
|
||
|
|
WRITE_SPECIALREG(pmcntenclr_el0, 0xfffffffful);
|
||
|
|
WRITE_SPECIALREG(pmcntenset_el0, hypctx->pmcntenset_el0);
|
||
|
|
WRITE_SPECIALREG(pmintenclr_el1, 0xfffffffful);
|
||
|
|
WRITE_SPECIALREG(pmintenset_el1, hypctx->pmintenset_el1);
|
||
|
|
WRITE_SPECIALREG(pmovsclr_el0, 0xfffffffful);
|
||
|
|
WRITE_SPECIALREG(pmovsset_el0, hypctx->pmovsset_el0);
|
||
|
|
|
||
|
|
switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
|
||
|
|
#define LOAD_PMU(x) \
|
||
|
|
case (x + 1): \
|
||
|
|
WRITE_SPECIALREG(pmevcntr ## x ## _el0, \
|
||
|
|
hypctx->pmevcntr_el0[x]); \
|
||
|
|
WRITE_SPECIALREG(pmevtyper ## x ## _el0, \
|
||
|
|
hypctx->pmevtyper_el0[x])
|
||
|
|
LOAD_PMU(30);
|
||
|
|
LOAD_PMU(29);
|
||
|
|
LOAD_PMU(28);
|
||
|
|
LOAD_PMU(27);
|
||
|
|
LOAD_PMU(26);
|
||
|
|
LOAD_PMU(25);
|
||
|
|
LOAD_PMU(24);
|
||
|
|
LOAD_PMU(23);
|
||
|
|
LOAD_PMU(22);
|
||
|
|
LOAD_PMU(21);
|
||
|
|
LOAD_PMU(20);
|
||
|
|
LOAD_PMU(19);
|
||
|
|
LOAD_PMU(18);
|
||
|
|
LOAD_PMU(17);
|
||
|
|
LOAD_PMU(16);
|
||
|
|
LOAD_PMU(15);
|
||
|
|
LOAD_PMU(14);
|
||
|
|
LOAD_PMU(13);
|
||
|
|
LOAD_PMU(12);
|
||
|
|
LOAD_PMU(11);
|
||
|
|
LOAD_PMU(10);
|
||
|
|
LOAD_PMU(9);
|
||
|
|
LOAD_PMU(8);
|
||
|
|
LOAD_PMU(7);
|
||
|
|
LOAD_PMU(6);
|
||
|
|
LOAD_PMU(5);
|
||
|
|
LOAD_PMU(4);
|
||
|
|
LOAD_PMU(3);
|
||
|
|
LOAD_PMU(2);
|
||
|
|
LOAD_PMU(1);
|
||
|
|
LOAD_PMU(0);
|
||
|
|
default: /* N == 0 when only PMCCNTR_EL0 is available */
|
||
|
|
break;
|
||
|
|
#undef LOAD_PMU
|
||
|
|
}
|
||
|
|
|
||
|
|
dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
|
||
|
|
switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
|
||
|
|
#define LOAD_DBG_BRP(x) \
|
||
|
|
case x: \
|
||
|
|
WRITE_SPECIALREG(dbgbcr ## x ## _el1, \
|
||
|
|
hypctx->dbgbcr_el1[x]); \
|
||
|
|
WRITE_SPECIALREG(dbgbvr ## x ## _el1, \
|
||
|
|
hypctx->dbgbvr_el1[x])
|
||
|
|
LOAD_DBG_BRP(15);
|
||
|
|
LOAD_DBG_BRP(14);
|
||
|
|
LOAD_DBG_BRP(13);
|
||
|
|
LOAD_DBG_BRP(12);
|
||
|
|
LOAD_DBG_BRP(11);
|
||
|
|
LOAD_DBG_BRP(10);
|
||
|
|
LOAD_DBG_BRP(9);
|
||
|
|
LOAD_DBG_BRP(8);
|
||
|
|
LOAD_DBG_BRP(7);
|
||
|
|
LOAD_DBG_BRP(6);
|
||
|
|
LOAD_DBG_BRP(5);
|
||
|
|
LOAD_DBG_BRP(4);
|
||
|
|
LOAD_DBG_BRP(3);
|
||
|
|
LOAD_DBG_BRP(2);
|
||
|
|
LOAD_DBG_BRP(1);
|
||
|
|
default:
|
||
|
|
LOAD_DBG_BRP(0);
|
||
|
|
#undef LOAD_DBG_BRP
|
||
|
|
}
|
||
|
|
|
||
|
|
switch (ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
|
||
|
|
#define LOAD_DBG_WRP(x) \
|
||
|
|
case x: \
|
||
|
|
WRITE_SPECIALREG(dbgwcr ## x ## _el1, \
|
||
|
|
hypctx->dbgwcr_el1[x]); \
|
||
|
|
WRITE_SPECIALREG(dbgwvr ## x ## _el1, \
|
||
|
|
hypctx->dbgwvr_el1[x])
|
||
|
|
LOAD_DBG_WRP(15);
|
||
|
|
LOAD_DBG_WRP(14);
|
||
|
|
LOAD_DBG_WRP(13);
|
||
|
|
LOAD_DBG_WRP(12);
|
||
|
|
LOAD_DBG_WRP(11);
|
||
|
|
LOAD_DBG_WRP(10);
|
||
|
|
LOAD_DBG_WRP(9);
|
||
|
|
LOAD_DBG_WRP(8);
|
||
|
|
LOAD_DBG_WRP(7);
|
||
|
|
LOAD_DBG_WRP(6);
|
||
|
|
LOAD_DBG_WRP(5);
|
||
|
|
LOAD_DBG_WRP(4);
|
||
|
|
LOAD_DBG_WRP(3);
|
||
|
|
LOAD_DBG_WRP(2);
|
||
|
|
LOAD_DBG_WRP(1);
|
||
|
|
default:
|
||
|
|
LOAD_DBG_WRP(0);
|
||
|
|
#undef LOAD_DBG_WRP
|
||
|
|
}
|
||
|
|
|
||
|
|
if (guest) {
|
||
|
|
/* Load the timer registers */
|
||
|
|
WRITE_SPECIALREG(cntkctl_el1, hypctx->vtimer_cpu.cntkctl_el1);
|
||
|
|
WRITE_SPECIALREG(cntv_cval_el0,
|
||
|
|
hypctx->vtimer_cpu.virt_timer.cntx_cval_el0);
|
||
|
|
WRITE_SPECIALREG(cntv_ctl_el0,
|
||
|
|
hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0);
|
||
|
|
WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2);
|
||
|
|
WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2);
|
||
|
|
|
||
|
|
/* Load the GICv3 registers */
|
||
|
|
WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
|
||
|
|
WRITE_SPECIALREG(ich_vmcr_el2,
|
||
|
|
hypctx->vgic_v3_regs.ich_vmcr_el2);
|
||
|
|
switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
|
||
|
|
#define LOAD_LR(x) \
|
||
|
|
case x: \
|
||
|
|
WRITE_SPECIALREG(ich_lr ## x ##_el2, \
|
||
|
|
hypctx->vgic_v3_regs.ich_lr_el2[x])
|
||
|
|
LOAD_LR(15);
|
||
|
|
LOAD_LR(14);
|
||
|
|
LOAD_LR(13);
|
||
|
|
LOAD_LR(12);
|
||
|
|
LOAD_LR(11);
|
||
|
|
LOAD_LR(10);
|
||
|
|
LOAD_LR(9);
|
||
|
|
LOAD_LR(8);
|
||
|
|
LOAD_LR(7);
|
||
|
|
LOAD_LR(6);
|
||
|
|
LOAD_LR(5);
|
||
|
|
LOAD_LR(4);
|
||
|
|
LOAD_LR(3);
|
||
|
|
LOAD_LR(2);
|
||
|
|
LOAD_LR(1);
|
||
|
|
default:
|
||
|
|
LOAD_LR(0);
|
||
|
|
#undef LOAD_LR
|
||
|
|
}
|
||
|
|
|
||
|
|
switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
|
||
|
|
#define LOAD_APR(x) \
|
||
|
|
case x: \
|
||
|
|
WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
|
||
|
|
hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
|
||
|
|
WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
|
||
|
|
hypctx->vgic_v3_regs.ich_ap1r_el2[x])
|
||
|
|
LOAD_APR(3);
|
||
|
|
LOAD_APR(2);
|
||
|
|
LOAD_APR(1);
|
||
|
|
default:
|
||
|
|
LOAD_APR(0);
|
||
|
|
#undef LOAD_APR
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static uint64_t
|
||
|
|
vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
|
||
|
|
{
|
||
|
|
struct hypctx host_hypctx;
|
||
|
|
uint64_t cntvoff_el2;
|
||
|
|
uint64_t ich_hcr_el2, ich_vmcr_el2, cnthctl_el2, cntkctl_el1;
|
||
|
|
uint64_t ret;
|
||
|
|
uint64_t s1e1r, hpfar_el2;
|
||
|
|
bool hpfar_valid;
|
||
|
|
|
||
|
|
vmm_hyp_reg_store(&host_hypctx, NULL, false);
|
||
|
|
|
||
|
|
/* Save the host special registers */
|
||
|
|
cnthctl_el2 = READ_SPECIALREG(cnthctl_el2);
|
||
|
|
cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
|
||
|
|
cntvoff_el2 = READ_SPECIALREG(cntvoff_el2);
|
||
|
|
|
||
|
|
ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
|
||
|
|
ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
|
||
|
|
|
||
|
|
vmm_hyp_reg_restore(hypctx, hyp, true);
|
||
|
|
|
||
|
|
/* Load the common hypervisor registers */
|
||
|
|
WRITE_SPECIALREG(vttbr_el2, hyp->vttbr_el2);
|
||
|
|
|
||
|
|
host_hypctx.mdcr_el2 = READ_SPECIALREG(mdcr_el2);
|
||
|
|
WRITE_SPECIALREG(mdcr_el2, hypctx->mdcr_el2);
|
||
|
|
|
||
|
|
/* Call into the guest */
|
||
|
|
ret = vmm_enter_guest(hypctx);
|
||
|
|
|
||
|
|
WRITE_SPECIALREG(mdcr_el2, host_hypctx.mdcr_el2);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
/* Store the exit info */
|
||
|
|
hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2);
|
||
|
|
vmm_hyp_reg_store(hypctx, hyp, true);
|
||
|
|
|
||
|
|
hpfar_valid = true;
|
||
|
|
if (ret == EXCP_TYPE_EL1_SYNC) {
|
||
|
|
switch (ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) {
|
||
|
|
case EXCP_INSN_ABORT_L:
|
||
|
|
case EXCP_DATA_ABORT_L:
|
||
|
|
/*
|
||
|
|
* The hpfar_el2 register is valid for:
|
||
|
|
* - Translation and Access faults.
|
||
|
|
* - Translation, Access, and permission faults on
|
||
|
|
* the translation table walk on the stage 1 tables.
|
||
|
|
* - A stage 2 Address size fault.
|
||
|
|
*
|
||
|
|
* As we only need it in the first 2 cases we can just
|
||
|
|
* exclude it on permission faults that are not from
|
||
|
|
* the stage 1 table walk.
|
||
|
|
*
|
||
|
|
* TODO: Add a case for Arm erratum 834220.
|
||
|
|
*/
|
||
|
|
if ((hypctx->tf.tf_esr & ISS_DATA_S1PTW) != 0)
|
||
|
|
break;
|
||
|
|
switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
|
||
|
|
case ISS_DATA_DFSC_PF_L1:
|
||
|
|
case ISS_DATA_DFSC_PF_L2:
|
||
|
|
case ISS_DATA_DFSC_PF_L3:
|
||
|
|
hpfar_valid = false;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (hpfar_valid) {
|
||
|
|
hypctx->exit_info.hpfar_el2 = READ_SPECIALREG(hpfar_el2);
|
||
|
|
} else {
|
||
|
|
/*
|
||
|
|
* TODO: There is a risk the at instruction could cause an
|
||
|
|
* exception here. We should handle it & return a failure.
|
||
|
|
*/
|
||
|
|
s1e1r =
|
||
|
|
arm64_address_translate_s1e1r(hypctx->exit_info.far_el2);
|
||
|
|
if (PAR_SUCCESS(s1e1r)) {
|
||
|
|
hpfar_el2 = (s1e1r & PAR_PA_MASK) >> PAR_PA_SHIFT;
|
||
|
|
hpfar_el2 <<= HPFAR_EL2_FIPA_SHIFT;
|
||
|
|
hypctx->exit_info.hpfar_el2 = hpfar_el2;
|
||
|
|
} else {
|
||
|
|
ret = EXCP_TYPE_REENTER;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
vmm_hyp_reg_restore(&host_hypctx, NULL, false);
|
||
|
|
|
||
|
|
/* Restore the host special registers */
|
||
|
|
WRITE_SPECIALREG(ich_hcr_el2, ich_hcr_el2);
|
||
|
|
WRITE_SPECIALREG(ich_vmcr_el2, ich_vmcr_el2);
|
||
|
|
|
||
|
|
WRITE_SPECIALREG(cnthctl_el2, cnthctl_el2);
|
||
|
|
WRITE_SPECIALREG(cntkctl_el1, cntkctl_el1);
|
||
|
|
WRITE_SPECIALREG(cntvoff_el2, cntvoff_el2);
|
||
|
|
|
||
|
|
return (ret);
|
||
|
|
}
|
||
|
|
|
||
|
|
static uint64_t
|
||
|
|
vmm_hyp_read_reg(uint64_t reg)
|
||
|
|
{
|
||
|
|
switch (reg) {
|
||
|
|
case HYP_REG_ICH_VTR:
|
||
|
|
return (READ_SPECIALREG(ich_vtr_el2));
|
||
|
|
case HYP_REG_CNTHCTL:
|
||
|
|
return (READ_SPECIALREG(cnthctl_el2));
|
||
|
|
}
|
||
|
|
|
||
|
|
return (0);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
vmm_clean_s2_tlbi(void)
|
||
|
|
{
|
||
|
|
dsb(ishst);
|
||
|
|
__asm __volatile("tlbi alle1is");
|
||
|
|
dsb(ish);
|
||
|
|
|
||
|
|
return (0);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
vm_s2_tlbi_range(uint64_t vttbr, vm_offset_t sva, vm_size_t eva,
|
||
|
|
bool final_only)
|
||
|
|
{
|
||
|
|
uint64_t end, r, start;
|
||
|
|
uint64_t host_vttbr;
|
||
|
|
|
||
|
|
#define TLBI_VA_SHIFT 12
|
||
|
|
#define TLBI_VA_MASK ((1ul << 44) - 1)
|
||
|
|
#define TLBI_VA(addr) (((addr) >> TLBI_VA_SHIFT) & TLBI_VA_MASK)
|
||
|
|
#define TLBI_VA_L3_INCR (L3_SIZE >> TLBI_VA_SHIFT)
|
||
|
|
|
||
|
|
/* Switch to the guest vttbr */
|
||
|
|
/* TODO: Handle Cortex-A57/A72 erratum 131936 */
|
||
|
|
host_vttbr = READ_SPECIALREG(vttbr_el2);
|
||
|
|
WRITE_SPECIALREG(vttbr_el2, vttbr);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
/*
|
||
|
|
* The CPU can cache the stage 1 + 2 combination so we need to ensure
|
||
|
|
* the stage 2 is invalidated first, then when this has completed we
|
||
|
|
* invalidate the stage 1 TLB. As we don't know which stage 1 virtual
|
||
|
|
* addresses point at the stage 2 IPA we need to invalidate the entire
|
||
|
|
* stage 1 TLB.
|
||
|
|
*/
|
||
|
|
|
||
|
|
start = TLBI_VA(sva);
|
||
|
|
end = TLBI_VA(eva);
|
||
|
|
for (r = start; r < end; r += TLBI_VA_L3_INCR) {
|
||
|
|
/* Invalidate the stage 2 TLB entry */
|
||
|
|
if (final_only)
|
||
|
|
__asm __volatile("tlbi ipas2le1is, %0" : : "r"(r));
|
||
|
|
else
|
||
|
|
__asm __volatile("tlbi ipas2e1is, %0" : : "r"(r));
|
||
|
|
}
|
||
|
|
/* Ensure the entry has been invalidated */
|
||
|
|
dsb(ish);
|
||
|
|
/* Invalidate the stage 1 TLB. */
|
||
|
|
__asm __volatile("tlbi vmalle1is");
|
||
|
|
dsb(ish);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
/* Switch back t othe host vttbr */
|
||
|
|
WRITE_SPECIALREG(vttbr_el2, host_vttbr);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
return (0);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
vm_s2_tlbi_all(uint64_t vttbr)
|
||
|
|
{
|
||
|
|
uint64_t host_vttbr;
|
||
|
|
|
||
|
|
/* Switch to the guest vttbr */
|
||
|
|
/* TODO: Handle Cortex-A57/A72 erratum 131936 */
|
||
|
|
host_vttbr = READ_SPECIALREG(vttbr_el2);
|
||
|
|
WRITE_SPECIALREG(vttbr_el2, vttbr);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
__asm __volatile("tlbi vmalls12e1is");
|
||
|
|
dsb(ish);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
/* Switch back t othe host vttbr */
|
||
|
|
WRITE_SPECIALREG(vttbr_el2, host_vttbr);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
return (0);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
vmm_dc_civac(uint64_t start, uint64_t len)
|
||
|
|
{
|
||
|
|
size_t line_size, end;
|
||
|
|
uint64_t ctr;
|
||
|
|
|
||
|
|
ctr = READ_SPECIALREG(ctr_el0);
|
||
|
|
line_size = sizeof(int) << CTR_DLINE_SIZE(ctr);
|
||
|
|
end = start + len;
|
||
|
|
dsb(ishst);
|
||
|
|
/* Clean and Invalidate the D-cache */
|
||
|
|
for (; start < end; start += line_size)
|
||
|
|
__asm __volatile("dc civac, %0" :: "r" (start) : "memory");
|
||
|
|
dsb(ish);
|
||
|
|
return (0);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
vmm_el2_tlbi(uint64_t type, uint64_t start, uint64_t len)
|
||
|
|
{
|
||
|
|
uint64_t end, r;
|
||
|
|
|
||
|
|
dsb(ishst);
|
||
|
|
switch (type) {
|
||
|
|
default:
|
||
|
|
case HYP_EL2_TLBI_ALL:
|
||
|
|
__asm __volatile("tlbi alle2" ::: "memory");
|
||
|
|
break;
|
||
|
|
case HYP_EL2_TLBI_VA:
|
||
|
|
end = TLBI_VA(start + len);
|
||
|
|
start = TLBI_VA(start);
|
||
|
|
for (r = start; r < end; r += TLBI_VA_L3_INCR) {
|
||
|
|
__asm __volatile("tlbi vae2is, %0" :: "r"(r));
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
dsb(ish);
|
||
|
|
|
||
|
|
return (0);
|
||
|
|
}
|
||
|
|
|
||
|
|
uint64_t
|
||
|
|
vmm_hyp_enter(uint64_t handle, uint64_t x1, uint64_t x2, uint64_t x3,
|
||
|
|
uint64_t x4, uint64_t x5, uint64_t x6, uint64_t x7)
|
||
|
|
{
|
||
|
|
uint64_t ret;
|
||
|
|
|
||
|
|
switch (handle) {
|
||
|
|
case HYP_ENTER_GUEST:
|
||
|
|
do {
|
||
|
|
ret = vmm_hyp_call_guest((struct hyp *)x1,
|
||
|
|
(struct hypctx *)x2);
|
||
|
|
} while (ret == EXCP_TYPE_REENTER);
|
||
|
|
return (ret);
|
||
|
|
case HYP_READ_REGISTER:
|
||
|
|
return (vmm_hyp_read_reg(x1));
|
||
|
|
case HYP_CLEAN_S2_TLBI:
|
||
|
|
return (vmm_clean_s2_tlbi());
|
||
|
|
case HYP_DC_CIVAC:
|
||
|
|
return (vmm_dc_civac(x1, x2));
|
||
|
|
case HYP_EL2_TLBI:
|
||
|
|
return (vmm_el2_tlbi(x1, x2, x3));
|
||
|
|
case HYP_S2_TLBI_RANGE:
|
||
|
|
return (vm_s2_tlbi_range(x1, x2, x3, x4));
|
||
|
|
case HYP_S2_TLBI_ALL:
|
||
|
|
return (vm_s2_tlbi_all(x1));
|
||
|
|
case HYP_CLEANUP: /* Handled in vmm_hyp_exception.S */
|
||
|
|
default:
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
return (0);
|
||
|
|
}
|