system/freebsd-src
system/freebsd-src
1
0
Fork 0
mirror of https://github.com/freebsd/freebsd-src synced 2024-07-21 18:27:22 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Import the kernel parts of bhyve/arm64

Browse source 
To support virtual machines on arm64 add the vmm code. This is based on
earlier work by Mihai Carabas and Alexandru Elisei at University
Politehnica of Bucharest, with further work by myself and Mark Johnston.

All AArch64 CPUs should work, however only the GICv3 interrupt
controller is supported. There is initial support to allow the GICv2
to be supported in the future. Only pure Armv8.0 virtualisation is
supported, the Virtualization Host Extensions are not currently used.

With a separate userspace patch and U-Boot port FreeBSD guests are able
to boot to multiuser mode, and the hypervisor can be tested with the
kvm unit tests. Linux partially boots, but hangs before entering
userspace. Other operating systems are untested.

Sponsored by:	Arm Ltd
Sponsored by:	Innovate UK
Sponsored by:	The FreeBSD Foundation
Sponsored by:	University Politehnica of Bucharest
Differential Revision:	https://reviews.freebsd.org/D37428
This commit is contained in:
Andrew Turner 2024-01-09 15:22:27 +00:00
parent 0f4071978e
commit 47e073941f
34 changed files with 11067 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

362
sys/arm64/include/vmm.h Normal file
View file

@ -0,0 +1,362 @@
/*
* Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_H_
#define _VMM_H_
#include <sys/param.h>
#include <sys/cpuset.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include "pte.h"
#include "pmap.h"
struct vcpu;
enum vm_suspend_how {
VM_SUSPEND_NONE,
VM_SUSPEND_RESET,
VM_SUSPEND_POWEROFF,
VM_SUSPEND_HALT,
VM_SUSPEND_LAST
};
/*
* Identifiers for architecturally defined registers.
*/
enum vm_reg_name {
VM_REG_GUEST_X0 = 0,
VM_REG_GUEST_X1,
VM_REG_GUEST_X2,
VM_REG_GUEST_X3,
VM_REG_GUEST_X4,
VM_REG_GUEST_X5,
VM_REG_GUEST_X6,
VM_REG_GUEST_X7,
VM_REG_GUEST_X8,
VM_REG_GUEST_X9,
VM_REG_GUEST_X10,
VM_REG_GUEST_X11,
VM_REG_GUEST_X12,
VM_REG_GUEST_X13,
VM_REG_GUEST_X14,
VM_REG_GUEST_X15,
VM_REG_GUEST_X16,
VM_REG_GUEST_X17,
VM_REG_GUEST_X18,
VM_REG_GUEST_X19,
VM_REG_GUEST_X20,
VM_REG_GUEST_X21,
VM_REG_GUEST_X22,
VM_REG_GUEST_X23,
VM_REG_GUEST_X24,
VM_REG_GUEST_X25,
VM_REG_GUEST_X26,
VM_REG_GUEST_X27,
VM_REG_GUEST_X28,
VM_REG_GUEST_X29,
VM_REG_GUEST_LR,
VM_REG_GUEST_SP,
VM_REG_GUEST_PC,
VM_REG_GUEST_CPSR,
VM_REG_GUEST_SCTLR_EL1,
VM_REG_GUEST_TTBR0_EL1,
VM_REG_GUEST_TTBR1_EL1,
VM_REG_GUEST_TCR_EL1,
VM_REG_GUEST_TCR2_EL1,
VM_REG_LAST
};
#define VM_INTINFO_VECTOR(info) ((info) & 0xff)
#define VM_INTINFO_DEL_ERRCODE 0x800
#define VM_INTINFO_RSVD 0x7ffff000
#define VM_INTINFO_VALID 0x80000000
#define VM_INTINFO_TYPE 0x700
#define VM_INTINFO_HWINTR (0 << 8)
#define VM_INTINFO_NMI (2 << 8)
#define VM_INTINFO_HWEXCEPTION (3 << 8)
#define VM_INTINFO_SWINTR (4 << 8)
#define VM_MAX_SUFFIXLEN 15
#define VM_GUEST_BASE_IPA 0x80000000UL /* Guest kernel start ipa */
#ifdef _KERNEL
#define VM_MAX_NAMELEN 32
struct vm;
struct vm_exception;
struct vm_exit;
struct vm_run;
struct vm_object;
struct vm_guest_paging;
struct vm_vgic_descr;
struct pmap;
struct vm_eventinfo {
void *rptr; /* rendezvous cookie */
int *sptr; /* suspend cookie */
int *iptr; /* reqidle cookie */
};
int vm_create(const char *name, struct vm **retvm);
struct vcpu *vm_alloc_vcpu(struct vm *vm, int vcpuid);
void vm_slock_vcpus(struct vm *vm);
void vm_unlock_vcpus(struct vm *vm);
void vm_destroy(struct vm *vm);
int vm_reinit(struct vm *vm);
const char *vm_name(struct vm *vm);
/*
* APIs that modify the guest memory map require all vcpus to be frozen.
*/
void vm_slock_memsegs(struct vm *vm);
void vm_xlock_memsegs(struct vm *vm);
void vm_unlock_memsegs(struct vm *vm);
int vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t off,
size_t len, int prot, int flags);
int vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len);
int vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem);
void vm_free_memseg(struct vm *vm, int ident);
/*
* APIs that inspect the guest memory map require only a *single* vcpu to
* be frozen. This acts like a read lock on the guest memory map since any
* modification requires *all* vcpus to be frozen.
*/
int vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
vm_ooffset_t *segoff, size_t *len, int *prot, int *flags);
int vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
struct vm_object **objptr);
vm_paddr_t vmm_sysmem_maxaddr(struct vm *vm);
void *vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len,
int prot, void **cookie);
void *vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len,
int prot, void **cookie);
void vm_gpa_release(void *cookie);
bool vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa);
int vm_gla2gpa_nofault(struct vcpu *vcpu, struct vm_guest_paging *paging,
uint64_t gla, int prot, uint64_t *gpa, int *is_fault);
uint16_t vm_get_maxcpus(struct vm *vm);
void vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
uint16_t *threads, uint16_t *maxcpus);
int vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
uint16_t threads, uint16_t maxcpus);
int vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval);
int vm_set_register(struct vcpu *vcpu, int reg, uint64_t val);
int vm_run(struct vcpu *vcpu);
int vm_suspend(struct vm *vm, enum vm_suspend_how how);
void* vm_get_cookie(struct vm *vm);
int vcpu_vcpuid(struct vcpu *vcpu);
void *vcpu_get_cookie(struct vcpu *vcpu);
struct vm *vcpu_vm(struct vcpu *vcpu);
struct vcpu *vm_vcpu(struct vm *vm, int cpu);
int vm_get_capability(struct vcpu *vcpu, int type, int *val);
int vm_set_capability(struct vcpu *vcpu, int type, int val);
int vm_activate_cpu(struct vcpu *vcpu);
int vm_suspend_cpu(struct vm *vm, struct vcpu *vcpu);
int vm_resume_cpu(struct vm *vm, struct vcpu *vcpu);
int vm_inject_exception(struct vcpu *vcpu, uint64_t esr, uint64_t far);
int vm_attach_vgic(struct vm *vm, struct vm_vgic_descr *descr);
int vm_assert_irq(struct vm *vm, uint32_t irq);
int vm_deassert_irq(struct vm *vm, uint32_t irq);
int vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,
int func);
struct vm_exit *vm_exitinfo(struct vcpu *vcpu);
void vm_exit_suspended(struct vcpu *vcpu, uint64_t pc);
void vm_exit_debug(struct vcpu *vcpu, uint64_t pc);
void vm_exit_rendezvous(struct vcpu *vcpu, uint64_t pc);
void vm_exit_astpending(struct vcpu *vcpu, uint64_t pc);
cpuset_t vm_active_cpus(struct vm *vm);
cpuset_t vm_debug_cpus(struct vm *vm);
cpuset_t vm_suspended_cpus(struct vm *vm);
static __inline bool
virt_enabled(void)
{
return (has_hyp());
}
static __inline int
vcpu_rendezvous_pending(struct vm_eventinfo *info)
{
return (*((uintptr_t *)(info->rptr)) != 0);
}
static __inline int
vcpu_suspended(struct vm_eventinfo *info)
{
return (*info->sptr);
}
int vcpu_debugged(struct vcpu *vcpu);
enum vcpu_state {
VCPU_IDLE,
VCPU_FROZEN,
VCPU_RUNNING,
VCPU_SLEEPING,
};
int vcpu_set_state(struct vcpu *vcpu, enum vcpu_state state, bool from_idle);
enum vcpu_state vcpu_get_state(struct vcpu *vcpu, int *hostcpu);
static int __inline
vcpu_is_running(struct vcpu *vcpu, int *hostcpu)
{
return (vcpu_get_state(vcpu, hostcpu) == VCPU_RUNNING);
}
#ifdef _SYS_PROC_H_
static int __inline
vcpu_should_yield(struct vcpu *vcpu)
{
struct thread *td;
td = curthread;
return (td->td_ast != 0 || td->td_owepreempt != 0);
}
#endif
void *vcpu_stats(struct vcpu *vcpu);
void vcpu_notify_event(struct vcpu *vcpu);
enum vm_reg_name vm_segment_name(int seg_encoding);
struct vm_copyinfo {
uint64_t gpa;
size_t len;
void *hva;
void *cookie;
};
#endif /* _KERNEL */
#define VM_DIR_READ 0
#define VM_DIR_WRITE 1
#define VM_GP_M_MASK 0x1f
#define VM_GP_MMU_ENABLED (1 << 5)
struct vm_guest_paging {
uint64_t ttbr0_addr;
uint64_t ttbr1_addr;
uint64_t tcr_el1;
uint64_t tcr2_el1;
int flags;
int padding;
};
struct vie {
uint8_t access_size:4, sign_extend:1, dir:1, unused:2;
enum vm_reg_name reg;
};
struct vre {
uint32_t inst_syndrome;
uint8_t dir:1, unused:7;
enum vm_reg_name reg;
};
/*
* Identifiers for optional vmm capabilities
*/
enum vm_cap_type {
VM_CAP_HALT_EXIT,
VM_CAP_MTRAP_EXIT,
VM_CAP_PAUSE_EXIT,
VM_CAP_UNRESTRICTED_GUEST,
VM_CAP_MAX
};
enum vm_exitcode {
VM_EXITCODE_BOGUS,
VM_EXITCODE_INST_EMUL,
VM_EXITCODE_REG_EMUL,
VM_EXITCODE_HVC,
VM_EXITCODE_SUSPENDED,
VM_EXITCODE_HYP,
VM_EXITCODE_WFI,
VM_EXITCODE_PAGING,
VM_EXITCODE_SMCCC,
VM_EXITCODE_DEBUG,
VM_EXITCODE_MAX
};
struct vm_exit {
enum vm_exitcode exitcode;
int inst_length;
uint64_t pc;
union {
/*
* ARM specific payload.
*/
struct {
uint32_t exception_nr;
uint32_t pad;
uint64_t esr_el2; /* Exception Syndrome Register */
uint64_t far_el2; /* Fault Address Register */
uint64_t hpfar_el2; /* Hypervisor IPA Fault Address Register */
} hyp;
struct {
struct vre vre;
} reg_emul;
struct {
uint64_t gpa;
uint64_t esr;
} paging;
struct {
uint64_t gpa;
struct vm_guest_paging paging;
struct vie vie;
} inst_emul;
/*
* A SMCCC call, e.g. starting a core via PSCI.
* Further arguments can be read by asking the kernel for
* all register values.
*/
struct {
uint64_t func_id;
uint64_t args[7];
} smccc_call;
struct {
enum vm_suspend_how how;
} suspended;
} u;
};
#endif /* _VMM_H_ */

272
sys/arm64/include/vmm_dev.h Normal file
View file

@ -0,0 +1,272 @@
/*
* Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_DEV_H_
#define _VMM_DEV_H_
#ifdef _KERNEL
void vmmdev_init(void);
int vmmdev_cleanup(void);
#endif
struct vm_memmap {
vm_paddr_t gpa;
int segid; /* memory segment */
vm_ooffset_t segoff; /* offset into memory segment */
size_t len; /* mmap length */
int prot; /* RWX */
int flags;
};
#define VM_MEMMAP_F_WIRED 0x01
struct vm_munmap {
vm_paddr_t gpa;
size_t len;
};
#define VM_MEMSEG_NAME(m) ((m)->name[0] != '\0' ? (m)->name : NULL)
struct vm_memseg {
int segid;
size_t len;
char name[VM_MAX_SUFFIXLEN + 1];
};
struct vm_register {
int cpuid;
int regnum; /* enum vm_reg_name */
uint64_t regval;
};
struct vm_register_set {
int cpuid;
unsigned int count;
const int *regnums; /* enum vm_reg_name */
uint64_t *regvals;
};
struct vm_run {
int cpuid;
cpuset_t *cpuset; /* CPU set storage */
size_t cpusetsize;
struct vm_exit *vm_exit;
};
struct vm_exception {
int cpuid;
uint64_t esr;
uint64_t far;
};
struct vm_msi {
uint64_t msg;
uint64_t addr;
int bus;
int slot;
int func;
};
struct vm_capability {
int cpuid;
enum vm_cap_type captype;
int capval;
int allcpus;
};
#define MAX_VM_STATS 64
struct vm_stats {
int cpuid; /* in */
int index; /* in */
int num_entries; /* out */
struct timeval tv;
uint64_t statbuf[MAX_VM_STATS];
};
struct vm_stat_desc {
int index; /* in */
char desc[128]; /* out */
};
struct vm_suspend {
enum vm_suspend_how how;
};
struct vm_gla2gpa {
int vcpuid; /* inputs */
int prot; /* PROT_READ or PROT_WRITE */
uint64_t gla;
struct vm_guest_paging paging;
int fault; /* outputs */
uint64_t gpa;
};
struct vm_activate_cpu {
int vcpuid;
};
struct vm_cpuset {
int which;
int cpusetsize;
cpuset_t *cpus;
};
#define VM_ACTIVE_CPUS 0
#define VM_SUSPENDED_CPUS 1
#define VM_DEBUG_CPUS 2
struct vm_vgic_version {
u_int version;
u_int flags;
};
struct vm_vgic_descr {
struct vm_vgic_version ver;
union {
struct {
uint64_t dist_start;
uint64_t dist_size;
uint64_t redist_start;
uint64_t redist_size;
} v3_regs;
};
};
struct vm_irq {
uint32_t irq;
};
struct vm_cpu_topology {
uint16_t sockets;
uint16_t cores;
uint16_t threads;
uint16_t maxcpus;
};
enum {
/* general routines */
IOCNUM_ABIVERS = 0,
IOCNUM_RUN = 1,
IOCNUM_SET_CAPABILITY = 2,
IOCNUM_GET_CAPABILITY = 3,
IOCNUM_SUSPEND = 4,
IOCNUM_REINIT = 5,
/* memory apis */
IOCNUM_GET_GPA_PMAP = 12,
IOCNUM_GLA2GPA_NOFAULT = 13,
IOCNUM_ALLOC_MEMSEG = 14,
IOCNUM_GET_MEMSEG = 15,
IOCNUM_MMAP_MEMSEG = 16,
IOCNUM_MMAP_GETNEXT = 17,
IOCNUM_MUNMAP_MEMSEG = 18,
/* register/state accessors */
IOCNUM_SET_REGISTER = 20,
IOCNUM_GET_REGISTER = 21,
IOCNUM_SET_REGISTER_SET = 24,
IOCNUM_GET_REGISTER_SET = 25,
/* statistics */
IOCNUM_VM_STATS = 50,
IOCNUM_VM_STAT_DESC = 51,
/* CPU Topology */
IOCNUM_SET_TOPOLOGY = 63,
IOCNUM_GET_TOPOLOGY = 64,
/* interrupt injection */
IOCNUM_ASSERT_IRQ = 80,
IOCNUM_DEASSERT_IRQ = 81,
IOCNUM_RAISE_MSI = 82,
IOCNUM_INJECT_EXCEPTION = 83,
/* vm_cpuset */
IOCNUM_ACTIVATE_CPU = 90,
IOCNUM_GET_CPUSET = 91,
IOCNUM_SUSPEND_CPU = 92,
IOCNUM_RESUME_CPU = 93,
/* vm_attach_vgic */
IOCNUM_GET_VGIC_VERSION = 110,
IOCNUM_ATTACH_VGIC = 111,
};
#define VM_RUN \
_IOWR('v', IOCNUM_RUN, struct vm_run)
#define VM_SUSPEND \
_IOW('v', IOCNUM_SUSPEND, struct vm_suspend)
#define VM_REINIT \
_IO('v', IOCNUM_REINIT)
#define VM_ALLOC_MEMSEG \
_IOW('v', IOCNUM_ALLOC_MEMSEG, struct vm_memseg)
#define VM_GET_MEMSEG \
_IOWR('v', IOCNUM_GET_MEMSEG, struct vm_memseg)
#define VM_MMAP_MEMSEG \
_IOW('v', IOCNUM_MMAP_MEMSEG, struct vm_memmap)
#define VM_MMAP_GETNEXT \
_IOWR('v', IOCNUM_MMAP_GETNEXT, struct vm_memmap)
#define VM_MUNMAP_MEMSEG \
_IOW('v', IOCNUM_MUNMAP_MEMSEG, struct vm_munmap)
#define VM_SET_REGISTER \
_IOW('v', IOCNUM_SET_REGISTER, struct vm_register)
#define VM_GET_REGISTER \
_IOWR('v', IOCNUM_GET_REGISTER, struct vm_register)
#define VM_SET_REGISTER_SET \
_IOW('v', IOCNUM_SET_REGISTER_SET, struct vm_register_set)
#define VM_GET_REGISTER_SET \
_IOWR('v', IOCNUM_GET_REGISTER_SET, struct vm_register_set)
#define VM_SET_CAPABILITY \
_IOW('v', IOCNUM_SET_CAPABILITY, struct vm_capability)
#define VM_GET_CAPABILITY \
_IOWR('v', IOCNUM_GET_CAPABILITY, struct vm_capability)
#define VM_STATS \
_IOWR('v', IOCNUM_VM_STATS, struct vm_stats)
#define VM_STAT_DESC \
_IOWR('v', IOCNUM_VM_STAT_DESC, struct vm_stat_desc)
#define VM_ASSERT_IRQ \
_IOW('v', IOCNUM_ASSERT_IRQ, struct vm_irq)
#define VM_DEASSERT_IRQ \
_IOW('v', IOCNUM_DEASSERT_IRQ, struct vm_irq)
#define VM_RAISE_MSI \
_IOW('v', IOCNUM_RAISE_MSI, struct vm_msi)
#define VM_INJECT_EXCEPTION \
_IOW('v', IOCNUM_INJECT_EXCEPTION, struct vm_exception)
#define VM_SET_TOPOLOGY \
_IOW('v', IOCNUM_SET_TOPOLOGY, struct vm_cpu_topology)
#define VM_GET_TOPOLOGY \
_IOR('v', IOCNUM_GET_TOPOLOGY, struct vm_cpu_topology)
#define VM_GLA2GPA_NOFAULT \
_IOWR('v', IOCNUM_GLA2GPA_NOFAULT, struct vm_gla2gpa)
#define VM_ACTIVATE_CPU \
_IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu)
#define VM_GET_CPUS \
_IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset)
#define VM_SUSPEND_CPU \
_IOW('v', IOCNUM_SUSPEND_CPU, struct vm_activate_cpu)
#define VM_RESUME_CPU \
_IOW('v', IOCNUM_RESUME_CPU, struct vm_activate_cpu)
#define VM_GET_VGIC_VERSION \
_IOR('v', IOCNUM_GET_VGIC_VERSION, struct vm_vgic_version)
#define VM_ATTACH_VGIC \
_IOW('v', IOCNUM_ATTACH_VGIC, struct vm_vgic_descr)
#endif

83
sys/arm64/include/vmm_instruction_emul.h Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_INSTRUCTION_EMUL_H_
#define _VMM_INSTRUCTION_EMUL_H_
/*
* Callback functions to read and write memory regions.
*/
typedef int (*mem_region_read_t)(struct vcpu *vcpu, uint64_t gpa,
uint64_t *rval, int rsize, void *arg);
typedef int (*mem_region_write_t)(struct vcpu *vcpu, uint64_t gpa,
uint64_t wval, int wsize, void *arg);
/*
* Callback functions to read and write registers.
*/
typedef int (*reg_read_t)(struct vcpu *vcpu, uint64_t *rval, void *arg);
typedef int (*reg_write_t)(struct vcpu *vcpu, uint64_t wval, void *arg);
/*
* Emulate the decoded 'vie' instruction when it contains a memory operation.
*
* The callbacks 'mrr' and 'mrw' emulate reads and writes to the memory region
* containing 'gpa'. 'mrarg' is an opaque argument that is passed into the
* callback functions.
*
* 'void *vm' should be 'struct vm *' when called from kernel context and
* 'struct vmctx *' when called from user context.
*
*/
int vmm_emulate_instruction(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
struct vm_guest_paging *paging, mem_region_read_t mrr,
mem_region_write_t mrw, void *mrarg);
/*
* Emulate the decoded 'vre' instruction when it contains a register access.
*
* The callbacks 'regread' and 'regwrite' emulate reads and writes to the
* register from 'vie'. 'regarg' is an opaque argument that is passed into the
* callback functions.
*
* 'void *vm' should be 'struct vm *' when called from kernel context and
* 'struct vmctx *' when called from user context.
*
*/
int vmm_emulate_register(struct vcpu *vcpu, struct vre *vre, reg_read_t regread,
reg_write_t regwrite, void *regarg);
#ifdef _KERNEL
void vm_register_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask,
reg_read_t reg_read, reg_write_t reg_write, void *arg);
void vm_deregister_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask);
void vm_register_inst_handler(struct vm *vm, uint64_t start, uint64_t size,
mem_region_read_t mmio_read, mem_region_write_t mmio_write);
void vm_deregister_inst_handler(struct vm *vm, uint64_t start, uint64_t size);
#endif
#endif /* _VMM_INSTRUCTION_EMUL_H_ */

1
sys/arm64/include/vmm_snapshot.h Normal file
View file

@ -0,0 +1 @@
/* $FreeBSD$ */

165
sys/arm64/vmm/arm64.h Normal file
View file

@ -0,0 +1,165 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_ARM64_H_
#define _VMM_ARM64_H_
#include <machine/reg.h>
#include <machine/hypervisor.h>
#include <machine/pcpu.h>
#include "mmu.h"
#include "io/vgic_v3.h"
#include "io/vtimer.h"
struct vgic_v3;
struct vgic_v3_cpu;
struct hypctx {
struct trapframe tf;
/*
* EL1 control registers.
*/
uint64_t elr_el1; /* Exception Link Register */
uint64_t sp_el0; /* Stack pointer */
uint64_t tpidr_el0; /* EL0 Software ID Register */
uint64_t tpidrro_el0; /* Read-only Thread ID Register */
uint64_t tpidr_el1; /* EL1 Software ID Register */
uint64_t vbar_el1; /* Vector Base Address Register */
uint64_t actlr_el1; /* Auxiliary Control Register */
uint64_t afsr0_el1; /* Auxiliary Fault Status Register 0 */
uint64_t afsr1_el1; /* Auxiliary Fault Status Register 1 */
uint64_t amair_el1; /* Auxiliary Memory Attribute Indirection Register */
uint64_t contextidr_el1; /* Current Process Identifier */
uint64_t cpacr_el1; /* Architectural Feature Access Control Register */
uint64_t csselr_el1; /* Cache Size Selection Register */
uint64_t esr_el1; /* Exception Syndrome Register */
uint64_t far_el1; /* Fault Address Register */
uint64_t mair_el1; /* Memory Attribute Indirection Register */
uint64_t mdccint_el1; /* Monitor DCC Interrupt Enable Register */
uint64_t mdscr_el1; /* Monitor Debug System Control Register */
uint64_t par_el1; /* Physical Address Register */
uint64_t sctlr_el1; /* System Control Register */
uint64_t tcr_el1; /* Translation Control Register */
uint64_t tcr2_el1; /* Translation Control Register 2 */
uint64_t ttbr0_el1; /* Translation Table Base Register 0 */
uint64_t ttbr1_el1; /* Translation Table Base Register 1 */
uint64_t spsr_el1; /* Saved Program Status Register */
uint64_t pmcr_el0; /* Performance Monitors Control Register */
uint64_t pmccntr_el0;
uint64_t pmccfiltr_el0;
uint64_t pmcntenset_el0;
uint64_t pmintenset_el1;
uint64_t pmovsset_el0;
uint64_t pmselr_el0;
uint64_t pmuserenr_el0;
uint64_t pmevcntr_el0[31];
uint64_t pmevtyper_el0[31];
uint64_t dbgbcr_el1[16]; /* Debug Breakpoint Control Registers */
uint64_t dbgbvr_el1[16]; /* Debug Breakpoint Value Registers */
uint64_t dbgwcr_el1[16]; /* Debug Watchpoint Control Registers */
uint64_t dbgwvr_el1[16]; /* Debug Watchpoint Value Registers */
/* EL2 control registers */
uint64_t cptr_el2; /* Architectural Feature Trap Register */
uint64_t hcr_el2; /* Hypervisor Configuration Register */
uint64_t mdcr_el2; /* Monitor Debug Configuration Register */
uint64_t vpidr_el2; /* Virtualization Processor ID Register */
uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */
uint64_t el2_addr; /* The address of this in el2 space */
struct hyp *hyp;
struct vcpu *vcpu;
struct {
uint64_t far_el2; /* Fault Address Register */
uint64_t hpfar_el2; /* Hypervisor IPA Fault Address Register */
} exit_info;
struct vtimer_cpu vtimer_cpu;
struct vgic_v3_regs vgic_v3_regs;
struct vgic_v3_cpu *vgic_cpu;
bool has_exception;
};
struct hyp {
struct vm *vm;
struct vtimer vtimer;
uint64_t vmid_generation;
uint64_t vttbr_el2;
uint64_t el2_addr; /* The address of this in el2 space */
bool vgic_attached;
struct vgic_v3 *vgic;
struct hypctx *ctx[];
};
#define DEFINE_VMMOPS_IFUNC(ret_type, opname, args) \
ret_type vmmops_##opname args;
DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum))
DEFINE_VMMOPS_IFUNC(int, modcleanup, (void))
DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap))
DEFINE_VMMOPS_IFUNC(int, gla2gpa, (void *vcpui, struct vm_guest_paging *paging,
uint64_t gla, int prot, uint64_t *gpa, int *is_fault))
DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap,
struct vm_eventinfo *info))
DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi))
DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu,
int vcpu_id))
DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui))
DEFINE_VMMOPS_IFUNC(int, exception, (void *vcpui, uint64_t esr, uint64_t far))
DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval))
DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val))
DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval))
DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val))
DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min,
vm_offset_t max))
DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace))
#ifdef notyet
#ifdef BHYVE_SNAPSHOT
DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta))
DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui,
struct vm_snapshot_meta *meta))
DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vcpui, uint64_t now))
#endif
#endif
uint64_t vmm_call_hyp(uint64_t, ...);
#if 0
#define eprintf(fmt, ...) printf("%s:%d " fmt, __func__, __LINE__, ##__VA_ARGS__)
#else
#define eprintf(fmt, ...) do {} while(0)
#endif
struct hypctx *arm64_get_active_vcpu(void);
void raise_data_insn_abort(struct hypctx *, uint64_t, bool, int);
#endif /* !_VMM_ARM64_H_ */

114
sys/arm64/vmm/hyp.h Normal file
View file

@ -0,0 +1,114 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_HYP_H_
#define _VMM_HYP_H_
/*
* The translation tables for the hypervisor mode will hold mappings for kernel
* virtual addresses and an identity mapping (VA == PA) necessary when
* enabling/disabling the MMU.
*
* When in EL2 exception level the translation table base register is TTBR0_EL2
* and the virtual addresses generated by the CPU must be at the bottom of the
* memory, with the first 16 bits all set to zero:
*
* 0x0000ffffffffffff End hyp address space
* 0x0000000000000000 Start of hyp address space
*
* To run code in hyp mode we need to convert kernel virtual addresses to
* addreses that fit into this address space.
*
* The kernel virtual address range is:
*
* 0xffff007fffffffff End of KVA
* 0xffff000000000000 Kernel base address & start of KVA
*
* (see /sys/arm64/include/vmparam.h).
*
* We could convert the kernel virtual addresses to valid EL2 addresses by
* setting the first 16 bits to zero and thus mapping the kernel addresses in
* the bottom half of the EL2 address space, but then they might clash with the
* identity mapping addresses. Instead we map the kernel addresses in the upper
* half of the EL2 address space.
*
* The hypervisor address space will look like this:
*
* 0x0000807fffffffff End of KVA mapping
* 0x0000800000000000 Start of KVA mapping
*
* 0x00007fffffffffff End of identity mapping
* 0x0000000000000000 Start of identity mapping
*
* With the scheme we have 47 bits at our disposable for the identity map and
* another 47 bits for the kernel virtual addresses. For a maximum physical
* memory size of 128TB we are guaranteed to not have any clashes between
* addresses.
*/
#define HYP_VM_MIN_ADDRESS 0x0000000000000000
#define HYP_VM_MAX_ADDRESS 0x0001000000000000
/*
* When the vmm code is installed the following handles can be used by
* the host to call into EL2.
*/
#define HYP_CLEANUP 0x00000001
#define HYP_ENTER_GUEST 0x00000002
#define HYP_READ_REGISTER 0x00000003
#define HYP_REG_ICH_VTR 0x1
#define HYP_REG_CNTHCTL 0x2
#define HYP_CLEAN_S2_TLBI 0x00000004
#define HYP_DC_CIVAC 0x00000005
#define HYP_EL2_TLBI 0x00000006
#define HYP_EL2_TLBI_ALL 0x1
#define HYP_EL2_TLBI_VA 0x2
#define HYP_S2_TLBI_RANGE 0x00000010
#define HYP_S2_TLBI_ALL 0x00000011
/*
* When taking asynchronous exceptions, or interrupts, with the exception of the
* SError interrupt, the exception syndrome register is not updated with the
* exception code. We need to differentiate between the different exception
* types taken to EL2.
*/
#define EXCP_TYPE_EL1_SYNC 0
#define EXCP_TYPE_EL1_IRQ 1
#define EXCP_TYPE_EL1_FIQ 2
#define EXCP_TYPE_EL1_ERROR 3
#define EXCP_TYPE_EL2_SYNC 4
#define EXCP_TYPE_EL2_IRQ 5
#define EXCP_TYPE_EL2_FIQ 6
#define EXCP_TYPE_EL2_ERROR 7
#define EXCP_TYPE_MAINT_IRQ 8
/* Used internally in vmm_hyp.c */
#define EXCP_TYPE_REENTER 9
#define HYP_GET_VECTOR_TABLE -1
#endif /* !_VMM_HYP_H_ */

122
sys/arm64/vmm/io/vgic.c Normal file
View file

@ -0,0 +1,122 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023 Arm Ltd
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include "vgic.h"
#include "vgic_if.h"
device_t vgic_dev;
bool
vgic_present(void)
{
return (vgic_dev != NULL);
}
void
vgic_init(void)
{
VGIC_INIT(vgic_dev);
}
int
vgic_attach_to_vm(struct hyp *hyp, struct vm_vgic_descr *descr)
{
return (VGIC_ATTACH_TO_VM(vgic_dev, hyp, descr));
}
void
vgic_detach_from_vm(struct hyp *hyp)
{
VGIC_DETACH_FROM_VM(vgic_dev, hyp);
}
void
vgic_vminit(struct hyp *hyp)
{
VGIC_VMINIT(vgic_dev, hyp);
}
void
vgic_cpuinit(struct hypctx *hypctx)
{
VGIC_CPUINIT(vgic_dev, hypctx);
}
void
vgic_cpucleanup(struct hypctx *hypctx)
{
VGIC_CPUCLEANUP(vgic_dev, hypctx);
}
void
vgic_vmcleanup(struct hyp *hyp)
{
VGIC_VMCLEANUP(vgic_dev, hyp);
}
int
vgic_max_cpu_count(struct hyp *hyp)
{
return (VGIC_MAX_CPU_COUNT(vgic_dev, hyp));
}
bool
vgic_has_pending_irq(struct hypctx *hypctx)
{
return (VGIC_HAS_PENDING_IRQ(vgic_dev, hypctx));
}
/* TODO: vcpuid -> hypctx ? */
/* TODO: Add a vgic interface */
int
vgic_inject_irq(struct hyp *hyp, int vcpuid, uint32_t irqid, bool level)
{
return (VGIC_INJECT_IRQ(vgic_dev, hyp, vcpuid, irqid, level));
}
int
vgic_inject_msi(struct hyp *hyp, uint64_t msg, uint64_t addr)
{
return (VGIC_INJECT_MSI(vgic_dev, hyp, msg, addr));
}
void
vgic_flush_hwstate(struct hypctx *hypctx)
{
VGIC_FLUSH_HWSTATE(vgic_dev, hypctx);
}
void
vgic_sync_hwstate(struct hypctx *hypctx)
{
VGIC_SYNC_HWSTATE(vgic_dev, hypctx);
}

52
sys/arm64/vmm/io/vgic.h Normal file
View file

@ -0,0 +1,52 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2023 Arm Ltd
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VGIC_H_
#define _VGIC_H_
struct hyp;
struct hypctx;
struct vm_vgic_descr;
extern device_t vgic_dev;
bool vgic_present(void);
void vgic_init(void);
int vgic_attach_to_vm(struct hyp *hyp, struct vm_vgic_descr *descr);
void vgic_detach_from_vm(struct hyp *hyp);
void vgic_vminit(struct hyp *hyp);
void vgic_cpuinit(struct hypctx *hypctx);
void vgic_cpucleanup(struct hypctx *hypctx);
void vgic_vmcleanup(struct hyp *hyp);
int vgic_max_cpu_count(struct hyp *hyp);
bool vgic_has_pending_irq(struct hypctx *hypctx);
int vgic_inject_irq(struct hyp *hyp, int vcpuid, uint32_t irqid, bool level);
int vgic_inject_msi(struct hyp *hyp, uint64_t msg, uint64_t addr);
void vgic_flush_hwstate(struct hypctx *hypctx);
void vgic_sync_hwstate(struct hypctx *hypctx);
#endif /* _VGIC_H_ */

104
sys/arm64/vmm/io/vgic_if.m Normal file
View file

@ -0,0 +1,104 @@
#-
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (c) 2023 Arm Ltd
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
INTERFACE vgic;
HEADER {
struct hyp;
struct hypctx;
struct vm_vgic_descr;
};
METHOD void init {
device_t dev;
}
METHOD int attach_to_vm {
device_t dev;
struct hyp *hyp;
struct vm_vgic_descr *descr;
};
METHOD void detach_from_vm {
device_t dev;
struct hyp *hyp;
}
METHOD void vminit {
device_t dev;
struct hyp *hyp;
}
METHOD void cpuinit {
device_t dev;
struct hypctx *hypctx;
}
METHOD void cpucleanup {
device_t dev;
struct hypctx *hypctx;
}
METHOD void vmcleanup {
device_t dev;
struct hyp *hyp;
}
METHOD int max_cpu_count {
device_t dev;
struct hyp *hyp;
}
METHOD bool has_pending_irq {
device_t dev;
struct hypctx *hypctx;
}
METHOD int inject_irq {
device_t dev;
struct hyp *hyp;
int vcpuid;
uint32_t irqid;
bool level;
}
METHOD int inject_msi {
device_t dev;
struct hyp *hyp;
uint64_t msg;
uint64_t addr;
}
METHOD void flush_hwstate {
device_t dev;
struct hypctx *hypctx;
}
METHOD void sync_hwstate {
device_t dev;
struct hypctx *hypctx;
}

2348
sys/arm64/vmm/io/vgic_v3.c Normal file
View file

File diff suppressed because it is too large Load diff

57
sys/arm64/vmm/io/vgic_v3.h Normal file
View file

@ -0,0 +1,57 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_VGIC_V3_H_
#define _VMM_VGIC_V3_H_
#define VGIC_ICH_LR_NUM_MAX 16
#define VGIC_ICH_APR_NUM_MAX 4
/* Registers accessed by EL2 */
struct vgic_v3_regs {
uint32_t ich_eisr_el2; /* End of Interrupt Status Register */
uint32_t ich_elrsr_el2; /* Empty List register Status Register (ICH_ELRSR_EL2) */
uint32_t ich_hcr_el2; /* Hyp Control Register */
uint32_t ich_misr_el2; /* Maintenance Interrupt State Register */
uint32_t ich_vmcr_el2; /* Virtual Machine Control Register */
/*
* The List Registers are part of the VM context and are modified on a
* world switch. They need to be allocated statically so they are
* mapped in the EL2 translation tables when struct hypctx is mapped.
*/
uint64_t ich_lr_el2[VGIC_ICH_LR_NUM_MAX];
uint16_t ich_lr_num;
/* Active Priorities Registers for Group 0 and 1 interrupts */
uint16_t ich_apr_num;
uint32_t ich_ap0r_el2[VGIC_ICH_APR_NUM_MAX];
uint32_t ich_ap1r_el2[VGIC_ICH_APR_NUM_MAX];
};
#endif /* !_VMM_VGIC_V3_H_ */

129
sys/arm64/vmm/io/vgic_v3_reg.h Normal file
View file

@ -0,0 +1,129 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2018 The FreeBSD Foundation
*
* This software was developed by Alexandru Elisei under sponsorship
* from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VGIC_V3_REG_H_
#define _VGIC_V3_REG_H_
/* Interrupt Controller End of Interrupt Status Register */
#define ICH_EISR_EL2_STATUS_MASK 0xffff
#define ICH_EISR_EL2_EOI_NOT_HANDLED(lr) ((1 << lr) & ICH_EISR_EL2_STATUS_MASK)
/* Interrupt Controller Empty List Register Status Register */
#define ICH_ELSR_EL2_STATUS_MASK 0xffff
#define ICH_ELSR_EL2_LR_EMPTY(x) ((1 << x) & ICH_ELSR_EL2_STATUS_MASK)
/* Interrupt Controller Hyp Control Register */
#define ICH_HCR_EL2_EOICOUNT_SHIFT 27
#define ICH_HCR_EL2_EOICOUNT_MASK (0x1f << ICH_HCR_EL2_EOICOUNT_SHIFT)
#define ICH_HCR_EL2_TDIR (1 << 14) /* Trap non-secure EL1 writes to IC{C, V}_DIR_EL1 */
#define ICH_HCR_EL2_TSEI (1 << 14) /* Trap System Error Interupts (SEI) to EL2 */
#define ICH_HCR_EL2_TALL1 (1 << 12) /* Trap non-secure EL1 accesses to IC{C, V}_* for Group 1 interrupts */
#define ICH_HCR_EL2_TALL0 (1 << 11) /* Trap non-secure EL1 accesses to IC{C, V}_* for Group 0 interrupts */
#define ICH_HCR_EL2_TC (1 << 10) /* Trap non-secure EL1 accesses to common IC{C, V}_* registers */
#define ICH_HCR_EL2_VGRP1DIE (1 << 7) /* VM Group 1 Disabled Interrupt Enable */
#define ICH_HCR_EL2_VGRP1EIE (1 << 6) /* VM Group 1 Enabled Interrupt Enable */
#define ICH_HCR_EL2_VGRP0DIE (1 << 5) /* VM Group 0 Disabled Interrupt Enable */
#define ICH_HCR_EL2_VGRP0EIE (1 << 4) /* VM Group 0 Enabled Interrupt Enable */
#define ICH_HCR_EL2_NPIE (1 << 3) /* No Pending Interrupt Enable */
#define ICH_HCR_EL2_LRENPIE (1 << 2) /* List Register Entry Not Present Interrupt Enable */
#define ICH_HCR_EL2_UIE (1 << 1) /* Underflow Interrupt Enable */
#define ICH_HCR_EL2_En (1 << 0) /* Global enable for the virtual CPU interface */
/* Interrupt Controller List Registers */
#define ICH_LR_EL2_VINTID_MASK 0xffffffff
#define ICH_LR_EL2_VINTID(x) ((x) & ICH_LR_EL2_VINTID_MASK)
#define ICH_LR_EL2_PINTID_SHIFT 32
#define ICH_LR_EL2_PINTID_MASK (0x3fUL << ICH_LR_EL2_PINTID_SHIFT)
/* Raise a maintanance IRQ when deactivated (only non-HW virqs) */
#define ICH_LR_EL2_EOI (1UL << 41)
#define ICH_LR_EL2_PRIO_SHIFT 48
#define ICH_LR_EL2_PRIO_MASK (0xffUL << ICH_LR_EL2_PRIO_SHIFT)
#define ICH_LR_EL2_GROUP_SHIFT 60
#define ICH_LR_EL2_GROUP1 (1UL << ICH_LR_EL2_GROUP_SHIFT)
#define ICH_LR_EL2_HW (1UL << 61)
#define ICH_LR_EL2_STATE_SHIFT 62
#define ICH_LR_EL2_STATE_MASK (0x3UL << ICH_LR_EL2_STATE_SHIFT)
#define ICH_LR_EL2_STATE(x) ((x) & ICH_LR_EL2_STATE_MASK)
#define ICH_LR_EL2_STATE_INACTIVE (0x0UL << ICH_LR_EL2_STATE_SHIFT)
#define ICH_LR_EL2_STATE_PENDING (0x1UL << ICH_LR_EL2_STATE_SHIFT)
#define ICH_LR_EL2_STATE_ACTIVE (0x2UL << ICH_LR_EL2_STATE_SHIFT)
#define ICH_LR_EL2_STATE_PENDING_ACTIVE (0x3UL << ICH_LR_EL2_STATE_SHIFT)
/* Interrupt Controller Maintenance Interrupt State Register */
#define ICH_MISR_EL2_VGRP1D (1 << 7) /* vPE Group 1 Disabled */
#define ICH_MISR_EL2_VGRP1E (1 << 6) /* vPE Group 1 Enabled */
#define ICH_MISR_EL2_VGRP0D (1 << 5) /* vPE Group 0 Disabled */
#define ICH_MISR_EL2_VGRP0E (1 << 4) /* vPE Group 0 Enabled */
#define ICH_MISR_EL2_NP (1 << 3) /* No Pending */
#define ICH_MISR_EL2_LRENP (1 << 2) /* List Register Entry Not Present */
#define ICH_MISR_EL2_U (1 << 1) /* Underflow */
#define ICH_MISR_EL2_EOI (1 << 0) /* End Of Interrupt */
/* Interrupt Controller Virtual Machine Control Register */
#define ICH_VMCR_EL2_VPMR_SHIFT 24
#define ICH_VMCR_EL2_VPMR_MASK (0xff << ICH_VMCR_EL2_VPMR_SHIFT)
#define ICH_VMCR_EL2_VPMR_PRIO_LOWEST (0xff << ICH_VMCR_EL2_VPMR_SHIFT)
#define ICH_VMCR_EL2_VPMR_PRIO_HIGHEST (0x00 << ICH_VMCR_EL2_VPMR_SHIFT)
#define ICH_VMCR_EL2_VBPR0_SHIFT 21
#define ICH_VMCR_EL2_VBPR0_MASK (0x7 << ICH_VMCR_EL2_VBPR0_SHIFT)
#define ICH_VMCR_EL2_VBPR0_NO_PREEMPTION \
(0x7 << ICH_VMCR_EL2_VBPR0_SHIFT)
#define ICH_VMCR_EL2_VBPR1_SHIFT 18
#define ICH_VMCR_EL2_VBPR1_MASK (0x7 << ICH_VMCR_EL2_VBPR1_SHIFT)
#define ICH_VMCR_EL2_VBPR1_NO_PREEMPTION \
(0x7 << ICH_VMCR_EL2_VBPR1_SHIFT)
#define ICH_VMCR_EL2_VEOIM (1 << 9) /* Virtual EOI mode */
#define ICH_VMCR_EL2_VCBPR (1 << 4) /* Virtual Common binary Point Register */
#define ICH_VMCR_EL2_VFIQEN (1 << 3) /* Virtual FIQ enable */
#define ICH_VMCR_EL2_VACKCTL (1 << 2) /* Virtual AckCtl */
#define ICH_VMCR_EL2_VENG1 (1 << 1) /* Virtual Group 1 Interrupt Enable */
#define ICH_VMCR_EL2_VENG0 (1 << 0) /* Virtual Group 0 Interrupt Enable */
/* Interrupt Controller VGIC Type Register */
#define ICH_VTR_EL2_PRIBITS_SHIFT 29
#define ICH_VTR_EL2_PRIBITS_MASK (0x7 << ICH_VTR_EL2_PRIBITS_SHIFT)
#define ICH_VTR_EL2_PRIBITS(x) \
((((x) & ICH_VTR_EL2_PRIBITS_MASK) >> ICH_VTR_EL2_PRIBITS_SHIFT) + 1)
#define ICH_VTR_EL2_PREBITS_SHIFT 26
#define ICH_VTR_EL2_PREBITS_MASK (0x7 << ICH_VTR_EL2_PREBITS_SHIFT)
#define ICH_VTR_EL2_PREBITS(x) \
(((x) & ICH_VTR_EL2_PREBITS_MASK) >> ICH_VTR_EL2_PREBITS_SHIFT)
#define ICH_VTR_EL2_SEIS (1 << 22) /* System Error Interrupt (SEI) Support */
#define ICH_VTR_EL2_A3V (1 << 21) /* Affinity 3 Valid */
#define ICH_VTR_EL2_NV4 (1 << 20) /* Direct injection of virtual interrupts. RES1 for GICv3 */
#define ICH_VTR_EL2_TDS (1 << 19) /* Implementation supports ICH_HCR_EL2.TDIR */
#define ICH_VTR_EL2_LISTREGS_MASK 0x1f
/*
* ICH_VTR_EL2.ListRegs holds the number of list registers, minus one. Add one
* to get the actual number of list registers.
*/
#define ICH_VTR_EL2_LISTREGS(x) (((x) & ICH_VTR_EL2_LISTREGS_MASK) + 1)
#endif /* !_VGIC_V3_REG_H_ */

503
sys/arm64/vmm/io/vtimer.c Normal file
View file

@ -0,0 +1,503 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2017 The FreeBSD Foundation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the company nor the name of the author may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/rman.h>
#include <sys/time.h>
#include <sys/timeet.h>
#include <sys/timetc.h>
#include <machine/bus.h>
#include <machine/machdep.h>
#include <machine/vmm.h>
#include <machine/armreg.h>
#include <arm64/vmm/arm64.h>
#include "vgic.h"
#include "vtimer.h"
#define RES1 0xffffffffffffffffUL
#define timer_enabled(ctl) \
(!((ctl) & CNTP_CTL_IMASK) && ((ctl) & CNTP_CTL_ENABLE))
static uint64_t cnthctl_el2_reg;
static uint32_t tmr_frq;
#define timer_condition_met(ctl) ((ctl) & CNTP_CTL_ISTATUS)
static void vtimer_schedule_irq(struct hypctx *hypctx, bool phys);
static int
vtimer_virtual_timer_intr(void *arg)
{
struct hypctx *hypctx;
uint64_t cntpct_el0;
uint32_t cntv_ctl;
hypctx = arm64_get_active_vcpu();
cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
if (!hypctx) {
/* vm_destroy() was called. */
eprintf("No active vcpu\n");
cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
goto out;
}
if (!timer_enabled(cntv_ctl)) {
eprintf("Timer not enabled\n");
goto out;
}
if (!timer_condition_met(cntv_ctl)) {
eprintf("Timer condition not met\n");
goto out;
}
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hypctx->hyp->vtimer.cntvoff_el2;
if (hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 < cntpct_el0)
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
GT_VIRT_IRQ, true);
cntv_ctl = hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0;
out:
/*
* Disable the timer interrupt. This will prevent the interrupt from
* being reasserted as soon as we exit the handler and getting stuck
* in an infinite loop.
*
* This is safe to do because the guest disabled the timer, and then
* enables it as part of the interrupt handling routine.
*/
cntv_ctl &= ~CNTP_CTL_ENABLE;
WRITE_SPECIALREG(cntv_ctl_el0, cntv_ctl);
return (FILTER_HANDLED);
}
int
vtimer_init(uint64_t cnthctl_el2)
{
cnthctl_el2_reg = cnthctl_el2;
/*
* The guest *MUST* use the same timer frequency as the host. The
* register CNTFRQ_EL0 is accessible to the guest and a different value
* in the guest dts file might have unforseen consequences.
*/
tmr_frq = READ_SPECIALREG(cntfrq_el0);
return (0);
}
void
vtimer_vminit(struct hyp *hyp)
{
uint64_t now;
/*
* Configure the Counter-timer Hypervisor Control Register for the VM.
*
* CNTHCTL_EL1PCEN: trap access to CNTP_{CTL, CVAL, TVAL}_EL0 from EL1
* CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0
*/
hyp->vtimer.cnthctl_el2 = cnthctl_el2_reg & ~CNTHCTL_EL1PCEN;
hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCTEN;
now = READ_SPECIALREG(cntpct_el0);
hyp->vtimer.cntvoff_el2 = now;
return;
}
void
vtimer_cpuinit(struct hypctx *hypctx)
{
struct vtimer_cpu *vtimer_cpu;
vtimer_cpu = &hypctx->vtimer_cpu;
/*
* Configure physical timer interrupts for the VCPU.
*
* CNTP_CTL_IMASK: mask interrupts
* ~CNTP_CTL_ENABLE: disable the timer
*/
vtimer_cpu->phys_timer.cntx_ctl_el0 = CNTP_CTL_IMASK & ~CNTP_CTL_ENABLE;
mtx_init(&vtimer_cpu->phys_timer.mtx, "vtimer phys callout mutex", NULL,
MTX_DEF);
callout_init_mtx(&vtimer_cpu->phys_timer.callout,
&vtimer_cpu->phys_timer.mtx, 0);
vtimer_cpu->phys_timer.irqid = GT_PHYS_NS_IRQ;
mtx_init(&vtimer_cpu->virt_timer.mtx, "vtimer virt callout mutex", NULL,
MTX_DEF);
callout_init_mtx(&vtimer_cpu->virt_timer.callout,
&vtimer_cpu->virt_timer.mtx, 0);
vtimer_cpu->virt_timer.irqid = GT_VIRT_IRQ;
}
void
vtimer_cpucleanup(struct hypctx *hypctx)
{
struct vtimer_cpu *vtimer_cpu;
vtimer_cpu = &hypctx->vtimer_cpu;
callout_drain(&vtimer_cpu->phys_timer.callout);
callout_drain(&vtimer_cpu->virt_timer.callout);
mtx_destroy(&vtimer_cpu->phys_timer.mtx);
mtx_destroy(&vtimer_cpu->virt_timer.mtx);
}
void
vtimer_vmcleanup(struct hyp *hyp)
{
struct hypctx *hypctx;
uint32_t cntv_ctl;
hypctx = arm64_get_active_vcpu();
if (!hypctx) {
/* The active VM was destroyed, stop the timer. */
cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
cntv_ctl &= ~CNTP_CTL_ENABLE;
WRITE_SPECIALREG(cntv_ctl_el0, cntv_ctl);
}
}
void
vtimer_cleanup(void)
{
}
void
vtimer_sync_hwstate(struct hypctx *hypctx)
{
struct vtimer_timer *timer;
uint64_t cntpct_el0;
timer = &hypctx->vtimer_cpu.virt_timer;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hypctx->hyp->vtimer.cntvoff_el2;
if (!timer_enabled(timer->cntx_ctl_el0)) {
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
timer->irqid, false);
} else if (timer->cntx_cval_el0 < cntpct_el0) {
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
timer->irqid, true);
} else {
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
timer->irqid, false);
vtimer_schedule_irq(hypctx, false);
}
}
static void
vtimer_inject_irq_callout_phys(void *context)
{
struct hypctx *hypctx;
hypctx = context;
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
hypctx->vtimer_cpu.phys_timer.irqid, true);
}
static void
vtimer_inject_irq_callout_virt(void *context)
{
struct hypctx *hypctx;
hypctx = context;
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
hypctx->vtimer_cpu.virt_timer.irqid, true);
}
static void
vtimer_schedule_irq(struct hypctx *hypctx, bool phys)
{
sbintime_t time;
struct vtimer_timer *timer;
uint64_t cntpct_el0;
uint64_t diff;
if (phys)
timer = &hypctx->vtimer_cpu.phys_timer;
else
timer = &hypctx->vtimer_cpu.virt_timer;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hypctx->hyp->vtimer.cntvoff_el2;
if (timer->cntx_cval_el0 < cntpct_el0) {
/* Timer set in the past, trigger interrupt */
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
timer->irqid, true);
} else {
diff = timer->cntx_cval_el0 - cntpct_el0;
time = diff * SBT_1S / tmr_frq;
if (phys)
callout_reset_sbt(&timer->callout, time, 0,
vtimer_inject_irq_callout_phys, hypctx, 0);
else
callout_reset_sbt(&timer->callout, time, 0,
vtimer_inject_irq_callout_virt, hypctx, 0);
}
}
static void
vtimer_remove_irq(struct hypctx *hypctx, struct vcpu *vcpu)
{
struct vtimer_cpu *vtimer_cpu;
struct vtimer_timer *timer;
vtimer_cpu = &hypctx->vtimer_cpu;
timer = &vtimer_cpu->phys_timer;
callout_drain(&timer->callout);
/*
* The interrupt needs to be deactivated here regardless of the callout
* function having been executed. The timer interrupt can be masked with
* the CNTP_CTL_EL0.IMASK bit instead of reading the IAR register.
* Masking the interrupt doesn't remove it from the list registers.
*/
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(vcpu), timer->irqid, false);
}
/*
* Timer emulation functions.
*
* The guest should use the virtual timer, however some software, e.g. u-boot,
* used the physical timer. Emulate this in software for the guest to use.
*
* Adjust for cntvoff_el2 so the physical and virtual timers are at similar
* times. This simplifies interrupt handling in the virtual timer as the
* adjustment will have already happened.
*/
int
vtimer_phys_ctl_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
{
struct hyp *hyp;
struct hypctx *hypctx;
struct vtimer_cpu *vtimer_cpu;
uint64_t cntpct_el0;
hypctx = vcpu_get_cookie(vcpu);
hyp = hypctx->hyp;
vtimer_cpu = &hypctx->vtimer_cpu;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
if (vtimer_cpu->phys_timer.cntx_cval_el0 < cntpct_el0)
/* Timer condition met */
*rval = vtimer_cpu->phys_timer.cntx_ctl_el0 | CNTP_CTL_ISTATUS;
else
*rval = vtimer_cpu->phys_timer.cntx_ctl_el0 & ~CNTP_CTL_ISTATUS;
return (0);
}
int
vtimer_phys_ctl_write(struct vcpu *vcpu, uint64_t wval, void *arg)
{
struct hypctx *hypctx;
struct vtimer_cpu *vtimer_cpu;
uint64_t ctl_el0;
bool timer_toggled_on;
hypctx = vcpu_get_cookie(vcpu);
vtimer_cpu = &hypctx->vtimer_cpu;
timer_toggled_on = false;
ctl_el0 = vtimer_cpu->phys_timer.cntx_ctl_el0;
if (!timer_enabled(ctl_el0) && timer_enabled(wval))
timer_toggled_on = true;
else if (timer_enabled(ctl_el0) && !timer_enabled(wval))
vtimer_remove_irq(hypctx, vcpu);
vtimer_cpu->phys_timer.cntx_ctl_el0 = wval;
if (timer_toggled_on)
vtimer_schedule_irq(hypctx, true);
return (0);
}
int
vtimer_phys_cnt_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
{
struct vm *vm;
struct hyp *hyp;
vm = vcpu_vm(vcpu);
hyp = vm_get_cookie(vm);
*rval = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
return (0);
}
int
vtimer_phys_cnt_write(struct vcpu *vcpu, uint64_t wval, void *arg)
{
return (0);
}
int
vtimer_phys_cval_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
{
struct hypctx *hypctx;
struct vtimer_cpu *vtimer_cpu;
hypctx = vcpu_get_cookie(vcpu);
vtimer_cpu = &hypctx->vtimer_cpu;
*rval = vtimer_cpu->phys_timer.cntx_cval_el0;
return (0);
}
int
vtimer_phys_cval_write(struct vcpu *vcpu, uint64_t wval, void *arg)
{
struct hypctx *hypctx;
struct vtimer_cpu *vtimer_cpu;
hypctx = vcpu_get_cookie(vcpu);
vtimer_cpu = &hypctx->vtimer_cpu;
vtimer_cpu->phys_timer.cntx_cval_el0 = wval;
vtimer_remove_irq(hypctx, vcpu);
if (timer_enabled(vtimer_cpu->phys_timer.cntx_ctl_el0)) {
vtimer_schedule_irq(hypctx, true);
}
return (0);
}
int
vtimer_phys_tval_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
{
struct hyp *hyp;
struct hypctx *hypctx;
struct vtimer_cpu *vtimer_cpu;
uint32_t cntpct_el0;
hypctx = vcpu_get_cookie(vcpu);
hyp = hypctx->hyp;
vtimer_cpu = &hypctx->vtimer_cpu;
if (!(vtimer_cpu->phys_timer.cntx_ctl_el0 & CNTP_CTL_ENABLE)) {
/*
* ARMv8 Architecture Manual, p. D7-2702: the result of reading
* TVAL when the timer is disabled is UNKNOWN. I have chosen to
* return the maximum value possible on 32 bits which means the
* timer will fire very far into the future.
*/
*rval = (uint32_t)RES1;
} else {
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hyp->vtimer.cntvoff_el2;
*rval = vtimer_cpu->phys_timer.cntx_cval_el0 - cntpct_el0;
}
return (0);
}
int
vtimer_phys_tval_write(struct vcpu *vcpu, uint64_t wval, void *arg)
{
struct hyp *hyp;
struct hypctx *hypctx;
struct vtimer_cpu *vtimer_cpu;
uint64_t cntpct_el0;
hypctx = vcpu_get_cookie(vcpu);
hyp = hypctx->hyp;
vtimer_cpu = &hypctx->vtimer_cpu;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
vtimer_cpu->phys_timer.cntx_cval_el0 = (int32_t)wval + cntpct_el0;
vtimer_remove_irq(hypctx, vcpu);
if (timer_enabled(vtimer_cpu->phys_timer.cntx_ctl_el0)) {
vtimer_schedule_irq(hypctx, true);
}
return (0);
}
struct vtimer_softc {
struct resource *res;
void *ihl;
int rid;
};
static int
vtimer_probe(device_t dev)
{
device_set_desc(dev, "Virtual timer");
return (BUS_PROBE_DEFAULT);
}
static int
vtimer_attach(device_t dev)
{
struct vtimer_softc *sc;
sc = device_get_softc(dev);
sc->rid = 0;
sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->rid, RF_ACTIVE);
if (sc->res == NULL)
return (ENXIO);
bus_setup_intr(dev, sc->res, INTR_TYPE_CLK, vtimer_virtual_timer_intr,
NULL, NULL, &sc->ihl);
return (0);
}
static device_method_t vtimer_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, vtimer_probe),
DEVMETHOD(device_attach, vtimer_attach),
/* End */
DEVMETHOD_END
};
DEFINE_CLASS_0(vtimer, vtimer_driver, vtimer_methods,
sizeof(struct vtimer_softc));
DRIVER_MODULE(vtimer, generic_timer, vtimer_driver, 0, 0);

85
sys/arm64/vmm/io/vtimer.h Normal file
View file

@ -0,0 +1,85 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2017 The FreeBSD Foundation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the company nor the name of the author may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_VTIMER_H_
#define _VMM_VTIMER_H_
#define GT_PHYS_NS_IRQ 30
#define GT_VIRT_IRQ 27
struct hyp;
struct hypctx;
struct vtimer {
uint64_t cnthctl_el2;
uint64_t cntvoff_el2;
};
struct vtimer_timer {
struct callout callout;
struct mtx mtx;
uint32_t irqid;
/*
* These registers are either emulated for the physical timer, or
* the guest has full access to them for the virtual timer.
* CNTx_CTL_EL0: Counter-timer Timer Control Register
* CNTx_CVAL_EL0: Counter-timer Timer CompareValue Register
*/
uint64_t cntx_cval_el0;
uint64_t cntx_ctl_el0;
};
struct vtimer_cpu {
struct vtimer_timer phys_timer;
struct vtimer_timer virt_timer;
uint32_t cntkctl_el1;
};
int vtimer_init(uint64_t cnthctl_el2);
void vtimer_vminit(struct hyp *);
void vtimer_cpuinit(struct hypctx *);
void vtimer_cpucleanup(struct hypctx *);
void vtimer_vmcleanup(struct hyp *);
void vtimer_cleanup(void);
void vtimer_sync_hwstate(struct hypctx *hypctx);
int vtimer_phys_ctl_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
int vtimer_phys_ctl_write(struct vcpu *vcpu, uint64_t wval, void *arg);
int vtimer_phys_cnt_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
int vtimer_phys_cnt_write(struct vcpu *vcpu, uint64_t wval, void *arg);
int vtimer_phys_cval_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
int vtimer_phys_cval_write(struct vcpu *vcpu, uint64_t wval, void *arg);
int vtimer_phys_tval_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
int vtimer_phys_tval_write(struct vcpu *vcpu, uint64_t wval, void *arg);
#endif

52
sys/arm64/vmm/mmu.h Normal file
View file

@ -0,0 +1,52 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
*
* This software was developed by Alexandru Elisei under sponsorship
* from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_MMU_H_
#define _VMM_MMU_H_
#include <machine/machdep.h>
#include <machine/vmparam.h>
#include <machine/vmm.h>
#include "hyp.h"
extern char vmm_hyp_code;
extern char vmm_hyp_code_end;
extern char _vmm_start;
extern char _vmm_end;
bool vmmpmap_init(void);
void vmmpmap_fini(void);
uint64_t vmmpmap_to_ttbr0(void);
bool vmmpmap_enter(vm_offset_t, vm_size_t, vm_paddr_t, vm_prot_t);
void vmmpmap_remove(vm_offset_t, vm_size_t, bool);
#endif

33
sys/arm64/vmm/reset.h Normal file
View file

@ -0,0 +1,33 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2018 Alexandru Elisei <alexandru.elisei@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_RESET_H_
#define _VMM_RESET_H_
void reset_vm_el01_regs(void *vcpu);
void reset_vm_el2_regs(void *vcpu);
#endif

1803
sys/arm64/vmm/vmm.c Normal file
View file

File diff suppressed because it is too large Load diff

1337
sys/arm64/vmm/vmm_arm64.c Normal file
View file

File diff suppressed because it is too large Load diff

39
sys/arm64/vmm/vmm_call.S Normal file
View file

@ -0,0 +1,39 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
*
* This software was developed by Alexandru Elisei under sponsorship
* from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <machine/asm.h>
.text
ENTRY(vmm_call_hyp)
hvc #0
ret
END(vmm_call_hyp)

1054
sys/arm64/vmm/vmm_dev.c Normal file
View file

File diff suppressed because it is too large Load diff

735
sys/arm64/vmm/vmm_hyp.c Normal file
View file

@ -0,0 +1,735 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Andrew Turner
*
* This work was supported by Innovate UK project 105694, "Digital Security
* by Design (DSbD) Technology Platform Prototype".
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include <sys/types.h>
#include <sys/proc.h>
#include <machine/armreg.h>
#include "arm64.h"
#include "hyp.h"
struct hypctx;
uint64_t vmm_hyp_enter(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t,
uint64_t, uint64_t, uint64_t);
uint64_t vmm_enter_guest(struct hypctx *);
static void
vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
{
uint64_t dfr0;
/* Store the guest VFP registers */
if (guest) {
/* Store the timer registers */
hypctx->vtimer_cpu.cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 =
READ_SPECIALREG(cntv_cval_el0);
hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 =
READ_SPECIALREG(cntv_ctl_el0);
/* Store the GICv3 registers */
hypctx->vgic_v3_regs.ich_eisr_el2 =
READ_SPECIALREG(ich_eisr_el2);
hypctx->vgic_v3_regs.ich_elrsr_el2 =
READ_SPECIALREG(ich_elrsr_el2);
hypctx->vgic_v3_regs.ich_hcr_el2 =
READ_SPECIALREG(ich_hcr_el2);
hypctx->vgic_v3_regs.ich_misr_el2 =
READ_SPECIALREG(ich_misr_el2);
hypctx->vgic_v3_regs.ich_vmcr_el2 =
READ_SPECIALREG(ich_vmcr_el2);
switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
#define STORE_LR(x) \
case x: \
hypctx->vgic_v3_regs.ich_lr_el2[x] = \
READ_SPECIALREG(ich_lr ## x ##_el2)
STORE_LR(15);
STORE_LR(14);
STORE_LR(13);
STORE_LR(12);
STORE_LR(11);
STORE_LR(10);
STORE_LR(9);
STORE_LR(8);
STORE_LR(7);
STORE_LR(6);
STORE_LR(5);
STORE_LR(4);
STORE_LR(3);
STORE_LR(2);
STORE_LR(1);
default:
STORE_LR(0);
#undef STORE_LR
}
switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
#define STORE_APR(x) \
case x: \
hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
READ_SPECIALREG(ich_ap0r ## x ##_el2); \
hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
READ_SPECIALREG(ich_ap1r ## x ##_el2)
STORE_APR(3);
STORE_APR(2);
STORE_APR(1);
default:
STORE_APR(0);
#undef STORE_APR
}
}
dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
#define STORE_DBG_BRP(x) \
case x: \
hypctx->dbgbcr_el1[x] = \
READ_SPECIALREG(dbgbcr ## x ## _el1); \
hypctx->dbgbvr_el1[x] = \
READ_SPECIALREG(dbgbvr ## x ## _el1)
STORE_DBG_BRP(15);
STORE_DBG_BRP(14);
STORE_DBG_BRP(13);
STORE_DBG_BRP(12);
STORE_DBG_BRP(11);
STORE_DBG_BRP(10);
STORE_DBG_BRP(9);
STORE_DBG_BRP(8);
STORE_DBG_BRP(7);
STORE_DBG_BRP(6);
STORE_DBG_BRP(5);
STORE_DBG_BRP(4);
STORE_DBG_BRP(3);
STORE_DBG_BRP(2);
STORE_DBG_BRP(1);
default:
STORE_DBG_BRP(0);
#undef STORE_DBG_BRP
}
switch (ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
#define STORE_DBG_WRP(x) \
case x: \
hypctx->dbgwcr_el1[x] = \
READ_SPECIALREG(dbgwcr ## x ## _el1); \
hypctx->dbgwvr_el1[x] = \
READ_SPECIALREG(dbgwvr ## x ## _el1)
STORE_DBG_WRP(15);
STORE_DBG_WRP(14);
STORE_DBG_WRP(13);
STORE_DBG_WRP(12);
STORE_DBG_WRP(11);
STORE_DBG_WRP(10);
STORE_DBG_WRP(9);
STORE_DBG_WRP(8);
STORE_DBG_WRP(7);
STORE_DBG_WRP(6);
STORE_DBG_WRP(5);
STORE_DBG_WRP(4);
STORE_DBG_WRP(3);
STORE_DBG_WRP(2);
STORE_DBG_WRP(1);
default:
STORE_DBG_WRP(0);
#undef STORE_DBG_WRP
}
/* Store the PMU registers */
hypctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0);
hypctx->pmccntr_el0 = READ_SPECIALREG(pmccntr_el0);
hypctx->pmccfiltr_el0 = READ_SPECIALREG(pmccfiltr_el0);
hypctx->pmcntenset_el0 = READ_SPECIALREG(pmcntenset_el0);
hypctx->pmintenset_el1 = READ_SPECIALREG(pmintenset_el1);
hypctx->pmovsset_el0 = READ_SPECIALREG(pmovsset_el0);
hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0);
switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
#define STORE_PMU(x) \
case (x + 1): \
hypctx->pmevcntr_el0[x] = \
READ_SPECIALREG(pmevcntr ## x ## _el0); \
hypctx->pmevtyper_el0[x] = \
READ_SPECIALREG(pmevtyper ## x ## _el0)
STORE_PMU(30);
STORE_PMU(29);
STORE_PMU(28);
STORE_PMU(27);
STORE_PMU(26);
STORE_PMU(25);
STORE_PMU(24);
STORE_PMU(23);
STORE_PMU(22);
STORE_PMU(21);
STORE_PMU(20);
STORE_PMU(19);
STORE_PMU(18);
STORE_PMU(17);
STORE_PMU(16);
STORE_PMU(15);
STORE_PMU(14);
STORE_PMU(13);
STORE_PMU(12);
STORE_PMU(11);
STORE_PMU(10);
STORE_PMU(9);
STORE_PMU(8);
STORE_PMU(7);
STORE_PMU(6);
STORE_PMU(5);
STORE_PMU(4);
STORE_PMU(3);
STORE_PMU(2);
STORE_PMU(1);
STORE_PMU(0);
default: /* N == 0 when only PMCCNTR_EL0 is available */
break;
#undef STORE_PMU
}
/* Store the special to from the trapframe */
hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1);
hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2);
hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2);
if (guest) {
hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2);
}
/* Store the guest special registers */
hypctx->elr_el1 = READ_SPECIALREG(elr_el1);
hypctx->sp_el0 = READ_SPECIALREG(sp_el0);
hypctx->tpidr_el0 = READ_SPECIALREG(tpidr_el0);
hypctx->tpidrro_el0 = READ_SPECIALREG(tpidrro_el0);
hypctx->tpidr_el1 = READ_SPECIALREG(tpidr_el1);
hypctx->vbar_el1 = READ_SPECIALREG(vbar_el1);
hypctx->actlr_el1 = READ_SPECIALREG(actlr_el1);
hypctx->afsr0_el1 = READ_SPECIALREG(afsr0_el1);
hypctx->afsr1_el1 = READ_SPECIALREG(afsr1_el1);
hypctx->amair_el1 = READ_SPECIALREG(amair_el1);
hypctx->contextidr_el1 = READ_SPECIALREG(contextidr_el1);
hypctx->cpacr_el1 = READ_SPECIALREG(cpacr_el1);
hypctx->csselr_el1 = READ_SPECIALREG(csselr_el1);
hypctx->esr_el1 = READ_SPECIALREG(esr_el1);
hypctx->far_el1 = READ_SPECIALREG(far_el1);
hypctx->mair_el1 = READ_SPECIALREG(mair_el1);
hypctx->mdccint_el1 = READ_SPECIALREG(mdccint_el1);
hypctx->mdscr_el1 = READ_SPECIALREG(mdscr_el1);
hypctx->par_el1 = READ_SPECIALREG(par_el1);
hypctx->sctlr_el1 = READ_SPECIALREG(sctlr_el1);
hypctx->spsr_el1 = READ_SPECIALREG(spsr_el1);
hypctx->tcr_el1 = READ_SPECIALREG(tcr_el1);
/* TODO: Support when this is not res0 */
hypctx->tcr2_el1 = 0;
hypctx->ttbr0_el1 = READ_SPECIALREG(ttbr0_el1);
hypctx->ttbr1_el1 = READ_SPECIALREG(ttbr1_el1);
hypctx->cptr_el2 = READ_SPECIALREG(cptr_el2);
hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2);
hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2);
hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2);
}
static void
vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
{
uint64_t dfr0;
/* Restore the special registers */
WRITE_SPECIALREG(elr_el1, hypctx->elr_el1);
WRITE_SPECIALREG(sp_el0, hypctx->sp_el0);
WRITE_SPECIALREG(tpidr_el0, hypctx->tpidr_el0);
WRITE_SPECIALREG(tpidrro_el0, hypctx->tpidrro_el0);
WRITE_SPECIALREG(tpidr_el1, hypctx->tpidr_el1);
WRITE_SPECIALREG(vbar_el1, hypctx->vbar_el1);
WRITE_SPECIALREG(actlr_el1, hypctx->actlr_el1);
WRITE_SPECIALREG(afsr0_el1, hypctx->afsr0_el1);
WRITE_SPECIALREG(afsr1_el1, hypctx->afsr1_el1);
WRITE_SPECIALREG(amair_el1, hypctx->amair_el1);
WRITE_SPECIALREG(contextidr_el1, hypctx->contextidr_el1);
WRITE_SPECIALREG(cpacr_el1, hypctx->cpacr_el1);
WRITE_SPECIALREG(csselr_el1, hypctx->csselr_el1);
WRITE_SPECIALREG(esr_el1, hypctx->esr_el1);
WRITE_SPECIALREG(far_el1, hypctx->far_el1);
WRITE_SPECIALREG(mdccint_el1, hypctx->mdccint_el1);
WRITE_SPECIALREG(mdscr_el1, hypctx->mdscr_el1);
WRITE_SPECIALREG(mair_el1, hypctx->mair_el1);
WRITE_SPECIALREG(par_el1, hypctx->par_el1);
WRITE_SPECIALREG(sctlr_el1, hypctx->sctlr_el1);
WRITE_SPECIALREG(tcr_el1, hypctx->tcr_el1);
/* TODO: tcr2_el1 */
WRITE_SPECIALREG(ttbr0_el1, hypctx->ttbr0_el1);
WRITE_SPECIALREG(ttbr1_el1, hypctx->ttbr1_el1);
WRITE_SPECIALREG(spsr_el1, hypctx->spsr_el1);
WRITE_SPECIALREG(cptr_el2, hypctx->cptr_el2);
WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2);
WRITE_SPECIALREG(vpidr_el2, hypctx->vpidr_el2);
WRITE_SPECIALREG(vmpidr_el2, hypctx->vmpidr_el2);
/* Load the special regs from the trapframe */
WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp);
WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr);
WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr);
/* Restore the PMU registers */
WRITE_SPECIALREG(pmcr_el0, hypctx->pmcr_el0);
WRITE_SPECIALREG(pmccntr_el0, hypctx->pmccntr_el0);
WRITE_SPECIALREG(pmccfiltr_el0, hypctx->pmccfiltr_el0);
/* Clear all events/interrupts then enable them */
WRITE_SPECIALREG(pmcntenclr_el0, 0xfffffffful);
WRITE_SPECIALREG(pmcntenset_el0, hypctx->pmcntenset_el0);
WRITE_SPECIALREG(pmintenclr_el1, 0xfffffffful);
WRITE_SPECIALREG(pmintenset_el1, hypctx->pmintenset_el1);
WRITE_SPECIALREG(pmovsclr_el0, 0xfffffffful);
WRITE_SPECIALREG(pmovsset_el0, hypctx->pmovsset_el0);
switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
#define LOAD_PMU(x) \
case (x + 1): \
WRITE_SPECIALREG(pmevcntr ## x ## _el0, \
hypctx->pmevcntr_el0[x]); \
WRITE_SPECIALREG(pmevtyper ## x ## _el0, \
hypctx->pmevtyper_el0[x])
LOAD_PMU(30);
LOAD_PMU(29);
LOAD_PMU(28);
LOAD_PMU(27);
LOAD_PMU(26);
LOAD_PMU(25);
LOAD_PMU(24);
LOAD_PMU(23);
LOAD_PMU(22);
LOAD_PMU(21);
LOAD_PMU(20);
LOAD_PMU(19);
LOAD_PMU(18);
LOAD_PMU(17);
LOAD_PMU(16);
LOAD_PMU(15);
LOAD_PMU(14);
LOAD_PMU(13);
LOAD_PMU(12);
LOAD_PMU(11);
LOAD_PMU(10);
LOAD_PMU(9);
LOAD_PMU(8);
LOAD_PMU(7);
LOAD_PMU(6);
LOAD_PMU(5);
LOAD_PMU(4);
LOAD_PMU(3);
LOAD_PMU(2);
LOAD_PMU(1);
LOAD_PMU(0);
default: /* N == 0 when only PMCCNTR_EL0 is available */
break;
#undef LOAD_PMU
}
dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
#define LOAD_DBG_BRP(x) \
case x: \
WRITE_SPECIALREG(dbgbcr ## x ## _el1, \
hypctx->dbgbcr_el1[x]); \
WRITE_SPECIALREG(dbgbvr ## x ## _el1, \
hypctx->dbgbvr_el1[x])
LOAD_DBG_BRP(15);
LOAD_DBG_BRP(14);
LOAD_DBG_BRP(13);
LOAD_DBG_BRP(12);
LOAD_DBG_BRP(11);
LOAD_DBG_BRP(10);
LOAD_DBG_BRP(9);
LOAD_DBG_BRP(8);
LOAD_DBG_BRP(7);
LOAD_DBG_BRP(6);
LOAD_DBG_BRP(5);
LOAD_DBG_BRP(4);
LOAD_DBG_BRP(3);
LOAD_DBG_BRP(2);
LOAD_DBG_BRP(1);
default:
LOAD_DBG_BRP(0);
#undef LOAD_DBG_BRP
}
switch (ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
#define LOAD_DBG_WRP(x) \
case x: \
WRITE_SPECIALREG(dbgwcr ## x ## _el1, \
hypctx->dbgwcr_el1[x]); \
WRITE_SPECIALREG(dbgwvr ## x ## _el1, \
hypctx->dbgwvr_el1[x])
LOAD_DBG_WRP(15);
LOAD_DBG_WRP(14);
LOAD_DBG_WRP(13);
LOAD_DBG_WRP(12);
LOAD_DBG_WRP(11);
LOAD_DBG_WRP(10);
LOAD_DBG_WRP(9);
LOAD_DBG_WRP(8);
LOAD_DBG_WRP(7);
LOAD_DBG_WRP(6);
LOAD_DBG_WRP(5);
LOAD_DBG_WRP(4);
LOAD_DBG_WRP(3);
LOAD_DBG_WRP(2);
LOAD_DBG_WRP(1);
default:
LOAD_DBG_WRP(0);
#undef LOAD_DBG_WRP
}
if (guest) {
/* Load the timer registers */
WRITE_SPECIALREG(cntkctl_el1, hypctx->vtimer_cpu.cntkctl_el1);
WRITE_SPECIALREG(cntv_cval_el0,
hypctx->vtimer_cpu.virt_timer.cntx_cval_el0);
WRITE_SPECIALREG(cntv_ctl_el0,
hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0);
WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2);
WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2);
/* Load the GICv3 registers */
WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
WRITE_SPECIALREG(ich_vmcr_el2,
hypctx->vgic_v3_regs.ich_vmcr_el2);
switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
#define LOAD_LR(x) \
case x: \
WRITE_SPECIALREG(ich_lr ## x ##_el2, \
hypctx->vgic_v3_regs.ich_lr_el2[x])
LOAD_LR(15);
LOAD_LR(14);
LOAD_LR(13);
LOAD_LR(12);
LOAD_LR(11);
LOAD_LR(10);
LOAD_LR(9);
LOAD_LR(8);
LOAD_LR(7);
LOAD_LR(6);
LOAD_LR(5);
LOAD_LR(4);
LOAD_LR(3);
LOAD_LR(2);
LOAD_LR(1);
default:
LOAD_LR(0);
#undef LOAD_LR
}
switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
#define LOAD_APR(x) \
case x: \
WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
hypctx->vgic_v3_regs.ich_ap1r_el2[x])
LOAD_APR(3);
LOAD_APR(2);
LOAD_APR(1);
default:
LOAD_APR(0);
#undef LOAD_APR
}
}
}
static uint64_t
vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
{
struct hypctx host_hypctx;
uint64_t cntvoff_el2;
uint64_t ich_hcr_el2, ich_vmcr_el2, cnthctl_el2, cntkctl_el1;
uint64_t ret;
uint64_t s1e1r, hpfar_el2;
bool hpfar_valid;
vmm_hyp_reg_store(&host_hypctx, NULL, false);
/* Save the host special registers */
cnthctl_el2 = READ_SPECIALREG(cnthctl_el2);
cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
cntvoff_el2 = READ_SPECIALREG(cntvoff_el2);
ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
vmm_hyp_reg_restore(hypctx, hyp, true);
/* Load the common hypervisor registers */
WRITE_SPECIALREG(vttbr_el2, hyp->vttbr_el2);
host_hypctx.mdcr_el2 = READ_SPECIALREG(mdcr_el2);
WRITE_SPECIALREG(mdcr_el2, hypctx->mdcr_el2);
/* Call into the guest */
ret = vmm_enter_guest(hypctx);
WRITE_SPECIALREG(mdcr_el2, host_hypctx.mdcr_el2);
isb();
/* Store the exit info */
hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2);
vmm_hyp_reg_store(hypctx, hyp, true);
hpfar_valid = true;
if (ret == EXCP_TYPE_EL1_SYNC) {
switch (ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) {
case EXCP_INSN_ABORT_L:
case EXCP_DATA_ABORT_L:
/*
* The hpfar_el2 register is valid for:
* - Translation and Access faults.
* - Translation, Access, and permission faults on
* the translation table walk on the stage 1 tables.
* - A stage 2 Address size fault.
*
* As we only need it in the first 2 cases we can just
* exclude it on permission faults that are not from
* the stage 1 table walk.
*
* TODO: Add a case for Arm erratum 834220.
*/
if ((hypctx->tf.tf_esr & ISS_DATA_S1PTW) != 0)
break;
switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
case ISS_DATA_DFSC_PF_L1:
case ISS_DATA_DFSC_PF_L2:
case ISS_DATA_DFSC_PF_L3:
hpfar_valid = false;
break;
}
break;
}
}
if (hpfar_valid) {
hypctx->exit_info.hpfar_el2 = READ_SPECIALREG(hpfar_el2);
} else {
/*
* TODO: There is a risk the at instruction could cause an
* exception here. We should handle it & return a failure.
*/
s1e1r =
arm64_address_translate_s1e1r(hypctx->exit_info.far_el2);
if (PAR_SUCCESS(s1e1r)) {
hpfar_el2 = (s1e1r & PAR_PA_MASK) >> PAR_PA_SHIFT;
hpfar_el2 <<= HPFAR_EL2_FIPA_SHIFT;
hypctx->exit_info.hpfar_el2 = hpfar_el2;
} else {
ret = EXCP_TYPE_REENTER;
}
}
vmm_hyp_reg_restore(&host_hypctx, NULL, false);
/* Restore the host special registers */
WRITE_SPECIALREG(ich_hcr_el2, ich_hcr_el2);
WRITE_SPECIALREG(ich_vmcr_el2, ich_vmcr_el2);
WRITE_SPECIALREG(cnthctl_el2, cnthctl_el2);
WRITE_SPECIALREG(cntkctl_el1, cntkctl_el1);
WRITE_SPECIALREG(cntvoff_el2, cntvoff_el2);
return (ret);
}
static uint64_t
vmm_hyp_read_reg(uint64_t reg)
{
switch (reg) {
case HYP_REG_ICH_VTR:
return (READ_SPECIALREG(ich_vtr_el2));
case HYP_REG_CNTHCTL:
return (READ_SPECIALREG(cnthctl_el2));
}
return (0);
}
static int
vmm_clean_s2_tlbi(void)
{
dsb(ishst);
__asm __volatile("tlbi alle1is");
dsb(ish);
return (0);
}
static int
vm_s2_tlbi_range(uint64_t vttbr, vm_offset_t sva, vm_size_t eva,
bool final_only)
{
uint64_t end, r, start;
uint64_t host_vttbr;
#define TLBI_VA_SHIFT 12
#define TLBI_VA_MASK ((1ul << 44) - 1)
#define TLBI_VA(addr) (((addr) >> TLBI_VA_SHIFT) & TLBI_VA_MASK)
#define TLBI_VA_L3_INCR (L3_SIZE >> TLBI_VA_SHIFT)
/* Switch to the guest vttbr */
/* TODO: Handle Cortex-A57/A72 erratum 131936 */
host_vttbr = READ_SPECIALREG(vttbr_el2);
WRITE_SPECIALREG(vttbr_el2, vttbr);
isb();
/*
* The CPU can cache the stage 1 + 2 combination so we need to ensure
* the stage 2 is invalidated first, then when this has completed we
* invalidate the stage 1 TLB. As we don't know which stage 1 virtual
* addresses point at the stage 2 IPA we need to invalidate the entire
* stage 1 TLB.
*/
start = TLBI_VA(sva);
end = TLBI_VA(eva);
for (r = start; r < end; r += TLBI_VA_L3_INCR) {
/* Invalidate the stage 2 TLB entry */
if (final_only)
__asm __volatile("tlbi ipas2le1is, %0" : : "r"(r));
else
__asm __volatile("tlbi ipas2e1is, %0" : : "r"(r));
}
/* Ensure the entry has been invalidated */
dsb(ish);
/* Invalidate the stage 1 TLB. */
__asm __volatile("tlbi vmalle1is");
dsb(ish);
isb();
/* Switch back t othe host vttbr */
WRITE_SPECIALREG(vttbr_el2, host_vttbr);
isb();
return (0);
}
static int
vm_s2_tlbi_all(uint64_t vttbr)
{
uint64_t host_vttbr;
/* Switch to the guest vttbr */
/* TODO: Handle Cortex-A57/A72 erratum 131936 */
host_vttbr = READ_SPECIALREG(vttbr_el2);
WRITE_SPECIALREG(vttbr_el2, vttbr);
isb();
__asm __volatile("tlbi vmalls12e1is");
dsb(ish);
isb();
/* Switch back t othe host vttbr */
WRITE_SPECIALREG(vttbr_el2, host_vttbr);
isb();
return (0);
}
static int
vmm_dc_civac(uint64_t start, uint64_t len)
{
size_t line_size, end;
uint64_t ctr;
ctr = READ_SPECIALREG(ctr_el0);
line_size = sizeof(int) << CTR_DLINE_SIZE(ctr);
end = start + len;
dsb(ishst);
/* Clean and Invalidate the D-cache */
for (; start < end; start += line_size)
__asm __volatile("dc civac, %0" :: "r" (start) : "memory");
dsb(ish);
return (0);
}
static int
vmm_el2_tlbi(uint64_t type, uint64_t start, uint64_t len)
{
uint64_t end, r;
dsb(ishst);
switch (type) {
default:
case HYP_EL2_TLBI_ALL:
__asm __volatile("tlbi alle2" ::: "memory");
break;
case HYP_EL2_TLBI_VA:
end = TLBI_VA(start + len);
start = TLBI_VA(start);
for (r = start; r < end; r += TLBI_VA_L3_INCR) {
__asm __volatile("tlbi vae2is, %0" :: "r"(r));
}
break;
}
dsb(ish);
return (0);
}
uint64_t
vmm_hyp_enter(uint64_t handle, uint64_t x1, uint64_t x2, uint64_t x3,
uint64_t x4, uint64_t x5, uint64_t x6, uint64_t x7)
{
uint64_t ret;
switch (handle) {
case HYP_ENTER_GUEST:
do {
ret = vmm_hyp_call_guest((struct hyp *)x1,
(struct hypctx *)x2);
} while (ret == EXCP_TYPE_REENTER);
return (ret);
case HYP_READ_REGISTER:
return (vmm_hyp_read_reg(x1));
case HYP_CLEAN_S2_TLBI:
return (vmm_clean_s2_tlbi());
case HYP_DC_CIVAC:
return (vmm_dc_civac(x1, x2));
case HYP_EL2_TLBI:
return (vmm_el2_tlbi(x1, x2, x3));
case HYP_S2_TLBI_RANGE:
return (vm_s2_tlbi_range(x1, x2, x3, x4));
case HYP_S2_TLBI_ALL:
return (vm_s2_tlbi_all(x1));
case HYP_CLEANUP: /* Handled in vmm_hyp_exception.S */
default:
break;
}
return (0);
}

39
sys/arm64/vmm/vmm_hyp_el2.S Normal file
View file

@ -0,0 +1,39 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2021 Andrew Turner
*
* This work was supported by Innovate UK project 105694, "Digital Security
* by Design (DSbD) Technology Platform Prototype".
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <machine/param.h>
.rodata
.align PAGE_SHIFT
.globl vmm_hyp_code
vmm_hyp_code:
.incbin "vmm_hyp_blob.bin"
.globl vmm_hyp_code_end
vmm_hyp_code_end:

384
sys/arm64/vmm/vmm_hyp_exception.S Normal file
View file

@ -0,0 +1,384 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
* Copyright (c) 2021 Andrew Turner
*
* This software was developed by Alexandru Elisei under sponsorship
* from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <machine/asm.h>
#include <machine/hypervisor.h>
#include "assym.inc"
#include "hyp.h"
.macro save_host_registers
/* TODO: Only store callee saved registers */
sub sp, sp, #(32 * 8)
str x30, [sp, #(30 * 8)]
stp x28, x29, [sp, #(28 * 8)]
stp x26, x27, [sp, #(26 * 8)]
stp x24, x25, [sp, #(24 * 8)]
stp x22, x23, [sp, #(22 * 8)]
stp x20, x21, [sp, #(20 * 8)]
stp x18, x19, [sp, #(18 * 8)]
stp x16, x17, [sp, #(16 * 8)]
stp x14, x15, [sp, #(14 * 8)]
stp x12, x13, [sp, #(12 * 8)]
stp x10, x11, [sp, #(10 * 8)]
stp x8, x9, [sp, #(8 * 8)]
stp x6, x7, [sp, #(6 * 8)]
stp x4, x5, [sp, #(4 * 8)]
stp x2, x3, [sp, #(2 * 8)]
stp x0, x1, [sp, #(0 * 8)]
.endm
.macro restore_host_registers
/* TODO: Only restore callee saved registers */
ldp x0, x1, [sp, #(0 * 8)]
ldp x2, x3, [sp, #(2 * 8)]
ldp x4, x5, [sp, #(4 * 8)]
ldp x6, x7, [sp, #(6 * 8)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(10 * 8)]
ldp x12, x13, [sp, #(12 * 8)]
ldp x14, x15, [sp, #(14 * 8)]
ldp x16, x17, [sp, #(16 * 8)]
ldp x18, x19, [sp, #(18 * 8)]
ldp x20, x21, [sp, #(20 * 8)]
ldp x22, x23, [sp, #(22 * 8)]
ldp x24, x25, [sp, #(24 * 8)]
ldp x26, x27, [sp, #(26 * 8)]
ldp x28, x29, [sp, #(28 * 8)]
ldr x30, [sp, #(30 * 8)]
add sp, sp, #(32 * 8)
.endm
.macro save_guest_registers
/* Back up x0 so we can use it as a temporary register */
stp x0, x1, [sp, #-(2 * 8)]!
/* Restore the hypctx pointer */
mrs x0, tpidr_el2
stp x2, x3, [x0, #(TF_X + 2 * 8)]
stp x4, x5, [x0, #(TF_X + 4 * 8)]
stp x6, x7, [x0, #(TF_X + 6 * 8)]
stp x8, x9, [x0, #(TF_X + 8 * 8)]
stp x10, x11, [x0, #(TF_X + 10 * 8)]
stp x12, x13, [x0, #(TF_X + 12 * 8)]
stp x14, x15, [x0, #(TF_X + 14 * 8)]
stp x16, x17, [x0, #(TF_X + 16 * 8)]
stp x18, x19, [x0, #(TF_X + 18 * 8)]
stp x20, x21, [x0, #(TF_X + 20 * 8)]
stp x22, x23, [x0, #(TF_X + 22 * 8)]
stp x24, x25, [x0, #(TF_X + 24 * 8)]
stp x26, x27, [x0, #(TF_X + 26 * 8)]
stp x28, x29, [x0, #(TF_X + 28 * 8)]
str lr, [x0, #(TF_LR)]
/* Restore the saved x0 & x1 and save them */
ldp x2, x3, [sp], #(2 * 8)
stp x2, x3, [x0, #(TF_X + 0 * 8)]
.endm
.macro restore_guest_registers
/*
* Copy the guest x0 and x1 to the stack so we can restore them
* after loading the other registers.
*/
ldp x2, x3, [x0, #(TF_X + 0 * 8)]
stp x2, x3, [sp, #-(2 * 8)]!
ldr lr, [x0, #(TF_LR)]
ldp x28, x29, [x0, #(TF_X + 28 * 8)]
ldp x26, x27, [x0, #(TF_X + 26 * 8)]
ldp x24, x25, [x0, #(TF_X + 24 * 8)]
ldp x22, x23, [x0, #(TF_X + 22 * 8)]
ldp x20, x21, [x0, #(TF_X + 20 * 8)]
ldp x18, x19, [x0, #(TF_X + 18 * 8)]
ldp x16, x17, [x0, #(TF_X + 16 * 8)]
ldp x14, x15, [x0, #(TF_X + 14 * 8)]
ldp x12, x13, [x0, #(TF_X + 12 * 8)]
ldp x10, x11, [x0, #(TF_X + 10 * 8)]
ldp x8, x9, [x0, #(TF_X + 8 * 8)]
ldp x6, x7, [x0, #(TF_X + 6 * 8)]
ldp x4, x5, [x0, #(TF_X + 4 * 8)]
ldp x2, x3, [x0, #(TF_X + 2 * 8)]
ldp x0, x1, [sp], #(2 * 8)
.endm
.macro vempty
.align 7
1: b 1b
.endm
.macro vector name
.align 7
b handle_\name
.endm
.section ".vmm_vectors","ax"
.align 11
hyp_init_vectors:
vempty /* Synchronous EL2t */
vempty /* IRQ EL2t */
vempty /* FIQ EL2t */
vempty /* Error EL2t */
vempty /* Synchronous EL2h */
vempty /* IRQ EL2h */
vempty /* FIQ EL2h */
vempty /* Error EL2h */
vector hyp_init /* Synchronous 64-bit EL1 */
vempty /* IRQ 64-bit EL1 */
vempty /* FIQ 64-bit EL1 */
vempty /* Error 64-bit EL1 */
vempty /* Synchronous 32-bit EL1 */
vempty /* IRQ 32-bit EL1 */
vempty /* FIQ 32-bit EL1 */
vempty /* Error 32-bit EL1 */
.text
.align 11
hyp_vectors:
vempty /* Synchronous EL2t */
vempty /* IRQ EL2t */
vempty /* FIQ EL2t */
vempty /* Error EL2t */
vector el2_el2h_sync /* Synchronous EL2h */
vector el2_el2h_irq /* IRQ EL2h */
vector el2_el2h_fiq /* FIQ EL2h */
vector el2_el2h_error /* Error EL2h */
vector el2_el1_sync64 /* Synchronous 64-bit EL1 */
vector el2_el1_irq64 /* IRQ 64-bit EL1 */
vector el2_el1_fiq64 /* FIQ 64-bit EL1 */
vector el2_el1_error64 /* Error 64-bit EL1 */
vempty /* Synchronous 32-bit EL1 */
vempty /* IRQ 32-bit EL1 */
vempty /* FIQ 32-bit EL1 */
vempty /* Error 32-bit EL1 */
/*
* Initialize the hypervisor mode with a new exception vector table, translation
* table and stack.
*
* Expecting:
* x0 - translation tables physical address
* x1 - stack top virtual address
* x2 - TCR_EL2 value
* x3 - SCTLR_EL2 value
* x4 - VTCR_EL2 value
*/
LENTRY(handle_hyp_init)
/* Install the new exception vectors */
adrp x6, hyp_vectors
add x6, x6, :lo12:hyp_vectors
msr vbar_el2, x6
/* Set the stack top address */
mov sp, x1
/* Use the host VTTBR_EL2 to tell the host and the guests apart */
mov x9, #VTTBR_HOST
msr vttbr_el2, x9
/* Load the base address for the translation tables */
msr ttbr0_el2, x0
/* Invalidate the TLB */
tlbi alle2
/* Use the same memory attributes as EL1 */
mrs x9, mair_el1
msr mair_el2, x9
/* Configure address translation */
msr tcr_el2, x2
isb
/* Set the system control register for EL2 */
msr sctlr_el2, x3
/* Set the Stage 2 translation control register */
msr vtcr_el2, x4
/* Return success */
mov x0, #0
/* MMU is up and running */
ERET
LEND(handle_hyp_init)
.macro do_world_switch_to_host
save_guest_registers
restore_host_registers
/* Restore host VTTBR */
mov x9, #VTTBR_HOST
msr vttbr_el2, x9
.endm
.macro handle_el2_excp type
/* Save registers before modifying so we can restore them */
str x9, [sp, #-16]!
/* Test if the exception happened when the host was running */
mrs x9, vttbr_el2
cmp x9, #VTTBR_HOST
beq 1f
/* We got the exception while the guest was running */
ldr x9, [sp], #16
do_world_switch_to_host
mov x0, \type
ret
1:
/* We got the exception while the host was running */
ldr x9, [sp], #16
mov x0, \type
ERET
.endm
LENTRY(handle_el2_el2h_sync)
handle_el2_excp #EXCP_TYPE_EL2_SYNC
LEND(handle_el2_el2h_sync)
LENTRY(handle_el2_el2h_irq)
handle_el2_excp #EXCP_TYPE_EL2_IRQ
LEND(handle_el2_el2h_irq)
LENTRY(handle_el2_el2h_fiq)
handle_el2_excp #EXCP_TYPE_EL2_FIQ
LEND(handle_el2_el2h_fiq)
LENTRY(handle_el2_el2h_error)
handle_el2_excp #EXCP_TYPE_EL2_ERROR
LEND(handle_el2_el2h_error)
LENTRY(handle_el2_el1_sync64)
/* Save registers before modifying so we can restore them */
str x9, [sp, #-16]!
/* Check for host hypervisor call */
mrs x9, vttbr_el2
cmp x9, #VTTBR_HOST
ldr x9, [sp], #16 /* Restore the temp register */
bne 1f
/*
* Called from the host
*/
/* Check if this is a cleanup call and handle in a controlled state */
cmp x0, #(HYP_CLEANUP)
b.eq vmm_cleanup
str lr, [sp, #-16]!
bl vmm_hyp_enter
ldr lr, [sp], #16
ERET
1: /* Guest exception taken to EL2 */
do_world_switch_to_host
mov x0, #EXCP_TYPE_EL1_SYNC
ret
LEND(handle_el2_el1_sync64)
/*
* We only trap IRQ, FIQ and SError exceptions when a guest is running. Do a
* world switch to host to handle these exceptions.
*/
LENTRY(handle_el2_el1_irq64)
do_world_switch_to_host
str x9, [sp, #-16]!
mrs x9, ich_misr_el2
cmp x9, xzr
beq 1f
mov x0, #EXCP_TYPE_MAINT_IRQ
b 2f
1:
mov x0, #EXCP_TYPE_EL1_IRQ
2:
ldr x9, [sp], #16
ret
LEND(handle_el2_el1_irq)
LENTRY(handle_el2_el1_fiq64)
do_world_switch_to_host
mov x0, #EXCP_TYPE_EL1_FIQ
ret
LEND(handle_el2_el1_fiq64)
LENTRY(handle_el2_el1_error64)
do_world_switch_to_host
mov x0, #EXCP_TYPE_EL1_ERROR
ret
LEND(handle_el2_el1_error64)
/*
* Usage:
* uint64_t vmm_enter_guest(struct hypctx *hypctx)
*
* Expecting:
* x0 - hypctx address
*/
ENTRY(vmm_enter_guest)
/* Save hypctx address */
msr tpidr_el2, x0
save_host_registers
restore_guest_registers
/* Enter guest */
ERET
END(vmm_enter_guest)
/*
* Usage:
* void vmm_cleanup(uint64_t handle, void *hyp_stub_vectors)
*
* Expecting:
* x1 - physical address of hyp_stub_vectors
*/
LENTRY(vmm_cleanup)
/* Restore the stub vectors */
msr vbar_el2, x1
/* Disable the MMU */
dsb sy
mrs x2, sctlr_el2
bic x2, x2, #SCTLR_EL2_M
msr sctlr_el2, x2
isb
ERET
LEND(vmm_cleanup)

102
sys/arm64/vmm/vmm_instruction_emul.c Normal file
View file

@ -0,0 +1,102 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifdef _KERNEL
#include <sys/param.h>
#include <sys/pcpu.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <vm/vm.h>
#include <machine/machdep.h>
#include <machine/vmm.h>
#else
#include <sys/types.h>
#include <sys/errno.h>
#include <sys/_iovec.h>
#include <machine/vmm.h>
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <vmmapi.h>
#endif
#include <machine/vmm_instruction_emul.h>
int
vmm_emulate_instruction(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
struct vm_guest_paging *paging __unused, mem_region_read_t memread,
mem_region_write_t memwrite, void *memarg)
{
uint64_t val;
int error;
if (vie->dir == VM_DIR_READ) {
error = memread(vcpu, gpa, &val, vie->access_size, memarg);
if (error)
goto out;
error = vm_set_register(vcpu, vie->reg, val);
} else {
error = vm_get_register(vcpu, vie->reg, &val);
if (error)
goto out;
/* Mask any unneeded bits from the register */
if (vie->access_size < 8)
val &= (1ul << (vie->access_size * 8)) - 1;
error = memwrite(vcpu, gpa, val, vie->access_size, memarg);
}
out:
return (error);
}
int
vmm_emulate_register(struct vcpu *vcpu, struct vre *vre, reg_read_t regread,
reg_write_t regwrite, void *regarg)
{
uint64_t val;
int error;
if (vre->dir == VM_DIR_READ) {
error = regread(vcpu, &val, regarg);
if (error)
goto out;
error = vm_set_register(vcpu, vre->reg, val);
} else {
error = vm_get_register(vcpu, vre->reg, &val);
if (error)
goto out;
error = regwrite(vcpu, val, regarg);
}
out:
return (error);
}

69
sys/arm64/vmm/vmm_ktr.h Normal file
View file

@ -0,0 +1,69 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2011 NetApp, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_KTR_H_
#define _VMM_KTR_H_
#include <sys/ktr.h>
#include <sys/pcpu.h>
#ifndef KTR_VMM
#define KTR_VMM KTR_GEN
#endif
#define VCPU_CTR0(vm, vcpuid, format) \
CTR2(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid))
#define VCPU_CTR1(vm, vcpuid, format, p1) \
CTR3(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1))
#define VCPU_CTR2(vm, vcpuid, format, p1, p2) \
CTR4(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1), (p2))
#define VCPU_CTR3(vm, vcpuid, format, p1, p2, p3) \
CTR5(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1), (p2), (p3))
#define VCPU_CTR4(vm, vcpuid, format, p1, p2, p3, p4) \
CTR6(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), \
(p1), (p2), (p3), (p4))
#define VM_CTR0(vm, format) \
CTR1(KTR_VMM, "vm %s: " format, vm_name((vm)))
#define VM_CTR1(vm, format, p1) \
CTR2(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1))
#define VM_CTR2(vm, format, p1, p2) \
CTR3(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2))
#define VM_CTR3(vm, format, p1, p2, p3) \
CTR4(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2), (p3))
#define VM_CTR4(vm, format, p1, p2, p3, p4) \
CTR5(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2), (p3), (p4))
#endif

430
sys/arm64/vmm/vmm_mmu.c Normal file
View file

@ -0,0 +1,430 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
*
* This software was developed by Alexandru Elisei under sponsorship
* from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include <sys/types.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_page.h>
#include <vm/vm_param.h>
#include <vm/vm_phys.h>
#include <machine/atomic.h>
#include <machine/machdep.h>
#include <machine/vm.h>
#include <machine/vmm.h>
#include <machine/vmparam.h>
#include "mmu.h"
#include "arm64.h"
static struct mtx vmmpmap_mtx;
static pt_entry_t *l0;
static vm_paddr_t l0_paddr;
bool
vmmpmap_init(void)
{
vm_page_t m;
m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
if (m == NULL)
return (false);
l0_paddr = VM_PAGE_TO_PHYS(m);
l0 = (pd_entry_t *)PHYS_TO_DMAP(l0_paddr);
mtx_init(&vmmpmap_mtx, "vmm pmap", NULL, MTX_DEF);
return (true);
}
static void
vmmpmap_release_l3(pd_entry_t l2e)
{
pt_entry_t *l3 __diagused;
vm_page_t m;
int i;
l3 = (pd_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
for (i = 0; i < Ln_ENTRIES; i++) {
KASSERT(l3[i] == 0, ("%s: l3 still mapped: %p %lx", __func__,
&l3[i], l3[i]));
}
m = PHYS_TO_VM_PAGE(l2e & ~ATTR_MASK);
vm_page_unwire_noq(m);
vm_page_free(m);
}
static void
vmmpmap_release_l2(pd_entry_t l1e)
{
pt_entry_t *l2;
vm_page_t m;
int i;
l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
for (i = 0; i < Ln_ENTRIES; i++) {
if (l2[i] != 0) {
vmmpmap_release_l3(l2[i]);
}
}
m = PHYS_TO_VM_PAGE(l1e & ~ATTR_MASK);
vm_page_unwire_noq(m);
vm_page_free(m);
}
static void
vmmpmap_release_l1(pd_entry_t l0e)
{
pt_entry_t *l1;
vm_page_t m;
int i;
l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
for (i = 0; i < Ln_ENTRIES; i++) {
if (l1[i] != 0) {
vmmpmap_release_l2(l1[i]);
}
}
m = PHYS_TO_VM_PAGE(l0e & ~ATTR_MASK);
vm_page_unwire_noq(m);
vm_page_free(m);
}
void
vmmpmap_fini(void)
{
vm_page_t m;
int i;
/* Remove the remaining entries */
for (i = 0; i < L0_ENTRIES; i++) {
if (l0[i] != 0) {
vmmpmap_release_l1(l0[i]);
}
}
m = PHYS_TO_VM_PAGE(l0_paddr);
vm_page_unwire_noq(m);
vm_page_free(m);
mtx_destroy(&vmmpmap_mtx);
}
uint64_t
vmmpmap_to_ttbr0(void)
{
return (l0_paddr);
}
/* Returns a pointer to the level 1 table, allocating if needed. */
static pt_entry_t *
vmmpmap_l1_table(vm_offset_t va)
{
pt_entry_t new_l0e, l0e, *l1;
vm_page_t m;
int rv;
m = NULL;
again:
l0e = atomic_load_64(&l0[pmap_l0_index(va)]);
if ((l0e & ATTR_DESCR_VALID) == 0) {
/* Allocate a page for the level 1 table */
if (m == NULL) {
m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
if (m == NULL)
return (NULL);
}
new_l0e = VM_PAGE_TO_PHYS(m) | L0_TABLE;
mtx_lock(&vmmpmap_mtx);
rv = atomic_cmpset_64(&l0[pmap_l0_index(va)], l0e, new_l0e);
mtx_unlock(&vmmpmap_mtx);
/* We may have raced another thread, try again */
if (rv == 0)
goto again;
/* The cmpset succeeded */
l0e = new_l0e;
} else if (m != NULL) {
/* We allocated a page that wasn't used */
vm_page_unwire_noq(m);
vm_page_free_zero(m);
}
l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
return (l1);
}
static pt_entry_t *
vmmpmap_l2_table(vm_offset_t va)
{
pt_entry_t new_l1e, l1e, *l1, *l2;
vm_page_t m;
int rv;
l1 = vmmpmap_l1_table(va);
if (l1 == NULL)
return (NULL);
m = NULL;
again:
l1e = atomic_load_64(&l1[pmap_l1_index(va)]);
if ((l1e & ATTR_DESCR_VALID) == 0) {
/* Allocate a page for the level 2 table */
if (m == NULL) {
m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
if (m == NULL)
return (NULL);
}
new_l1e = VM_PAGE_TO_PHYS(m) | L1_TABLE;
mtx_lock(&vmmpmap_mtx);
rv = atomic_cmpset_64(&l1[pmap_l1_index(va)], l1e, new_l1e);
mtx_unlock(&vmmpmap_mtx);
/* We may have raced another thread, try again */
if (rv == 0)
goto again;
/* The cmpset succeeded */
l1e = new_l1e;
} else if (m != NULL) {
/* We allocated a page that wasn't used */
vm_page_unwire_noq(m);
vm_page_free_zero(m);
}
l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
return (l2);
}
static pd_entry_t *
vmmpmap_l3_table(vm_offset_t va)
{
pt_entry_t new_l2e, l2e, *l2, *l3;
vm_page_t m;
int rv;
l2 = vmmpmap_l2_table(va);
if (l2 == NULL)
return (NULL);
m = NULL;
again:
l2e = atomic_load_64(&l2[pmap_l2_index(va)]);
if ((l2e & ATTR_DESCR_VALID) == 0) {
/* Allocate a page for the level 3 table */
if (m == NULL) {
m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
if (m == NULL)
return (NULL);
}
new_l2e = VM_PAGE_TO_PHYS(m) | L2_TABLE;
mtx_lock(&vmmpmap_mtx);
rv = atomic_cmpset_64(&l2[pmap_l2_index(va)], l2e, new_l2e);
mtx_unlock(&vmmpmap_mtx);
/* We may have raced another thread, try again */
if (rv == 0)
goto again;
/* The cmpset succeeded */
l2e = new_l2e;
} else if (m != NULL) {
/* We allocated a page that wasn't used */
vm_page_unwire_noq(m);
vm_page_free_zero(m);
}
l3 = (pt_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
return (l3);
}
/*
* Creates an EL2 entry in the hyp_pmap. Similar to pmap_kenter.
*/
bool
vmmpmap_enter(vm_offset_t va, vm_size_t size, vm_paddr_t pa, vm_prot_t prot)
{
pd_entry_t l3e, *l3;
KASSERT((pa & L3_OFFSET) == 0,
("%s: Invalid physical address", __func__));
KASSERT((va & L3_OFFSET) == 0,
("%s: Invalid virtual address", __func__));
KASSERT((size & PAGE_MASK) == 0,
("%s: Mapping is not page-sized", __func__));
l3e = ATTR_DEFAULT | L3_PAGE;
/* This bit is res1 at EL2 */
l3e |= ATTR_S1_AP(ATTR_S1_AP_USER);
/* Only normal memory is used at EL2 */
l3e |= ATTR_S1_IDX(VM_MEMATTR_DEFAULT);
if ((prot & VM_PROT_EXECUTE) == 0) {
/* PXN is res0 at EL2. UXN is XN */
l3e |= ATTR_S1_UXN;
}
if ((prot & VM_PROT_WRITE) == 0) {
l3e |= ATTR_S1_AP(ATTR_S1_AP_RO);
}
while (size > 0) {
l3 = vmmpmap_l3_table(va);
if (l3 == NULL)
return (false);
#ifdef INVARIANTS
/*
* Ensure no other threads can write to l3 between the KASSERT
* and store.
*/
mtx_lock(&vmmpmap_mtx);
#endif
KASSERT(atomic_load_64(&l3[pmap_l3_index(va)]) == 0,
("%s: VA already mapped", __func__));
atomic_store_64(&l3[pmap_l3_index(va)], l3e | pa);
#ifdef INVARIANTS
mtx_unlock(&vmmpmap_mtx);
#endif
size -= PAGE_SIZE;
pa += PAGE_SIZE;
va += PAGE_SIZE;
}
return (true);
}
void
vmmpmap_remove(vm_offset_t va, vm_size_t size, bool invalidate)
{
pt_entry_t l0e, *l1, l1e, *l2, l2e;
pd_entry_t *l3, l3e, **l3_list;
vm_offset_t eva, va_next, sva;
size_t i;
KASSERT((va & L3_OFFSET) == 0,
("%s: Invalid virtual address", __func__));
KASSERT((size & PAGE_MASK) == 0,
("%s: Mapping is not page-sized", __func__));
if (invalidate) {
l3_list = malloc((size / PAGE_SIZE) * sizeof(l3_list[0]),
M_TEMP, M_WAITOK | M_ZERO);
}
sva = va;
eva = va + size;
mtx_lock(&vmmpmap_mtx);
for (i = 0; va < eva; va = va_next) {
l0e = atomic_load_64(&l0[pmap_l0_index(va)]);
if (l0e == 0) {
va_next = (va + L0_SIZE) & ~L0_OFFSET;
if (va_next < va)
va_next = eva;
continue;
}
MPASS((l0e & ATTR_DESCR_MASK) == L0_TABLE);
l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
l1e = atomic_load_64(&l1[pmap_l1_index(va)]);
if (l1e == 0) {
va_next = (va + L1_SIZE) & ~L1_OFFSET;
if (va_next < va)
va_next = eva;
continue;
}
MPASS((l1e & ATTR_DESCR_MASK) == L1_TABLE);
l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
l2e = atomic_load_64(&l2[pmap_l2_index(va)]);
if (l2e == 0) {
va_next = (va + L2_SIZE) & ~L2_OFFSET;
if (va_next < va)
va_next = eva;
continue;
}
MPASS((l2e & ATTR_DESCR_MASK) == L2_TABLE);
l3 = (pd_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
if (invalidate) {
l3e = atomic_load_64(&l3[pmap_l3_index(va)]);
MPASS(l3e != 0);
/*
* Mark memory as read-only so we can invalidate
* the cache.
*/
l3e &= ~ATTR_S1_AP_MASK;
l3e |= ATTR_S1_AP(ATTR_S1_AP_RO);
atomic_store_64(&l3[pmap_l3_index(va)], l3e);
l3_list[i] = &l3[pmap_l3_index(va)];
i++;
} else {
/*
* The caller is responsible for clearing the cache &
* handling the TLB
*/
atomic_store_64(&l3[pmap_l3_index(va)], 0);
}
va_next = (va + L3_SIZE) & ~L3_OFFSET;
if (va_next < va)
va_next = eva;
}
mtx_unlock(&vmmpmap_mtx);
if (invalidate) {
/* Invalidate the memory from the D-cache */
vmm_call_hyp(HYP_DC_CIVAC, sva, size);
for (i = 0; i < (size / PAGE_SIZE); i++) {
atomic_store_64(l3_list[i], 0);
}
vmm_call_hyp(HYP_EL2_TLBI, HYP_EL2_TLBI_VA, sva, size);
free(l3_list, M_TEMP);
}
}

177
sys/arm64/vmm/vmm_reset.c Normal file
View file

@ -0,0 +1,177 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (C) 2018 Alexandru Elisei <alexandru.elisei@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <machine/armreg.h>
#include <machine/cpu.h>
#include <machine/hypervisor.h>
#include "arm64.h"
#include "reset.h"
/*
* Make the architecturally UNKNOWN value 0. As a bonus, we don't have to
* manually set all those RES0 fields.
*/
#define ARCH_UNKNOWN 0
#define set_arch_unknown(reg) (memset(&(reg), ARCH_UNKNOWN, sizeof(reg)))
void
reset_vm_el01_regs(void *vcpu)
{
struct hypctx *el2ctx;
el2ctx = vcpu;
set_arch_unknown(el2ctx->tf);
set_arch_unknown(el2ctx->actlr_el1);
set_arch_unknown(el2ctx->afsr0_el1);
set_arch_unknown(el2ctx->afsr1_el1);
set_arch_unknown(el2ctx->amair_el1);
set_arch_unknown(el2ctx->contextidr_el1);
set_arch_unknown(el2ctx->cpacr_el1);
set_arch_unknown(el2ctx->csselr_el1);
set_arch_unknown(el2ctx->elr_el1);
set_arch_unknown(el2ctx->esr_el1);
set_arch_unknown(el2ctx->far_el1);
set_arch_unknown(el2ctx->mair_el1);
set_arch_unknown(el2ctx->mdccint_el1);
set_arch_unknown(el2ctx->mdscr_el1);
set_arch_unknown(el2ctx->par_el1);
/*
* Guest starts with:
* ~SCTLR_M: MMU off
* ~SCTLR_C: data cache off
* SCTLR_CP15BEN: memory barrier instruction enable from EL0; RAO/WI
* ~SCTLR_I: instruction cache off
*/
el2ctx->sctlr_el1 = SCTLR_RES1;
el2ctx->sctlr_el1 &= ~SCTLR_M & ~SCTLR_C & ~SCTLR_I;
el2ctx->sctlr_el1 |= SCTLR_CP15BEN;
set_arch_unknown(el2ctx->sp_el0);
set_arch_unknown(el2ctx->tcr_el1);
set_arch_unknown(el2ctx->tpidr_el0);
set_arch_unknown(el2ctx->tpidr_el1);
set_arch_unknown(el2ctx->tpidrro_el0);
set_arch_unknown(el2ctx->ttbr0_el1);
set_arch_unknown(el2ctx->ttbr1_el1);
set_arch_unknown(el2ctx->vbar_el1);
set_arch_unknown(el2ctx->spsr_el1);
set_arch_unknown(el2ctx->dbgbcr_el1);
set_arch_unknown(el2ctx->dbgbvr_el1);
set_arch_unknown(el2ctx->dbgwcr_el1);
set_arch_unknown(el2ctx->dbgwvr_el1);
el2ctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0) & PMCR_N_MASK;
/* PMCR_LC is unknown when AArch32 is supported or RES1 otherwise */
el2ctx->pmcr_el0 |= PMCR_LC;
set_arch_unknown(el2ctx->pmccntr_el0);
set_arch_unknown(el2ctx->pmccfiltr_el0);
set_arch_unknown(el2ctx->pmcntenset_el0);
set_arch_unknown(el2ctx->pmintenset_el1);
set_arch_unknown(el2ctx->pmovsset_el0);
set_arch_unknown(el2ctx->pmuserenr_el0);
memset(el2ctx->pmevcntr_el0, 0, sizeof(el2ctx->pmevcntr_el0));
memset(el2ctx->pmevtyper_el0, 0, sizeof(el2ctx->pmevtyper_el0));
}
void
reset_vm_el2_regs(void *vcpu)
{
struct hypctx *el2ctx;
uint64_t cpu_aff, vcpuid;
el2ctx = vcpu;
vcpuid = vcpu_vcpuid(el2ctx->vcpu);
/*
* Set the Hypervisor Configuration Register:
*
* HCR_RW: use AArch64 for EL1
* HCR_TID3: handle ID registers in the vmm to privide a common
* set of featers on all vcpus
* HCR_TWI: Trap WFI to the hypervisor
* HCR_BSU_IS: barrier instructions apply to the inner shareable
* domain
* HCR_FB: broadcast maintenance operations
* HCR_AMO: route physical SError interrupts to EL2
* HCR_IMO: route physical IRQ interrupts to EL2
* HCR_FMO: route physical FIQ interrupts to EL2
* HCR_SWIO: turn set/way invalidate into set/way clean and
* invalidate
* HCR_VM: use stage 2 translation
*/
el2ctx->hcr_el2 = HCR_RW | HCR_TID3 | HCR_TWI | HCR_BSU_IS | HCR_FB |
HCR_AMO | HCR_IMO | HCR_FMO | HCR_SWIO | HCR_VM;
/* TODO: Trap all extensions we don't support */
el2ctx->mdcr_el2 = 0;
/* PMCR_EL0.N is read from MDCR_EL2.HPMN */
el2ctx->mdcr_el2 |= (el2ctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT;
el2ctx->vmpidr_el2 = VMPIDR_EL2_RES1;
/* The guest will detect a multi-core, single-threaded CPU */
el2ctx->vmpidr_el2 &= ~VMPIDR_EL2_U & ~VMPIDR_EL2_MT;
/*
* Generate the guest MPIDR value. We only support 16 CPUs at affinity
* level 0 to simplify the vgicv3 driver (see writing sgi1r_el1).
*/
cpu_aff = (vcpuid & 0xf) << MPIDR_AFF0_SHIFT |
((vcpuid >> 4) & 0xff) << MPIDR_AFF1_SHIFT |
((vcpuid >> 12) & 0xff) << MPIDR_AFF2_SHIFT |
((vcpuid >> 20) & 0xff) << MPIDR_AFF3_SHIFT;
el2ctx->vmpidr_el2 |= cpu_aff;
/* Use the same CPU identification information as the host */
el2ctx->vpidr_el2 = CPU_IMPL_TO_MIDR(CPU_IMPL_ARM);
el2ctx->vpidr_el2 |= CPU_VAR_TO_MIDR(0);
el2ctx->vpidr_el2 |= CPU_ARCH_TO_MIDR(0xf);
el2ctx->vpidr_el2 |= CPU_PART_TO_MIDR(CPU_PART_FOUNDATION);
el2ctx->vpidr_el2 |= CPU_REV_TO_MIDR(0);
/*
* Don't trap accesses to CPACR_EL1, trace, SVE, Advanced SIMD
* and floating point functionality to EL2.
*/
el2ctx->cptr_el2 = CPTR_RES1;
/*
* Disable interrupts in the guest. The guest OS will re-enable
* them.
*/
el2ctx->tf.tf_spsr = PSR_D | PSR_A | PSR_I | PSR_F;
/* Use the EL1 stack when taking exceptions to EL1 */
el2ctx->tf.tf_spsr |= PSR_M_EL1h;
}

165
sys/arm64/vmm/vmm_stat.c Normal file
View file

@ -0,0 +1,165 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2011 NetApp, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <machine/machdep.h>
#include <machine/vmm.h>
#include "vmm_stat.h"
/*
* 'vst_num_elems' is the total number of addressable statistic elements
* 'vst_num_types' is the number of unique statistic types
*
* It is always true that 'vst_num_elems' is greater than or equal to
* 'vst_num_types'. This is because a stat type may represent more than
* one element (for e.g. VMM_STAT_ARRAY).
*/
static int vst_num_elems, vst_num_types;
static struct vmm_stat_type *vsttab[MAX_VMM_STAT_ELEMS];
static MALLOC_DEFINE(M_VMM_STAT, "vmm stat", "vmm stat");
#define vst_size ((size_t)vst_num_elems * sizeof(uint64_t))
void
vmm_stat_register(void *arg)
{
struct vmm_stat_type *vst = arg;
/* We require all stats to identify themselves with a description */
if (vst->desc == NULL)
return;
if (vst_num_elems + vst->nelems >= MAX_VMM_STAT_ELEMS) {
printf("Cannot accommodate vmm stat type \"%s\"!\n", vst->desc);
return;
}
vst->index = vst_num_elems;
vst_num_elems += vst->nelems;
vsttab[vst_num_types++] = vst;
}
int
vmm_stat_copy(struct vcpu *vcpu, int index, int count, int *num_stats,
uint64_t *buf)
{
struct vmm_stat_type *vst;
uint64_t *stats;
int i, tocopy;
if (index < 0 || count < 0)
return (EINVAL);
if (index > vst_num_elems)
return (ENOENT);
if (index == vst_num_elems) {
*num_stats = 0;
return (0);
}
tocopy = min(vst_num_elems - index, count);
/* Let stats functions update their counters */
for (i = 0; i < vst_num_types; i++) {
vst = vsttab[i];
if (vst->func != NULL)
(*vst->func)(vcpu, vst);
}
/* Copy over the stats */
stats = vcpu_stats(vcpu);
memcpy(buf, stats + index, tocopy * sizeof(stats[0]));
*num_stats = tocopy;
return (0);
}
void *
vmm_stat_alloc(void)
{
return (malloc(vst_size, M_VMM_STAT, M_WAITOK));
}
void
vmm_stat_init(void *vp)
{
bzero(vp, vst_size);
}
void
vmm_stat_free(void *vp)
{
free(vp, M_VMM_STAT);
}
int
vmm_stat_desc_copy(int index, char *buf, int bufsize)
{
int i;
struct vmm_stat_type *vst;
for (i = 0; i < vst_num_types; i++) {
vst = vsttab[i];
if (index >= vst->index && index < vst->index + vst->nelems) {
if (vst->nelems > 1) {
snprintf(buf, bufsize, "%s[%d]",
vst->desc, index - vst->index);
} else {
strlcpy(buf, vst->desc, bufsize);
}
return (0); /* found it */
}
}
return (EINVAL);
}
/* global statistics */
VMM_STAT(VMEXIT_COUNT, "total number of vm exits");
VMM_STAT(VMEXIT_UNKNOWN, "number of vmexits for the unknown exception");
VMM_STAT(VMEXIT_WFI, "number of times wfi was intercepted");
VMM_STAT(VMEXIT_WFE, "number of times wfe was intercepted");
VMM_STAT(VMEXIT_HVC, "number of times hvc was intercepted");
VMM_STAT(VMEXIT_MSR, "number of times msr/mrs was intercepted");
VMM_STAT(VMEXIT_DATA_ABORT, "number of vmexits for a data abort");
VMM_STAT(VMEXIT_INSN_ABORT, "number of vmexits for an instruction abort");
VMM_STAT(VMEXIT_UNHANDLED_SYNC, "number of vmexits for an unhandled synchronous exception");
VMM_STAT(VMEXIT_IRQ, "number of vmexits for an irq");
VMM_STAT(VMEXIT_FIQ, "number of vmexits for an interrupt");
VMM_STAT(VMEXIT_UNHANDLED_EL2, "number of vmexits for an unhandled EL2 exception");
VMM_STAT(VMEXIT_UNHANDLED, "number of vmexits for an unhandled exception");

145
sys/arm64/vmm/vmm_stat.h Normal file
View file

@ -0,0 +1,145 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2011 NetApp, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _VMM_STAT_H_
#define _VMM_STAT_H_
struct vm;
#define MAX_VMM_STAT_ELEMS 64 /* arbitrary */
enum vmm_stat_scope {
VMM_STAT_SCOPE_ANY,
};
struct vmm_stat_type;
typedef void (*vmm_stat_func_t)(struct vcpu *vcpu,
struct vmm_stat_type *stat);
struct vmm_stat_type {
int index; /* position in the stats buffer */
int nelems; /* standalone or array */
const char *desc; /* description of statistic */
vmm_stat_func_t func;
enum vmm_stat_scope scope;
};
void vmm_stat_register(void *arg);
#define VMM_STAT_FDEFINE(type, nelems, desc, func, scope) \
struct vmm_stat_type type[1] = { \
{ -1, nelems, desc, func, scope } \
}; \
SYSINIT(type##_stat, SI_SUB_KLD, SI_ORDER_ANY, vmm_stat_register, type)
#define VMM_STAT_DEFINE(type, nelems, desc, scope) \
VMM_STAT_FDEFINE(type, nelems, desc, NULL, scope)
#define VMM_STAT_DECLARE(type) \
extern struct vmm_stat_type type[1]
#define VMM_STAT(type, desc) \
VMM_STAT_DEFINE(type, 1, desc, VMM_STAT_SCOPE_ANY)
#define VMM_STAT_FUNC(type, desc, func) \
VMM_STAT_FDEFINE(type, 1, desc, func, VMM_STAT_SCOPE_ANY)
#define VMM_STAT_ARRAY(type, nelems, desc) \
VMM_STAT_DEFINE(type, nelems, desc, VMM_STAT_SCOPE_ANY)
void *vmm_stat_alloc(void);
void vmm_stat_init(void *vp);
void vmm_stat_free(void *vp);
int vmm_stat_copy(struct vcpu *vcpu, int index, int count,
int *num_stats, uint64_t *buf);
int vmm_stat_desc_copy(int index, char *buf, int buflen);
static void __inline
vmm_stat_array_incr(struct vcpu *vcpu, struct vmm_stat_type *vst, int statidx,
uint64_t x)
{
#ifdef VMM_KEEP_STATS
uint64_t *stats;
stats = vcpu_stats(vcpu);
if (vst->index >= 0 && statidx < vst->nelems)
stats[vst->index + statidx] += x;
#endif
}
static void __inline
vmm_stat_array_set(struct vcpu *vcpu, struct vmm_stat_type *vst, int statidx,
uint64_t val)
{
#ifdef VMM_KEEP_STATS
uint64_t *stats;
stats = vcpu_stats(vcpu);
if (vst->index >= 0 && statidx < vst->nelems)
stats[vst->index + statidx] = val;
#endif
}
static void __inline
vmm_stat_incr(struct vcpu *vcpu, struct vmm_stat_type *vst, uint64_t x)
{
#ifdef VMM_KEEP_STATS
vmm_stat_array_incr(vcpu, vst, 0, x);
#endif
}
static void __inline
vmm_stat_set(struct vcpu *vcpu, struct vmm_stat_type *vst, uint64_t val)
{
#ifdef VMM_KEEP_STATS
vmm_stat_array_set(vcpu, vst, 0, val);
#endif
}
VMM_STAT_DECLARE(VMEXIT_COUNT);
VMM_STAT_DECLARE(VMEXIT_UNKNOWN);
VMM_STAT_DECLARE(VMEXIT_WFI);
VMM_STAT_DECLARE(VMEXIT_WFE);
VMM_STAT_DECLARE(VMEXIT_HVC);
VMM_STAT_DECLARE(VMEXIT_MSR);
VMM_STAT_DECLARE(VMEXIT_DATA_ABORT);
VMM_STAT_DECLARE(VMEXIT_INSN_ABORT);
VMM_STAT_DECLARE(VMEXIT_UNHANDLED_SYNC);
VMM_STAT_DECLARE(VMEXIT_IRQ);
VMM_STAT_DECLARE(VMEXIT_FIQ);
VMM_STAT_DECLARE(VMEXIT_UNHANDLED_EL2);
VMM_STAT_DECLARE(VMEXIT_UNHANDLED);
#endif

33
sys/conf/files.arm64
View file

@ -116,6 +116,39 @@ arm64/iommu/smmu_quirks.c optional iommu
dev/iommu/busdma_iommu.c optional iommu
dev/iommu/iommu_gas.c optional iommu
arm64/vmm/vmm.c optional vmm
arm64/vmm/vmm_dev.c optional vmm
arm64/vmm/vmm_instruction_emul.c optional vmm
arm64/vmm/vmm_stat.c optional vmm
arm64/vmm/vmm_arm64.c optional vmm
arm64/vmm/vmm_reset.c optional vmm
arm64/vmm/vmm_call.S optional vmm
arm64/vmm/vmm_hyp_exception.S optional vmm \
compile-with "${NORMAL_C:N-fsanitize*:N-mbranch-protection*} -fpie" \
no-obj
arm64/vmm/vmm_hyp.c optional vmm \
compile-with "${NORMAL_C:N-fsanitize*:N-mbranch-protection*} -fpie" \
no-obj
vmm_hyp_blob.elf.full optional vmm \
dependency "vmm_hyp.o vmm_hyp_exception.o" \
compile-with "${SYSTEM_LD_BASECMD} -o ${.TARGET} ${.ALLSRC} --defsym=text_start='0x0'" \
no-obj no-implicit-rule
vmm_hyp_blob.elf optional vmm \
dependency "vmm_hyp_blob.elf.full" \
compile-with "${OBJCOPY} --strip-debug ${.ALLSRC} ${.TARGET}" \
no-obj no-implicit-rule
vmm_hyp_blob.bin optional vmm \
dependency vmm_hyp_blob.elf \
compile-with "${OBJCOPY} --output-target=binary ${.ALLSRC} ${.TARGET}" \
no-obj no-implicit-rule
arm64/vmm/vmm_hyp_el2.S optional vmm \
dependency vmm_hyp_blob.bin
arm64/vmm/vmm_mmu.c optional vmm
arm64/vmm/io/vgic.c optional vmm
arm64/vmm/io/vgic_v3.c optional vmm
arm64/vmm/io/vgic_if.m optional vmm
arm64/vmm/io/vtimer.c optional vmm
crypto/armv8/armv8_crypto.c optional armv8crypto
armv8_crypto_wrap.o optional armv8crypto \
dependency "$S/crypto/armv8/armv8_crypto_wrap.c" \

2
sys/conf/ldscript.arm64
View file

@ -6,6 +6,7 @@ SECTIONS
{
/* Read-only sections, merged into text segment: */
. = text_start; /* This is set using --defsym= on the command line. */
.vmm_vectors : { *(.vmm_vectors) }
.text :
{
*(.text)
@ -16,6 +17,7 @@ SECTIONS
} =0x9090
_etext = .;
PROVIDE (etext = .);
.fini : { *(.fini) } =0x9090
.rodata : { *(.rodata*) *(.gnu.linkonce.r*) }
.rodata1 : { *(.rodata1) }

3
sys/conf/options.arm64
View file

@ -19,6 +19,9 @@ EMUL_SWP opt_global.h
# EFI Runtime services support
EFIRT opt_efirt.h
# Bhyve
VMM opt_global.h
# SoC Support
SOC_ALLWINNER_A64 opt_soc.h
SOC_ALLWINNER_H5 opt_soc.h

2
sys/modules/Makefile
View file

@ -841,7 +841,9 @@ _sgx= sgx
_sgx_linux= sgx_linux
_smartpqi= smartpqi
_p2sb= p2sb
.endif
.if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64"
.if ${MK_BHYVE} != "no" || defined(ALL_MODULES)
.if ${KERN_OPTS:MSMP}
_vmm= vmm

80
sys/modules/vmm/Makefile
View file

@ -3,31 +3,79 @@
KMOD= vmm
SRCS= opt_acpi.h opt_bhyve_snapshot.h opt_ddb.h
SRCS+= device_if.h bus_if.h pci_if.h pcib_if.h acpi_if.h vnode_if.h
SRCS= opt_acpi.h opt_ddb.h device_if.h bus_if.h pci_if.h pcib_if.h acpi_if.h
CFLAGS+= -DVMM_KEEP_STATS
CFLAGS+= -I${SRCTOP}/sys/${MACHINE}/vmm
CFLAGS+= -I${SRCTOP}/sys/${MACHINE}/vmm/io
# generic vmm support
.PATH: ${SRCTOP}/sys/${MACHINE}/vmm
SRCS+= vmm.c \
vmm_dev.c \
vmm_instruction_emul.c \
vmm_stat.c
.if ${MACHINE_CPUARCH} == "aarch64"
DPSRCS+= assym.inc
# TODO: Add the new EL2 code
SRCS+= vmm_arm64.c \
vmm_reset.c \
vmm_call.S \
vmm_mmu.c \
vmm_hyp_el2.S
.PATH: ${SRCTOP}/sys/${MACHINE}/vmm/io
SRCS+= vgic.c \
vgic_if.h \
vgic_if.c \
vgic_v3.c \
vtimer.c
SRCS+= vmm_hyp_exception.S vmm_hyp.c
CLEANFILES+= vmm_hyp_blob.elf.full
CLEANFILES+= vmm_hyp_blob.elf vmm_hyp_blob.bin
vmm_hyp_exception.o: vmm_hyp_exception.S
${CC} -c -x assembler-with-cpp -DLOCORE \
${CFLAGS:N-fsanitize*:N-mbranch-protection*} \
${.IMPSRC} -o ${.TARGET} -fpie
vmm_hyp.o: vmm_hyp.c
${CC} -c ${CFLAGS:N-fsanitize*:N-mbranch-protection*} \
${.IMPSRC} -o ${.TARGET} -fpie
vmm_hyp_blob.elf.full: vmm_hyp_exception.o vmm_hyp.o
${LD} -m ${LD_EMULATION} -Bdynamic -T ${SYSDIR}/conf/ldscript.arm64 \
${_LDFLAGS} --no-warn-mismatch --warn-common --export-dynamic \
--dynamic-linker /red/herring -X -o ${.TARGET} ${.ALLSRC} \
--defsym=text_start='0x0'
vmm_hyp_blob.elf: vmm_hyp_blob.elf.full
${OBJCOPY} --strip-debug ${.ALLSRC} ${.TARGET}
vmm_hyp_blob.bin: vmm_hyp_blob.elf
${OBJCOPY} --output-target=binary ${.ALLSRC} ${.TARGET}
vmm_hyp_el2.o: vmm_hyp_blob.bin
.elif ${MACHINE_CPUARCH} == "amd64"
DPSRCS+= vmx_assym.h svm_assym.h
DPSRCS+= vmx_genassym.c svm_genassym.c offset.inc
CFLAGS+= -DVMM_KEEP_STATS
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/io
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/intel
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/amd
# generic vmm support
.PATH: ${SRCTOP}/sys/amd64/vmm
SRCS+= vmm.c \
vmm_dev.c \
vmm_host.c \
vmm_instruction_emul.c \
SRCS+= vmm_host.c \
vmm_ioport.c \
vmm_lapic.c \
vmm_mem.c \
vmm_stat.c \
vmm_util.c \
x86.c
.PATH: ${SRCTOP}/sys/amd64/vmm/io
.PATH: ${SRCTOP}/sys/${MACHINE}/vmm/io
SRCS+= iommu.c \
ppt.c \
vatpic.c \
@ -62,10 +110,11 @@ SRCS+= vmcb.c \
SRCS.BHYVE_SNAPSHOT= vmm_snapshot.c
CLEANFILES= vmx_assym.h vmx_genassym.o svm_assym.h svm_genassym.o
CLEANFILES+= vmx_assym.h vmx_genassym.o svm_assym.h svm_genassym.o
OBJS_DEPEND_GUESS.vmx_support.o+= vmx_assym.h
OBJS_DEPEND_GUESS.svm_support.o+= svm_assym.h
.endif
vmx_assym.h: vmx_genassym.o
sh ${SYSDIR}/kern/genassym.sh vmx_genassym.o > ${.TARGET}
@ -81,6 +130,9 @@ svm_support.o:
${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
${.IMPSRC} -o ${.TARGET}
hyp_genassym.o: offset.inc
${CC} -c ${CFLAGS:N-flto:N-fno-common} -fcommon ${.IMPSRC}
vmx_genassym.o: offset.inc
${CC} -c ${CFLAGS:N-flto*:N-fno-common:N-fsanitize*:N-fno-sanitize*} \
-fcommon ${.IMPSRC}