system/linux
system/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux synced 2024-07-22 03:01:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

* Refactor and clean up TDX hypercall/module call infrastructure

Browse source 
* Handle retrying/resuming page conversion hypercalls
  * Make sure to use the (shockingly) reliable TSC in TDX guests
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEV76QKkVc4xCGURexaDWVMHDJkrAFAmVBlqMACgkQaDWVMHDJ
 krBrhBAArKay0MvzmdzS4IQs8JqkmuMEHI6WabYv2POPjJNXrn5MelLH972pLuX9
 NJ3+yeOLmNMYwqu5qwLCxyeO5CtqEyT2lNumUrxAtHQG4+oS2RYJYUalxMuoGxt8
 fAHxbItFg0TobBSUtwcnN2R2WdXwPuUW0Co+pJfLlZV4umVM7QANO1nf1g8YmlDD
 sVtpDaeKJRdylmwgWgAyGow0tDKd6oZB9j/vOHvZRrEQ+DMjEtG75fjwbjbu43Cl
 tI/fbxKjzAkOFcZ7PEPsQ8jE1h9DXU+JzTML9Nu/cPMalxMeBg3Dol/JOEbqgreI
 4W8Lg7g071EkUcQDxpwfe4aS6rsfsbwUIV4gJVkg9ZhlT7RayWsFik2CfBpJ4IMQ
 TM8BxtCEGCz3cxVvg3mstX9rRA7eNlXOzcKE/8Y7cpSsp94bA9jtf2GgUSUoi9St
 y+fIEei8mgeHutdiFh8psrmR7hp6iX/ldMFqHtjNo6xatf2KjdVHhVSU13Jz544z
 43ATNi1gZeHOgfwlAlIxLPDVDJidHuux3f6g2vfMkAqItyEqFauC1HA1pIDgckoY
 9FpBPp9vNUToSPp6reB6z/PkEBIrG2XtQh82JLt2CnCb6aTUtnPds+psjtT4sSE/
 a9SQvZLWWmpj+BlI2yrtfJzhy7SwhltgdjItQHidmCNEn0PYfTc=
 =FJ1Y
 -----END PGP SIGNATURE-----

Merge tag 'x86_tdx_for_6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 TDX updates from Dave Hansen:
 "The majority of this is a rework of the assembly and C wrappers that
  are used to talk to the TDX module and VMM. This is a nice cleanup in
  general but is also clearing the way for using this code when Linux is
  the TDX VMM.

  There are also some tidbits to make TDX guests play nicer with Hyper-V
  and to take advantage the hardware TSC.

  Summary:

   - Refactor and clean up TDX hypercall/module call infrastructure

   - Handle retrying/resuming page conversion hypercalls

   - Make sure to use the (shockingly) reliable TSC in TDX guests"

[ TLA reminder: TDX is "Trust Domain Extensions", Intel's guest VM
  confidentiality technology ]

* tag 'x86_tdx_for_6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/tdx: Mark TSC reliable
  x86/tdx: Fix __noreturn build warning around __tdx_hypercall_failed()
  x86/virt/tdx: Make TDX_MODULE_CALL handle SEAMCALL #UD and #GP
  x86/virt/tdx: Wire up basic SEAMCALL functions
  x86/tdx: Remove 'struct tdx_hypercall_args'
  x86/tdx: Reimplement __tdx_hypercall() using TDX_MODULE_CALL asm
  x86/tdx: Make TDX_HYPERCALL asm similar to TDX_MODULE_CALL
  x86/tdx: Extend TDX_MODULE_CALL to support more TDCALL/SEAMCALL leafs
  x86/tdx: Pass TDCALL/SEAMCALL input/output registers via a structure
  x86/tdx: Rename __tdx_module_call() to __tdcall()
  x86/tdx: Make macros of TDCALLs consistent with the spec
  x86/tdx: Skip saving output regs when SEAMCALL fails with VMFailInvalid
  x86/tdx: Zero out the missing RSI in TDX_HYPERCALL macro
  x86/tdx: Retry partially-completed page conversion hypercalls
This commit is contained in:
Linus Torvalds 2023-11-01 10:28:32 -10:00
parent f00593e099 9ee4318c15
commit 8999ad99f4
16 changed files with 495 additions and 370 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
arch/x86/Kconfig
View file

@ -1940,6 +1940,18 @@ config X86_USER_SHADOW_STACK
If unsure, say N.
config INTEL_TDX_HOST
bool "Intel Trust Domain Extensions (TDX) host support"
depends on CPU_SUP_INTEL
depends on X86_64
depends on KVM_INTEL
help
Intel Trust Domain Extensions (TDX) protects guest VMs from malicious
host and certain physical attacks. This option enables necessary TDX
support in the host kernel to run confidential VMs.
If unsure, say N.
config EFI
bool "EFI runtime service support"
depends on ACPI

2
arch/x86/Makefile
View file

@ -253,6 +253,8 @@ archheaders:
libs-y += arch/x86/lib/
core-y += arch/x86/virt/
# drivers-y are linked after core-y
drivers-$(CONFIG_MATH_EMULATION) += arch/x86/math-emu/
drivers-$(CONFIG_PCI) += arch/x86/pci/

6
arch/x86/boot/compressed/tdx.c
View file

@ -18,7 +18,7 @@ void __tdx_hypercall_failed(void)
static inline unsigned int tdx_io_in(int size, u16 port)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
.r12 = size,
@ -26,7 +26,7 @@ static inline unsigned int tdx_io_in(int size, u16 port)
.r14 = port,
};
if (__tdx_hypercall_ret(&args))
if (__tdx_hypercall(&args))
return UINT_MAX;
return args.r11;
@ -34,7 +34,7 @@ static inline unsigned int tdx_io_in(int size, u16 port)
static inline void tdx_io_out(int size, u16 port, u32 value)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
.r12 = size,

236
arch/x86/coco/tdx/tdcall.S
View file

@ -1,239 +1,63 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/asm-offsets.h>
#include <asm/asm.h>
#include <asm/frame.h>
#include <asm/unwind_hints.h>
#include <linux/linkage.h>
#include <linux/bits.h>
#include <linux/errno.h>
#include "../../virt/vmx/tdx/tdxcall.S"
/*
* Bitmasks of exposed registers (with VMM).
*/
#define TDX_RDX BIT(2)
#define TDX_RBX BIT(3)
#define TDX_RSI BIT(6)
#define TDX_RDI BIT(7)
#define TDX_R8 BIT(8)
#define TDX_R9 BIT(9)
#define TDX_R10 BIT(10)
#define TDX_R11 BIT(11)
#define TDX_R12 BIT(12)
#define TDX_R13 BIT(13)
#define TDX_R14 BIT(14)
#define TDX_R15 BIT(15)
/*
* These registers are clobbered to hold arguments for each
* TDVMCALL. They are safe to expose to the VMM.
* Each bit in this mask represents a register ID. Bit field
* details can be found in TDX GHCI specification, section
* titled "TDCALL [TDG.VP.VMCALL] leaf".
*/
#define TDVMCALL_EXPOSE_REGS_MASK \
( TDX_RDX | TDX_RBX | TDX_RSI | TDX_RDI | TDX_R8 | TDX_R9 | \
TDX_R10 | TDX_R11 | TDX_R12 | TDX_R13 | TDX_R14 | TDX_R15 )
.section .noinstr.text, "ax"
/*
* __tdx_module_call() - Used by TDX guests to request services from
* the TDX module (does not include VMM services) using TDCALL instruction.
* __tdcall() - Used by TDX guests to request services from the TDX
* module (does not include VMM services) using TDCALL instruction.
*
* Transforms function call register arguments into the TDCALL register ABI.
* After TDCALL operation, TDX module output is saved in @out (if it is
* provided by the user).
* __tdcall() function ABI:
*
*-------------------------------------------------------------------------
* TDCALL ABI:
*-------------------------------------------------------------------------
* Input Registers:
* @fn (RDI) - TDCALL Leaf ID, moved to RAX
* @args (RSI) - struct tdx_module_args for input
*
* RAX - TDCALL Leaf number.
* RCX,RDX,R8-R9 - TDCALL Leaf specific input registers.
*
* Output Registers:
*
* RAX - TDCALL instruction error code.
* RCX,RDX,R8-R11 - TDCALL Leaf specific output registers.
*
*-------------------------------------------------------------------------
*
* __tdx_module_call() function ABI:
*
* @fn (RDI) - TDCALL Leaf ID, moved to RAX
* @rcx (RSI) - Input parameter 1, moved to RCX
* @rdx (RDX) - Input parameter 2, moved to RDX
* @r8 (RCX) - Input parameter 3, moved to R8
* @r9 (R8) - Input parameter 4, moved to R9
*
* @out (R9) - struct tdx_module_output pointer
* stored temporarily in R12 (not
* shared with the TDX module). It
* can be NULL.
* Only RCX/RDX/R8-R11 are used as input registers.
*
* Return status of TDCALL via RAX.
*/
SYM_FUNC_START(__tdx_module_call)
FRAME_BEGIN
SYM_FUNC_START(__tdcall)
TDX_MODULE_CALL host=0
FRAME_END
RET
SYM_FUNC_END(__tdx_module_call)
SYM_FUNC_END(__tdcall)
/*
* TDX_HYPERCALL - Make hypercalls to a TDX VMM using TDVMCALL leaf of TDCALL
* instruction
* __tdcall_ret() - Used by TDX guests to request services from the TDX
* module (does not include VMM services) using TDCALL instruction, with
* saving output registers to the 'struct tdx_module_args' used as input.
*
* Transforms values in function call argument struct tdx_hypercall_args @args
* into the TDCALL register ABI. After TDCALL operation, VMM output is saved
* back in @args, if \ret is 1.
* __tdcall_ret() function ABI:
*
*-------------------------------------------------------------------------
* TD VMCALL ABI:
*-------------------------------------------------------------------------
* @fn (RDI) - TDCALL Leaf ID, moved to RAX
* @args (RSI) - struct tdx_module_args for input and output
*
* Input Registers:
*
* RAX - TDCALL instruction leaf number (0 - TDG.VP.VMCALL)
* RCX - BITMAP which controls which part of TD Guest GPR
* is passed as-is to the VMM and back.
* R10 - Set 0 to indicate TDCALL follows standard TDX ABI
* specification. Non zero value indicates vendor
* specific ABI.
* R11 - VMCALL sub function number
* RBX, RDX, RDI, RSI - Used to pass VMCALL sub function specific arguments.
* R8-R9, R12-R15 - Same as above.
*
* Output Registers:
*
* RAX - TDCALL instruction status (Not related to hypercall
* output).
* RBX, RDX, RDI, RSI - Hypercall sub function specific output values.
* R8-R15 - Same as above.
* Only RCX/RDX/R8-R11 are used as input/output registers.
*
* Return status of TDCALL via RAX.
*/
.macro TDX_HYPERCALL ret:req
FRAME_BEGIN
/* Save callee-saved GPRs as mandated by the x86_64 ABI */
push %r15
push %r14
push %r13
push %r12
push %rbx
/* Free RDI to be used as TDVMCALL arguments */
movq %rdi, %rax
/* Copy hypercall registers from arg struct: */
movq TDX_HYPERCALL_r8(%rax), %r8
movq TDX_HYPERCALL_r9(%rax), %r9
movq TDX_HYPERCALL_r10(%rax), %r10
movq TDX_HYPERCALL_r11(%rax), %r11
movq TDX_HYPERCALL_r12(%rax), %r12
movq TDX_HYPERCALL_r13(%rax), %r13
movq TDX_HYPERCALL_r14(%rax), %r14
movq TDX_HYPERCALL_r15(%rax), %r15
movq TDX_HYPERCALL_rdi(%rax), %rdi
movq TDX_HYPERCALL_rsi(%rax), %rsi
movq TDX_HYPERCALL_rbx(%rax), %rbx
movq TDX_HYPERCALL_rdx(%rax), %rdx
push %rax
/* Mangle function call ABI into TDCALL ABI: */
/* Set TDCALL leaf ID (TDVMCALL (0)) in RAX */
xor %eax, %eax
movl $TDVMCALL_EXPOSE_REGS_MASK, %ecx
tdcall
/*
* RAX!=0 indicates a failure of the TDVMCALL mechanism itself and that
* something has gone horribly wrong with the TDX module.
*
* The return status of the hypercall operation is in a separate
* register (in R10). Hypercall errors are a part of normal operation
* and are handled by callers.
*/
testq %rax, %rax
jne .Lpanic\@
pop %rax
.if \ret
movq %r8, TDX_HYPERCALL_r8(%rax)
movq %r9, TDX_HYPERCALL_r9(%rax)
movq %r10, TDX_HYPERCALL_r10(%rax)
movq %r11, TDX_HYPERCALL_r11(%rax)
movq %r12, TDX_HYPERCALL_r12(%rax)
movq %r13, TDX_HYPERCALL_r13(%rax)
movq %r14, TDX_HYPERCALL_r14(%rax)
movq %r15, TDX_HYPERCALL_r15(%rax)
movq %rdi, TDX_HYPERCALL_rdi(%rax)
movq %rsi, TDX_HYPERCALL_rsi(%rax)
movq %rbx, TDX_HYPERCALL_rbx(%rax)
movq %rdx, TDX_HYPERCALL_rdx(%rax)
.endif
/* TDVMCALL leaf return code is in R10 */
movq %r10, %rax
/*
* Zero out registers exposed to the VMM to avoid speculative execution
* with VMM-controlled values. This needs to include all registers
* present in TDVMCALL_EXPOSE_REGS_MASK, except RBX, and R12-R15 which
* will be restored.
*/
xor %r8d, %r8d
xor %r9d, %r9d
xor %r10d, %r10d
xor %r11d, %r11d
xor %rdi, %rdi
xor %rdx, %rdx
/* Restore callee-saved GPRs as mandated by the x86_64 ABI */
pop %rbx
pop %r12
pop %r13
pop %r14
pop %r15
FRAME_END
RET
.Lpanic\@:
call __tdx_hypercall_failed
/* __tdx_hypercall_failed never returns */
REACHABLE
jmp .Lpanic\@
.endm
SYM_FUNC_START(__tdcall_ret)
TDX_MODULE_CALL host=0 ret=1
SYM_FUNC_END(__tdcall_ret)
/*
* __tdcall_saved_ret() - Used by TDX guests to request services from the
* TDX module (including VMM services) using TDCALL instruction, with
* saving output registers to the 'struct tdx_module_args' used as input.
*
* __tdx_hypercall() function ABI:
* __tdcall_saved_ret() function ABI:
*
* @args (RDI) - struct tdx_hypercall_args for input
* @fn (RDI) - TDCALL leaf ID, moved to RAX
* @args (RSI) - struct tdx_module_args for input/output
*
* All registers in @args are used as input/output registers.
*
* On successful completion, return the hypercall error code.
*/
SYM_FUNC_START(__tdx_hypercall)
TDX_HYPERCALL ret=0
SYM_FUNC_END(__tdx_hypercall)
/*
*
* __tdx_hypercall_ret() function ABI:
*
* @args (RDI) - struct tdx_hypercall_args for input and output
*
* On successful completion, return the hypercall error code.
*/
SYM_FUNC_START(__tdx_hypercall_ret)
TDX_HYPERCALL ret=1
SYM_FUNC_END(__tdx_hypercall_ret)
SYM_FUNC_START(__tdcall_saved_ret)
TDX_MODULE_CALL host=0 ret=1 saved=1
SYM_FUNC_END(__tdcall_saved_ret)

28
arch/x86/coco/tdx/tdx-shared.c
View file

@ -5,7 +5,7 @@ static unsigned long try_accept_one(phys_addr_t start, unsigned long len,
enum pg_level pg_level)
{
unsigned long accept_size = page_level_size(pg_level);
u64 tdcall_rcx;
struct tdx_module_args args = {};
u8 page_size;
if (!IS_ALIGNED(start, accept_size))
@ -34,8 +34,8 @@ static unsigned long try_accept_one(phys_addr_t start, unsigned long len,
return 0;
}
tdcall_rcx = start | page_size;
if (__tdx_module_call(TDX_ACCEPT_PAGE, tdcall_rcx, 0, 0, 0, NULL))
args.rcx = start | page_size;
if (__tdcall(TDG_MEM_PAGE_ACCEPT, &args))
return 0;
return accept_size;
@ -45,7 +45,7 @@ bool tdx_accept_memory(phys_addr_t start, phys_addr_t end)
{
/*
* For shared->private conversion, accept the page using
* TDX_ACCEPT_PAGE TDX module call.
* TDG_MEM_PAGE_ACCEPT TDX module call.
*/
while (start < end) {
unsigned long len = end - start;
@ -69,3 +69,23 @@ bool tdx_accept_memory(phys_addr_t start, phys_addr_t end)
return true;
}
noinstr u64 __tdx_hypercall(struct tdx_module_args *args)
{
/*
* For TDVMCALL explicitly set RCX to the bitmap of shared registers.
* The caller isn't expected to set @args->rcx anyway.
*/
args->rcx = TDVMCALL_EXPOSE_REGS_MASK;
/*
* Failure of __tdcall_saved_ret() indicates a failure of the TDVMCALL
* mechanism itself and that something has gone horribly wrong with
* the TDX module. __tdx_hypercall_failed() never returns.
*/
if (__tdcall_saved_ret(TDG_VP_VMCALL, args))
__tdx_hypercall_failed();
/* TDVMCALL leaf return code is in R10 */
return args->r10;
}

138
arch/x86/coco/tdx/tdx.c
View file

@ -38,7 +38,7 @@
#define TDREPORT_SUBTYPE_0 0
/* Called from __tdx_hypercall() for unrecoverable failure */
noinstr void __tdx_hypercall_failed(void)
noinstr void __noreturn __tdx_hypercall_failed(void)
{
instrumentation_begin();
panic("TDVMCALL failed. TDX module bug?");
@ -48,7 +48,7 @@ noinstr void __tdx_hypercall_failed(void)
long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2,
unsigned long p3, unsigned long p4)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = nr,
.r11 = p1,
.r12 = p2,
@ -66,10 +66,9 @@ EXPORT_SYMBOL_GPL(tdx_kvm_hypercall);
* should only be used for calls that have no legitimate reason to fail
* or where the kernel can not survive the call failing.
*/
static inline void tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9,
struct tdx_module_output *out)
static inline void tdcall(u64 fn, struct tdx_module_args *args)
{
if (__tdx_module_call(fn, rcx, rdx, r8, r9, out))
if (__tdcall_ret(fn, args))
panic("TDCALL %lld failed (Buggy TDX module!)\n", fn);
}
@ -89,11 +88,14 @@ static inline void tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9,
*/
int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
{
struct tdx_module_args args = {
.rcx = virt_to_phys(tdreport),
.rdx = virt_to_phys(reportdata),
.r8 = TDREPORT_SUBTYPE_0,
};
u64 ret;
ret = __tdx_module_call(TDX_GET_REPORT, virt_to_phys(tdreport),
virt_to_phys(reportdata), TDREPORT_SUBTYPE_0,
0, NULL);
ret = __tdcall(TDG_MR_REPORT, &args);
if (ret) {
if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND)
return -EINVAL;
@ -106,7 +108,7 @@ EXPORT_SYMBOL_GPL(tdx_mcall_get_report0);
static void __noreturn tdx_panic(const char *msg)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = TDVMCALL_REPORT_FATAL_ERROR,
.r12 = 0, /* Error code: 0 is Panic */
@ -141,7 +143,7 @@ static void __noreturn tdx_panic(const char *msg)
static void tdx_parse_tdinfo(u64 *cc_mask)
{
struct tdx_module_output out;
struct tdx_module_args args = {};
unsigned int gpa_width;
u64 td_attr;
@ -152,7 +154,7 @@ static void tdx_parse_tdinfo(u64 *cc_mask)
* Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL
* [TDG.VP.INFO].
*/
tdx_module_call(TDX_GET_INFO, 0, 0, 0, 0, &out);
tdcall(TDG_VP_INFO, &args);
/*
* The highest bit of a guest physical address is the "sharing" bit.
@ -161,7 +163,7 @@ static void tdx_parse_tdinfo(u64 *cc_mask)
* The GPA width that comes out of this call is critical. TDX guests
* can not meaningfully run without it.
*/
gpa_width = out.rcx & GENMASK(5, 0);
gpa_width = args.rcx & GENMASK(5, 0);
*cc_mask = BIT_ULL(gpa_width - 1);
/*
@ -169,7 +171,7 @@ static void tdx_parse_tdinfo(u64 *cc_mask)
* memory. Ensure that no #VE will be delivered for accesses to
* TD-private memory. Only VMM-shared memory (MMIO) will #VE.
*/
td_attr = out.rdx;
td_attr = args.rdx;
if (!(td_attr & ATTR_SEPT_VE_DISABLE)) {
const char *msg = "TD misconfiguration: SEPT_VE_DISABLE attribute must be set.";
@ -228,7 +230,7 @@ static int ve_instr_len(struct ve_info *ve)
static u64 __cpuidle __halt(const bool irq_disabled)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_HLT),
.r12 = irq_disabled,
@ -272,7 +274,7 @@ void __cpuidle tdx_safe_halt(void)
static int read_msr(struct pt_regs *regs, struct ve_info *ve)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_MSR_READ),
.r12 = regs->cx,
@ -283,7 +285,7 @@ static int read_msr(struct pt_regs *regs, struct ve_info *ve)
* can be found in TDX Guest-Host-Communication Interface
* (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>".
*/
if (__tdx_hypercall_ret(&args))
if (__tdx_hypercall(&args))
return -EIO;
regs->ax = lower_32_bits(args.r11);
@ -293,7 +295,7 @@ static int read_msr(struct pt_regs *regs, struct ve_info *ve)
static int write_msr(struct pt_regs *regs, struct ve_info *ve)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_MSR_WRITE),
.r12 = regs->cx,
@ -313,7 +315,7 @@ static int write_msr(struct pt_regs *regs, struct ve_info *ve)
static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_CPUID),
.r12 = regs->ax,
@ -337,7 +339,7 @@ static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
* ABI can be found in TDX Guest-Host-Communication Interface
* (GHCI), section titled "VP.VMCALL<Instruction.CPUID>".
*/
if (__tdx_hypercall_ret(&args))
if (__tdx_hypercall(&args))
return -EIO;
/*
@ -355,7 +357,7 @@ static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
static bool mmio_read(int size, unsigned long addr, unsigned long *val)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_EPT_VIOLATION),
.r12 = size,
@ -364,8 +366,9 @@ static bool mmio_read(int size, unsigned long addr, unsigned long *val)
.r15 = *val,
};
if (__tdx_hypercall_ret(&args))
if (__tdx_hypercall(&args))
return false;
*val = args.r11;
return true;
}
@ -483,7 +486,7 @@ static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)
static bool handle_in(struct pt_regs *regs, int size, int port)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
.r12 = size,
@ -498,7 +501,7 @@ static bool handle_in(struct pt_regs *regs, int size, int port)
* in TDX Guest-Host-Communication Interface (GHCI) section titled
* "TDG.VP.VMCALL<Instruction.IO>".
*/
success = !__tdx_hypercall_ret(&args);
success = !__tdx_hypercall(&args);
/* Update part of the register affected by the emulated instruction */
regs->ax &= ~mask;
@ -577,7 +580,7 @@ __init bool tdx_early_handle_ve(struct pt_regs *regs)
void tdx_get_ve_info(struct ve_info *ve)
{
struct tdx_module_output out;
struct tdx_module_args args = {};
/*
* Called during #VE handling to retrieve the #VE info from the
@ -594,15 +597,15 @@ void tdx_get_ve_info(struct ve_info *ve)
* Note, the TDX module treats virtual NMIs as inhibited if the #VE
* valid flag is set. It means that NMI=>#VE will not result in a #DF.
*/
tdx_module_call(TDX_GET_VEINFO, 0, 0, 0, 0, &out);
tdcall(TDG_VP_VEINFO_GET, &args);
/* Transfer the output parameters */
ve->exit_reason = out.rcx;
ve->exit_qual = out.rdx;
ve->gla = out.r8;
ve->gpa = out.r9;
ve->instr_len = lower_32_bits(out.r10);
ve->instr_info = upper_32_bits(out.r10);
ve->exit_reason = args.rcx;
ve->exit_qual = args.rdx;
ve->gla = args.r8;
ve->gpa = args.r9;
ve->instr_len = lower_32_bits(args.r10);
ve->instr_info = upper_32_bits(args.r10);
}
/*
@ -702,6 +705,57 @@ static bool tdx_cache_flush_required(void)
return true;
}
/*
* Notify the VMM about page mapping conversion. More info about ABI
* can be found in TDX Guest-Host-Communication Interface (GHCI),
* section "TDG.VP.VMCALL<MapGPA>".
*/
static bool tdx_map_gpa(phys_addr_t start, phys_addr_t end, bool enc)
{
/* Retrying the hypercall a second time should succeed; use 3 just in case */
const int max_retries_per_page = 3;
int retry_count = 0;
if (!enc) {
/* Set the shared (decrypted) bits: */
start |= cc_mkdec(0);
end |= cc_mkdec(0);
}
while (retry_count < max_retries_per_page) {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = TDVMCALL_MAP_GPA,
.r12 = start,
.r13 = end - start };
u64 map_fail_paddr;
u64 ret = __tdx_hypercall(&args);
if (ret != TDVMCALL_STATUS_RETRY)
return !ret;
/*
* The guest must retry the operation for the pages in the
* region starting at the GPA specified in R11. R11 comes
* from the untrusted VMM. Sanity check it.
*/
map_fail_paddr = args.r11;
if (map_fail_paddr < start || map_fail_paddr >= end)
return false;
/* "Consume" a retry without forward progress */
if (map_fail_paddr == start) {
retry_count++;
continue;
}
start = map_fail_paddr;
retry_count = 0;
}
return false;
}
/*
* Inform the VMM of the guest's intent for this physical page: shared with
* the VMM or private to the guest. The VMM is expected to change its mapping
@ -712,18 +766,7 @@ static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)
phys_addr_t start = __pa(vaddr);
phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE);
if (!enc) {
/* Set the shared (decrypted) bits: */
start |= cc_mkdec(0);
end |= cc_mkdec(0);
}
/*
* Notify the VMM about page mapping conversion. More info about ABI
* can be found in TDX Guest-Host-Communication Interface (GHCI),
* section "TDG.VP.VMCALL<MapGPA>"
*/
if (_tdx_hypercall(TDVMCALL_MAP_GPA, start, end - start, 0, 0))
if (!tdx_map_gpa(start, end, enc))
return false;
/* shared->private conversion requires memory to be accepted before use */
@ -759,6 +802,10 @@ static bool tdx_enc_status_change_finish(unsigned long vaddr, int numpages,
void __init tdx_early_init(void)
{
struct tdx_module_args args = {
.rdx = TDCS_NOTIFY_ENABLES,
.r9 = -1ULL,
};
u64 cc_mask;
u32 eax, sig[3];
@ -769,12 +816,15 @@ void __init tdx_early_init(void)
setup_force_cpu_cap(X86_FEATURE_TDX_GUEST);
/* TSC is the only reliable clock in TDX guest */
setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
cc_vendor = CC_VENDOR_INTEL;
tdx_parse_tdinfo(&cc_mask);
cc_set_mask(cc_mask);
/* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */
tdx_module_call(TDX_WR, 0, TDCS_NOTIFY_ENABLES, 0, -1ULL, NULL);
tdcall(TDG_VM_WR, &args);
/*
* All bits above GPA width are reserved and kernel treats shared bit

10
arch/x86/hyperv/ivm.c
View file

@ -384,7 +384,7 @@ static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}
#ifdef CONFIG_INTEL_TDX_GUEST
static void hv_tdx_msr_write(u64 msr, u64 val)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = EXIT_REASON_MSR_WRITE,
.r12 = msr,
@ -398,13 +398,13 @@ static void hv_tdx_msr_write(u64 msr, u64 val)
static void hv_tdx_msr_read(u64 msr, u64 *val)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = EXIT_REASON_MSR_READ,
.r12 = msr,
};
u64 ret = __tdx_hypercall_ret(&args);
u64 ret = __tdx_hypercall(&args);
if (WARN_ONCE(ret, "Failed to emulate MSR read: %lld\n", ret))
*val = 0;
@ -414,13 +414,13 @@ static void hv_tdx_msr_read(u64 msr, u64 *val)
u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2)
{
struct tdx_hypercall_args args = { };
struct tdx_module_args args = { };
args.r10 = control;
args.rdx = param1;
args.r8 = param2;
(void)__tdx_hypercall_ret(&args);
(void)__tdx_hypercall(&args);
return args.r11;
}

91
arch/x86/include/asm/shared/tdx.h
View file

@ -11,11 +11,12 @@
#define TDX_IDENT "IntelTDX "
/* TDX module Call Leaf IDs */
#define TDX_GET_INFO 1
#define TDX_GET_VEINFO 3
#define TDX_GET_REPORT 4
#define TDX_ACCEPT_PAGE 6
#define TDX_WR 8
#define TDG_VP_VMCALL 0
#define TDG_VP_INFO 1
#define TDG_VP_VEINFO_GET 3
#define TDG_MR_REPORT 4
#define TDG_MEM_PAGE_ACCEPT 6
#define TDG_VM_WR 8
/* TDCS fields. To be used by TDG.VM.WR and TDG.VM.RD module calls */
#define TDCS_NOTIFY_ENABLES 0x9100000000000010
@ -24,32 +25,70 @@
#define TDVMCALL_MAP_GPA 0x10001
#define TDVMCALL_REPORT_FATAL_ERROR 0x10003
#ifndef __ASSEMBLY__
#define TDVMCALL_STATUS_RETRY 1
/*
* Used in __tdx_hypercall() to pass down and get back registers' values of
* the TDCALL instruction when requesting services from the VMM.
*
* This is a software only structure and not part of the TDX module/VMM ABI.
* Bitmasks of exposed registers (with VMM).
*/
struct tdx_hypercall_args {
#define TDX_RDX BIT(2)
#define TDX_RBX BIT(3)
#define TDX_RSI BIT(6)
#define TDX_RDI BIT(7)
#define TDX_R8 BIT(8)
#define TDX_R9 BIT(9)
#define TDX_R10 BIT(10)
#define TDX_R11 BIT(11)
#define TDX_R12 BIT(12)
#define TDX_R13 BIT(13)
#define TDX_R14 BIT(14)
#define TDX_R15 BIT(15)
/*
* These registers are clobbered to hold arguments for each
* TDVMCALL. They are safe to expose to the VMM.
* Each bit in this mask represents a register ID. Bit field
* details can be found in TDX GHCI specification, section
* titled "TDCALL [TDG.VP.VMCALL] leaf".
*/
#define TDVMCALL_EXPOSE_REGS_MASK \
(TDX_RDX | TDX_RBX | TDX_RSI | TDX_RDI | TDX_R8 | TDX_R9 | \
TDX_R10 | TDX_R11 | TDX_R12 | TDX_R13 | TDX_R14 | TDX_R15)
#ifndef __ASSEMBLY__
#include <linux/compiler_attributes.h>
/*
* Used in __tdcall*() to gather the input/output registers' values of the
* TDCALL instruction when requesting services from the TDX module. This is a
* software only structure and not part of the TDX module/VMM ABI
*/
struct tdx_module_args {
/* callee-clobbered */
u64 rcx;
u64 rdx;
u64 r8;
u64 r9;
/* extra callee-clobbered */
u64 r10;
u64 r11;
/* callee-saved + rdi/rsi */
u64 r12;
u64 r13;
u64 r14;
u64 r15;
u64 rbx;
u64 rdi;
u64 rsi;
u64 rbx;
u64 rdx;
};
/* Used to communicate with the TDX module */
u64 __tdcall(u64 fn, struct tdx_module_args *args);
u64 __tdcall_ret(u64 fn, struct tdx_module_args *args);
u64 __tdcall_saved_ret(u64 fn, struct tdx_module_args *args);
/* Used to request services from the VMM */
u64 __tdx_hypercall(struct tdx_hypercall_args *args);
u64 __tdx_hypercall_ret(struct tdx_hypercall_args *args);
u64 __tdx_hypercall(struct tdx_module_args *args);
/*
* Wrapper for standard use of __tdx_hypercall with no output aside from
@ -57,7 +96,7 @@ u64 __tdx_hypercall_ret(struct tdx_hypercall_args *args);
*/
static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15)
{
struct tdx_hypercall_args args = {
struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = fn,
.r12 = r12,
@ -71,25 +110,7 @@ static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15)
/* Called from __tdx_hypercall() for unrecoverable failure */
void __tdx_hypercall_failed(void);
/*
* Used in __tdx_module_call() to gather the output registers' values of the
* TDCALL instruction when requesting services from the TDX module. This is a
* software only structure and not part of the TDX module/VMM ABI
*/
struct tdx_module_output {
u64 rcx;
u64 rdx;
u64 r8;
u64 r9;
u64 r10;
u64 r11;
};
/* Used to communicate with the TDX module */
u64 __tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9,
struct tdx_module_output *out);
void __noreturn __tdx_hypercall_failed(void);
bool tdx_accept_memory(phys_addr_t start, phys_addr_t end);

11
arch/x86/include/asm/tdx.h
View file

@ -8,6 +8,7 @@
#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/trapnr.h>
#include <asm/shared/tdx.h>
/*
@ -20,6 +21,9 @@
#define TDX_SW_ERROR (TDX_ERROR | GENMASK_ULL(47, 40))
#define TDX_SEAMCALL_VMFAILINVALID (TDX_SW_ERROR | _UL(0xFFFF0000))
#define TDX_SEAMCALL_GP (TDX_SW_ERROR | X86_TRAP_GP)
#define TDX_SEAMCALL_UD (TDX_SW_ERROR | X86_TRAP_UD)
#ifndef __ASSEMBLY__
/*
@ -72,5 +76,12 @@ static inline long tdx_kvm_hypercall(unsigned int nr, unsigned long p1,
return -ENODEV;
}
#endif /* CONFIG_INTEL_TDX_GUEST && CONFIG_KVM_GUEST */
#ifdef CONFIG_INTEL_TDX_HOST
u64 __seamcall(u64 fn, struct tdx_module_args *args);
u64 __seamcall_ret(u64 fn, struct tdx_module_args *args);
u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args);
#endif /* CONFIG_INTEL_TDX_HOST */
#endif /* !__ASSEMBLY__ */
#endif /* _ASM_X86_TDX_H */

33
arch/x86/kernel/asm-offsets.c
View file

@ -68,26 +68,19 @@ static void __used common(void)
#endif
BLANK();
OFFSET(TDX_MODULE_rcx, tdx_module_output, rcx);
OFFSET(TDX_MODULE_rdx, tdx_module_output, rdx);
OFFSET(TDX_MODULE_r8, tdx_module_output, r8);
OFFSET(TDX_MODULE_r9, tdx_module_output, r9);
OFFSET(TDX_MODULE_r10, tdx_module_output, r10);
OFFSET(TDX_MODULE_r11, tdx_module_output, r11);
BLANK();
OFFSET(TDX_HYPERCALL_r8, tdx_hypercall_args, r8);
OFFSET(TDX_HYPERCALL_r9, tdx_hypercall_args, r9);
OFFSET(TDX_HYPERCALL_r10, tdx_hypercall_args, r10);
OFFSET(TDX_HYPERCALL_r11, tdx_hypercall_args, r11);
OFFSET(TDX_HYPERCALL_r12, tdx_hypercall_args, r12);
OFFSET(TDX_HYPERCALL_r13, tdx_hypercall_args, r13);
OFFSET(TDX_HYPERCALL_r14, tdx_hypercall_args, r14);
OFFSET(TDX_HYPERCALL_r15, tdx_hypercall_args, r15);
OFFSET(TDX_HYPERCALL_rdi, tdx_hypercall_args, rdi);
OFFSET(TDX_HYPERCALL_rsi, tdx_hypercall_args, rsi);
OFFSET(TDX_HYPERCALL_rbx, tdx_hypercall_args, rbx);
OFFSET(TDX_HYPERCALL_rdx, tdx_hypercall_args, rdx);
OFFSET(TDX_MODULE_rcx, tdx_module_args, rcx);
OFFSET(TDX_MODULE_rdx, tdx_module_args, rdx);
OFFSET(TDX_MODULE_r8, tdx_module_args, r8);
OFFSET(TDX_MODULE_r9, tdx_module_args, r9);
OFFSET(TDX_MODULE_r10, tdx_module_args, r10);
OFFSET(TDX_MODULE_r11, tdx_module_args, r11);
OFFSET(TDX_MODULE_r12, tdx_module_args, r12);
OFFSET(TDX_MODULE_r13, tdx_module_args, r13);
OFFSET(TDX_MODULE_r14, tdx_module_args, r14);
OFFSET(TDX_MODULE_r15, tdx_module_args, r15);
OFFSET(TDX_MODULE_rbx, tdx_module_args, rbx);
OFFSET(TDX_MODULE_rdi, tdx_module_args, rdi);
OFFSET(TDX_MODULE_rsi, tdx_module_args, rsi);
BLANK();
OFFSET(BP_scratch, boot_params, scratch);

2
arch/x86/virt/Makefile Normal file
View file

@ -0,0 +1,2 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-y += vmx/

2
arch/x86/virt/vmx/Makefile Normal file
View file

@ -0,0 +1,2 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_INTEL_TDX_HOST) += tdx/

2
arch/x86/virt/vmx/tdx/Makefile Normal file
View file

@ -0,0 +1,2 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-y += seamcall.o

61
arch/x86/virt/vmx/tdx/seamcall.S Normal file
View file

@ -0,0 +1,61 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/linkage.h>
#include <asm/frame.h>
#include "tdxcall.S"
/*
* __seamcall() - Host-side interface functions to SEAM software
* (the P-SEAMLDR or the TDX module).
*
* __seamcall() function ABI:
*
* @fn (RDI) - SEAMCALL Leaf number, moved to RAX
* @args (RSI) - struct tdx_module_args for input
*
* Only RCX/RDX/R8-R11 are used as input registers.
*
* Return (via RAX) TDX_SEAMCALL_VMFAILINVALID if the SEAMCALL itself
* fails, or the completion status of the SEAMCALL leaf function.
*/
SYM_FUNC_START(__seamcall)
TDX_MODULE_CALL host=1
SYM_FUNC_END(__seamcall)
/*
* __seamcall_ret() - Host-side interface functions to SEAM software
* (the P-SEAMLDR or the TDX module), with saving output registers to
* the 'struct tdx_module_args' used as input.
*
* __seamcall_ret() function ABI:
*
* @fn (RDI) - SEAMCALL Leaf number, moved to RAX
* @args (RSI) - struct tdx_module_args for input and output
*
* Only RCX/RDX/R8-R11 are used as input/output registers.
*
* Return (via RAX) TDX_SEAMCALL_VMFAILINVALID if the SEAMCALL itself
* fails, or the completion status of the SEAMCALL leaf function.
*/
SYM_FUNC_START(__seamcall_ret)
TDX_MODULE_CALL host=1 ret=1
SYM_FUNC_END(__seamcall_ret)
/*
* __seamcall_saved_ret() - Host-side interface functions to SEAM software
* (the P-SEAMLDR or the TDX module), with saving output registers to the
* 'struct tdx_module_args' used as input.
*
* __seamcall_saved_ret() function ABI:
*
* @fn (RDI) - SEAMCALL Leaf number, moved to RAX
* @args (RSI) - struct tdx_module_args for input and output
*
* All registers in @args are used as input/output registers.
*
* Return (via RAX) TDX_SEAMCALL_VMFAILINVALID if the SEAMCALL itself
* fails, or the completion status of the SEAMCALL leaf function.
*/
SYM_FUNC_START(__seamcall_saved_ret)
TDX_MODULE_CALL host=1 ret=1 saved=1
SYM_FUNC_END(__seamcall_saved_ret)

230
arch/x86/virt/vmx/tdx/tdxcall.S
View file

@ -1,5 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/asm-offsets.h>
#include <asm/frame.h>
#include <asm/asm.h>
#include <asm/tdx.h>
/*
@ -16,35 +18,75 @@
* TDX module and hypercalls to the VMM.
* SEAMCALL - used by TDX hosts to make requests to the
* TDX module.
*
*-------------------------------------------------------------------------
* TDCALL/SEAMCALL ABI:
*-------------------------------------------------------------------------
* Input Registers:
*
* RAX - TDCALL/SEAMCALL Leaf number.
* RCX,RDX,RDI,RSI,RBX,R8-R15 - TDCALL/SEAMCALL Leaf specific input registers.
*
* Output Registers:
*
* RAX - TDCALL/SEAMCALL instruction error code.
* RCX,RDX,RDI,RSI,RBX,R8-R15 - TDCALL/SEAMCALL Leaf specific output registers.
*
*-------------------------------------------------------------------------
*
* So while the common core (RAX,RCX,RDX,R8-R11) fits nicely in the
* callee-clobbered registers and even leaves RDI,RSI free to act as a
* base pointer, some leafs (e.g., VP.ENTER) make a giant mess of things.
*
* For simplicity, assume that anything that needs the callee-saved regs
* also tramples on RDI,RSI. This isn't strictly true, see for example
* TDH.EXPORT.MEM.
*/
.macro TDX_MODULE_CALL host:req
/*
* R12 will be used as temporary storage for struct tdx_module_output
* pointer. Since R12-R15 registers are not used by TDCALL/SEAMCALL
* services supported by this function, it can be reused.
*/
.macro TDX_MODULE_CALL host:req ret=0 saved=0
FRAME_BEGIN
/* Callee saved, so preserve it */
push %r12
/*
* Push output pointer to stack.
* After the operation, it will be fetched into R12 register.
*/
push %r9
/* Mangle function call ABI into TDCALL/SEAMCALL ABI: */
/* Move Leaf ID to RAX */
mov %rdi, %rax
/* Move input 4 to R9 */
mov %r8, %r9
/* Move input 3 to R8 */
mov %rcx, %r8
/* Move input 1 to RCX */
mov %rsi, %rcx
/* Leave input param 2 in RDX */
.if \host
/* Move other input regs from 'struct tdx_module_args' */
movq TDX_MODULE_rcx(%rsi), %rcx
movq TDX_MODULE_rdx(%rsi), %rdx
movq TDX_MODULE_r8(%rsi), %r8
movq TDX_MODULE_r9(%rsi), %r9
movq TDX_MODULE_r10(%rsi), %r10
movq TDX_MODULE_r11(%rsi), %r11
.if \saved
/*
* Move additional input regs from the structure. For simplicity
* assume that anything needs the callee-saved regs also tramples
* on RDI/RSI (see VP.ENTER).
*/
/* Save those callee-saved GPRs as mandated by the x86_64 ABI */
pushq %rbx
pushq %r12
pushq %r13
pushq %r14
pushq %r15
movq TDX_MODULE_r12(%rsi), %r12
movq TDX_MODULE_r13(%rsi), %r13
movq TDX_MODULE_r14(%rsi), %r14
movq TDX_MODULE_r15(%rsi), %r15
movq TDX_MODULE_rbx(%rsi), %rbx
.if \ret
/* Save the structure pointer as RSI is about to be clobbered */
pushq %rsi
.endif
movq TDX_MODULE_rdi(%rsi), %rdi
/* RSI needs to be done at last */
movq TDX_MODULE_rsi(%rsi), %rsi
.endif /* \saved */
.if \host
.Lseamcall\@:
seamcall
/*
* SEAMCALL instruction is essentially a VMExit from VMX root
@ -57,40 +99,122 @@
* This value will never be used as actual SEAMCALL error code as
* it is from the Reserved status code class.
*/
jnc .Lno_vmfailinvalid
mov $TDX_SEAMCALL_VMFAILINVALID, %rax
.Lno_vmfailinvalid:
.else
jc .Lseamcall_vmfailinvalid\@
.else
tdcall
.endif
.endif
.if \ret
.if \saved
/*
* Fetch output pointer from stack to R12 (It is used
* as temporary storage)
*/
pop %r12
/*
* Since this macro can be invoked with NULL as an output pointer,
* check if caller provided an output struct before storing output
* registers.
* Restore the structure from stack to save the output registers
*
* Update output registers, even if the call failed (RAX != 0).
* Other registers may contain details of the failure.
* In case of VP.ENTER returns due to TDVMCALL, all registers are
* valid thus no register can be used as spare to restore the
* structure from the stack (see "TDH.VP.ENTER Output Operands
* Definition on TDCALL(TDG.VP.VMCALL) Following a TD Entry").
* For this case, need to make one register as spare by saving it
* to the stack and then manually load the structure pointer to
* the spare register.
*
* Note for other TDCALLs/SEAMCALLs there are spare registers
* thus no need for such hack but just use this for all.
*/
test %r12, %r12
jz .Lno_output_struct
pushq %rax /* save the TDCALL/SEAMCALL return code */
movq 8(%rsp), %rax /* restore the structure pointer */
movq %rsi, TDX_MODULE_rsi(%rax) /* save RSI */
popq %rax /* restore the return code */
popq %rsi /* pop the structure pointer */
/* Copy result registers to output struct: */
movq %rcx, TDX_MODULE_rcx(%r12)
movq %rdx, TDX_MODULE_rdx(%r12)
movq %r8, TDX_MODULE_r8(%r12)
movq %r9, TDX_MODULE_r9(%r12)
movq %r10, TDX_MODULE_r10(%r12)
movq %r11, TDX_MODULE_r11(%r12)
/* Copy additional output regs to the structure */
movq %r12, TDX_MODULE_r12(%rsi)
movq %r13, TDX_MODULE_r13(%rsi)
movq %r14, TDX_MODULE_r14(%rsi)
movq %r15, TDX_MODULE_r15(%rsi)
movq %rbx, TDX_MODULE_rbx(%rsi)
movq %rdi, TDX_MODULE_rdi(%rsi)
.endif /* \saved */
/* Copy output registers to the structure */
movq %rcx, TDX_MODULE_rcx(%rsi)
movq %rdx, TDX_MODULE_rdx(%rsi)
movq %r8, TDX_MODULE_r8(%rsi)
movq %r9, TDX_MODULE_r9(%rsi)
movq %r10, TDX_MODULE_r10(%rsi)
movq %r11, TDX_MODULE_r11(%rsi)
.endif /* \ret */
.if \saved && \ret
/*
* Clear registers shared by guest for VP.VMCALL/VP.ENTER to prevent
* speculative use of guest's/VMM's values, including those are
* restored from the stack.
*
* See arch/x86/kvm/vmx/vmenter.S:
*
* In theory, a L1 cache miss when restoring register from stack
* could lead to speculative execution with guest's values.
*
* Note: RBP/RSP are not used as shared register. RSI has been
* restored already.
*
* XOR is cheap, thus unconditionally do for all leafs.
*/
xorl %ecx, %ecx
xorl %edx, %edx
xorl %r8d, %r8d
xorl %r9d, %r9d
xorl %r10d, %r10d
xorl %r11d, %r11d
xorl %r12d, %r12d
xorl %r13d, %r13d
xorl %r14d, %r14d
xorl %r15d, %r15d
xorl %ebx, %ebx
xorl %edi, %edi
.endif /* \ret && \host */
.if \host
.Lout\@:
.endif
.if \saved
/* Restore callee-saved GPRs as mandated by the x86_64 ABI */
popq %r15
popq %r14
popq %r13
popq %r12
popq %rbx
.endif /* \saved */
FRAME_END
RET
.if \host
.Lseamcall_vmfailinvalid\@:
mov $TDX_SEAMCALL_VMFAILINVALID, %rax
jmp .Lseamcall_fail\@
.Lseamcall_trap\@:
/*
* SEAMCALL caused #GP or #UD. By reaching here RAX contains
* the trap number. Convert the trap number to the TDX error
* code by setting TDX_SW_ERROR to the high 32-bits of RAX.
*
* Note cannot OR TDX_SW_ERROR directly to RAX as OR instruction
* only accepts 32-bit immediate at most.
*/
movq $TDX_SW_ERROR, %rdi
orq %rdi, %rax
.Lseamcall_fail\@:
.if \ret && \saved
/* pop the unused structure pointer back to RSI */
popq %rsi
.endif
jmp .Lout\@
_ASM_EXTABLE_FAULT(.Lseamcall\@, .Lseamcall_trap\@)
.endif /* \host */
.Lno_output_struct:
/* Restore the state of R12 register */
pop %r12
.endm

1
tools/objtool/noreturns.h
View file

@ -11,6 +11,7 @@ NORETURN(__kunit_abort)
NORETURN(__module_put_and_kthread_exit)
NORETURN(__reiserfs_panic)
NORETURN(__stack_chk_fail)
NORETURN(__tdx_hypercall_failed)
NORETURN(__ubsan_handle_builtin_unreachable)
NORETURN(arch_call_rest_init)
NORETURN(arch_cpu_idle_dead)