system/freebsd-src
system/freebsd-src
1
0
Fork 0
mirror of https://github.com/freebsd/freebsd-src synced 2024-10-03 15:15:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

linux(4): Implement vdso getcpu for x86.

Browse source 
This is modeled after f2395455 (by kib@).

MFC after:		2 weeks

(cherry picked from commit 5a6a4fb284)
This commit is contained in:
Dmitry Chagin 2022-05-08 17:20:52 +03:00
parent f360bdf68d
commit e7a07ad1db
15 changed files with 171 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
sys/amd64/linux/linux_sysvec.c
View file

@ -172,6 +172,7 @@ LINUX_VDSO_SYM_INTPTR(linux_rt_sigcode);
LINUX_VDSO_SYM_CHAR(linux_platform);
LINUX_VDSO_SYM_INTPTR(kern_timekeep_base);
LINUX_VDSO_SYM_INTPTR(kern_tsc_selector);
LINUX_VDSO_SYM_INTPTR(kern_cpu_selector);
/*
* If FreeBSD & Linux have a difference of opinion about what a trap
@ -836,6 +837,12 @@ linux_exec_sysvec_init(void *param)
*ktsc_selector = linux_vdso_tsc_selector_idx();
if (bootverbose)
printf("Linux x86-64 vDSO tsc_selector: %lu\n", *ktsc_selector);
tkoff = kern_cpu_selector - linux_vdso_base;
ktsc_selector = (l_uintptr_t *)(linux_vdso_mapping + tkoff);
*ktsc_selector = linux_vdso_cpu_selector_idx();
if (bootverbose)
printf("Linux x86-64 vDSO cpu_selector: %lu\n", *ktsc_selector);
}
SYSINIT(elf_linux_exec_sysvec_init, SI_SUB_EXEC + 1, SI_ORDER_ANY,
linux_exec_sysvec_init, &elf_linux_sysvec);

1
sys/amd64/linux/linux_vdso.lds.s
View file

@ -68,6 +68,7 @@ VERSION
linux_platform;
kern_timekeep_base;
kern_tsc_selector;
kern_cpu_selector;
local: *;
};
}

2
sys/amd64/linux/linux_vdso_gtod.c
View file

@ -49,8 +49,10 @@ __FBSDID("$FreeBSD$");
/* The kernel fixup this at vDSO install */
uintptr_t *kern_timekeep_base = NULL;
uint32_t kern_tsc_selector = 0;
uint32_t kern_cpu_selector = 0;
#include <x86/linux/linux_vdso_gettc_x86.inc>
#include <x86/linux/linux_vdso_getcpu_x86.inc>
/* for debug purpose */
static int

7
sys/amd64/linux32/linux32_sysvec.c
View file

@ -185,6 +185,7 @@ LINUX_VDSO_SYM_INTPTR(__kernel_sigreturn);
LINUX_VDSO_SYM_INTPTR(__kernel_rt_sigreturn);
LINUX_VDSO_SYM_INTPTR(kern_timekeep_base);
LINUX_VDSO_SYM_INTPTR(kern_tsc_selector);
LINUX_VDSO_SYM_INTPTR(kern_cpu_selector);
LINUX_VDSO_SYM_CHAR(linux_platform);
/*
@ -999,6 +1000,12 @@ linux_exec_sysvec_init(void *param)
*ktsc_selector = linux_vdso_tsc_selector_idx();
if (bootverbose)
printf("Linux i386 vDSO tsc_selector: %u\n", *ktsc_selector);
tkoff = kern_cpu_selector - linux_vdso_base;
ktsc_selector = (l_uintptr_t *)(linux_vdso_mapping + tkoff);
*ktsc_selector = linux_vdso_cpu_selector_idx();
if (bootverbose)
printf("Linux i386 vDSO cpu_selector: %u\n", *ktsc_selector);
}
SYSINIT(elf_linux_exec_sysvec_init, SI_SUB_EXEC + 1, SI_ORDER_ANY,
linux_exec_sysvec_init, &elf_linux_sysvec);

2
sys/amd64/linux32/linux32_vdso.lds.s
View file

@ -58,6 +58,7 @@ VERSION
__vdso_clock_gettime;
__vdso_gettimeofday;
__vdso_time;
__vdso_getcpu;
__vdso_clock_getres;
__vdso_clock_gettime64;
};
@ -75,6 +76,7 @@ VERSION
linux_platform;
kern_timekeep_base;
kern_tsc_selector;
kern_cpu_selector;
local: *;
};
}

17
sys/amd64/linux32/linux32_vdso_gtod.c
View file

@ -50,8 +50,10 @@ __FBSDID("$FreeBSD$");
/* The kernel fixup this at vDSO install */
uintptr_t *kern_timekeep_base = NULL;
uint32_t kern_tsc_selector = 0;
uint32_t kern_cpu_selector = 0;
#include <x86/linux/linux_vdso_gettc_x86.inc>
#include <x86/linux/linux_vdso_getcpu_x86.inc>
static int
write(int fd, const void *buf, size_t size)
@ -128,6 +130,21 @@ __vdso_clock_getres_fallback(clockid_t clock_id, struct l_timespec *ts)
return (res);
}
static int
__vdso_getcpu_fallback(uint32_t *cpu, uint32_t *node, void *cache)
{
int res;
__asm__ __volatile__
(
"int $0x80"
: "=a"(res)
: "a"(LINUX32_SYS_linux_getcpu), "D"(cpu), "S"(node), "d"(cache)
: "cc", "memory"
);
return (res);
}
static int
__vdso_time_fallback(long *tm)
{

11
sys/compat/linux/linux_vdso_gtod.inc
View file

@ -352,12 +352,19 @@ int clock_gettime64(clockid_t clock_id, struct l_timespec64 *lts)
__attribute__((weak, alias("__vdso_clock_gettime64")));
#endif
#if defined(__amd64__) && !defined(COMPAT_LINUX32)
#if defined(__i386__) || defined(__amd64__)
int
__vdso_getcpu(uint32_t *cpu, uint32_t *node, void *cache)
{
int ret;
return (__vdso_getcpu_fallback(cpu, node, cache));
if (node != NULL)
return (__vdso_getcpu_fallback(cpu, node, cache));
ret = __vdso_getcpu_try();
if (ret < 0)
return (__vdso_getcpu_fallback(cpu, node, cache));
*cpu = ret;
return (0);
}
#endif

7
sys/i386/linux/linux_sysvec.c
View file

@ -160,6 +160,7 @@ LINUX_VDSO_SYM_INTPTR(__kernel_sigreturn);
LINUX_VDSO_SYM_INTPTR(__kernel_rt_sigreturn);
LINUX_VDSO_SYM_INTPTR(kern_timekeep_base);
LINUX_VDSO_SYM_INTPTR(kern_tsc_selector);
LINUX_VDSO_SYM_INTPTR(kern_cpu_selector);
/*
* If FreeBSD & Linux have a difference of opinion about what a trap
@ -941,6 +942,12 @@ linux_exec_sysvec_init(void *param)
*ktsc_selector = linux_vdso_tsc_selector_idx();
if (bootverbose)
printf("Linux i386 vDSO tsc_selector: %u\n", *ktsc_selector);
tkoff = kern_cpu_selector - linux_vdso_base;
ktsc_selector = (l_uintptr_t *)(linux_vdso_mapping + tkoff);
*ktsc_selector = linux_vdso_cpu_selector_idx();
if (bootverbose)
printf("Linux i386 vDSO cpu_selector: %u\n", *ktsc_selector);
}
SYSINIT(elf_linux_exec_sysvec_init, SI_SUB_EXEC + 1, SI_ORDER_ANY,
linux_exec_sysvec_init, &elf_linux_sysvec);

2
sys/i386/linux/linux_vdso.lds.s
View file

@ -58,6 +58,7 @@ VERSION
__vdso_clock_gettime;
__vdso_gettimeofday;
__vdso_time;
__vdso_getcpu;
__vdso_clock_getres;
__vdso_clock_gettime64;
};
@ -75,6 +76,7 @@ VERSION
linux_platform;
kern_timekeep_base;
kern_tsc_selector;
kern_cpu_selector;
local: *;
};
}

17
sys/i386/linux/linux_vdso_gtod.c
View file

@ -49,8 +49,10 @@ __FBSDID("$FreeBSD$");
/* The kernel fixup this at vDSO install */
uintptr_t *kern_timekeep_base = NULL;
uint32_t kern_tsc_selector = 0;
uint32_t kern_cpu_selector = 0;
#include <x86/linux/linux_vdso_gettc_x86.inc>
#include <x86/linux/linux_vdso_getcpu_x86.inc>
static int
write(int fd, const void *buf, size_t size)
@ -127,6 +129,21 @@ __vdso_clock_getres_fallback(clockid_t clock_id, struct l_timespec *ts)
return (res);
}
static int
__vdso_getcpu_fallback(uint32_t *cpu, uint32_t *node, void *cache)
{
int res;
__asm__ __volatile__
(
"int $0x80"
: "=a"(res)
: "a"(LINUX_SYS_linux_getcpu), "D"(cpu), "S"(node), "d"(cache)
: "cc", "memory"
);
return (res);
}
static int
__vdso_time_fallback(long *tm)
{

2
sys/modules/linux/Makefile
View file

@ -20,7 +20,7 @@ SRCS= linux_fork.c linux${SFX}_dummy_machdep.c linux_file.c linux_event.c \
opt_inet6.h opt_compat.h opt_posix.h opt_usb.h vnode_if.h \
device_if.h bus_if.h
.if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64"
SRCS+= linux_dummy_x86.c linux_vdso_tsc_selector_x86.c
SRCS+= linux_dummy_x86.c linux_vdso_selector_x86.c
VDSODEPS=linux_vdso_gettc_x86.inc
.endif
.if ${MACHINE_CPUARCH} == "amd64"

2
sys/modules/linux64/Makefile
View file

@ -15,7 +15,7 @@ SRCS= linux_elf64.c linux_fork.c linux_dummy_machdep.c linux_file.c \
vnode_if.h device_if.h bus_if.h \
linux_support.s
.if ${MACHINE_CPUARCH} == "amd64"
SRCS+= linux_dummy_x86.c linux_vdso_tsc_selector_x86.c
SRCS+= linux_dummy_x86.c linux_vdso_selector_x86.c
.endif
DPSRCS= assym.inc linux_genassym.c

68
sys/x86/linux/linux_vdso_getcpu_x86.inc Normal file
View file

@ -0,0 +1,68 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2021 The FreeBSD Foundation
* Copyright (c) 2022 Dmitry Chagin <dchagin@FreeBSD.org>
*
* Portions of this software were developed by Konstantin Belousov
* under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <x86/linux/linux_x86.h>
static int
__vdso_getcpu_rdpid(void)
{
register_t res;
__asm("rdpid %0" : "=r" (res));
return ((int)res);
}
static int
__vdso_getcpu_rdtscp(void)
{
int res;
__asm("rdtscp" : "=c" (res) : : "eax", "edx");
return (res);
}
static int
__vdso_getcpu_try(void)
{
int res;
switch (kern_cpu_selector) {
case LINUX_VDSO_CPU_RDTSCP:
res = __vdso_getcpu_rdtscp();
break;
case LINUX_VDSO_CPU_RDPID:
res = __vdso_getcpu_rdpid();
break;
default:
res = -1;
}
return (res);
}

26
sys/x86/linux/linux_vdso_tsc_selector_x86.c → sys/x86/linux/linux_vdso_selector_x86.c
View file

@ -1,6 +1,6 @@
/*-
* Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
* Copyright (c) 2016, 2017, 2019 The FreeBSD Foundation
* Copyright (c) 2016, 2017, 2019, 2021 The FreeBSD Foundation
* Copyright (c) 2021 Dmitry Chagin <dchagin@FreeBSD.org>
*
* Portions of this software were developed by Konstantin Belousov
@ -36,6 +36,8 @@ __FBSDID("$FreeBSD$");
#include <x86/x86_var.h>
#include <x86/specialreg.h>
#include <machine/cpufunc.h>
#include <x86/linux/linux_x86.h>
int
@ -55,3 +57,25 @@ linux_vdso_tsc_selector_idx()
return (2);
return (amd_cpu ? 1 : 0);
}
int
linux_vdso_cpu_selector_idx()
{
u_int amd_feature, cpu_exthigh, p[4];
if ((cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0)
return (LINUX_VDSO_CPU_RDPID);
amd_feature = 0;
if (cpu_feature != 0) {
do_cpuid(0x80000000, p);
cpu_exthigh = p[0];
if (cpu_exthigh >= 0x80000001) {
do_cpuid(0x80000001, p);
amd_feature = p[3];
}
}
return ((amd_feature & AMDID_RDTSCP) == 0 ?
LINUX_VDSO_CPU_DEFAULT : LINUX_VDSO_CPU_RDTSCP);
}

5
sys/x86/linux/linux_x86.h
View file

@ -28,6 +28,11 @@
#ifndef _X86_INCLUDE_LINUX_LINUX_X86_H_
#define _X86_INCLUDE_LINUX_LINUX_X86_H_
#define LINUX_VDSO_CPU_DEFAULT 0
#define LINUX_VDSO_CPU_RDPID 1
#define LINUX_VDSO_CPU_RDTSCP 2
int linux_vdso_tsc_selector_idx(void);
int linux_vdso_cpu_selector_idx(void);
#endif /* _X86_INCLUDE_LINUX_LINUX_X86_H_ */