Kernel: Take some baby steps towards x86_64

Make more of the kernel compile in 64-bit mode, and make some things
pointer-size-agnostic (by using FlatPtr.)

There's a lot of work to do here before the kernel will even compile.
This commit is contained in:
Andreas Kling 2021-02-25 16:18:36 +01:00
parent eb08a0edd5
commit 8f70528f30
12 changed files with 187 additions and 118 deletions

View file

@ -318,7 +318,7 @@ struct SC_futex_params {
u32 val;
union {
const timespec* timeout;
u32 val2;
uintptr_t val2;
};
u32* userspace_address2;
u32 val3;

View file

@ -210,9 +210,7 @@ void page_fault_handler(TrapFrame* trap)
clac();
auto& regs = *trap->regs;
u32 fault_address;
asm("movl %%cr2, %%eax"
: "=a"(fault_address));
auto fault_address = read_cr2();
if constexpr (PAGE_FAULT_DEBUG) {
u32 fault_page_directory = read_cr3();
@ -717,14 +715,22 @@ void exit_trap(TrapFrame* trap)
return Processor::current().exit_trap(*trap);
}
UNMAP_AFTER_INIT void write_cr0(u32 value)
UNMAP_AFTER_INIT void write_cr0(FlatPtr value)
{
asm volatile("movl %%eax, %%cr0" ::"a"(value));
#if ARCH(I386)
asm volatile("mov %%eax, %%cr0" ::"a"(value));
#else
asm volatile("mov %%rax, %%cr0" ::"a"(value));
#endif
}
UNMAP_AFTER_INIT void write_cr4(u32 value)
UNMAP_AFTER_INIT void write_cr4(FlatPtr value)
{
asm volatile("movl %%eax, %%cr4" ::"a"(value));
#if ARCH(I386)
asm volatile("mov %%eax, %%cr4" ::"a"(value));
#else
asm volatile("mov %%rax, %%cr4" ::"a"(value));
#endif
}
UNMAP_AFTER_INIT static void sse_init()
@ -733,50 +739,80 @@ UNMAP_AFTER_INIT static void sse_init()
write_cr4(read_cr4() | 0x600);
}
u32 read_cr0()
FlatPtr read_cr0()
{
u32 cr0;
asm("movl %%cr0, %%eax"
FlatPtr cr0;
#if ARCH(I386)
asm("mov %%cr0, %%eax"
: "=a"(cr0));
#else
asm("mov %%cr0, %%rax"
: "=a"(cr0));
#endif
return cr0;
}
u32 read_cr2()
FlatPtr read_cr2()
{
u32 cr2;
asm("movl %%cr2, %%eax"
FlatPtr cr2;
#if ARCH(I386)
asm("mov %%cr2, %%eax"
: "=a"(cr2));
#else
asm("mov %%cr2, %%rax"
: "=a"(cr2));
#endif
return cr2;
}
u32 read_cr3()
FlatPtr read_cr3()
{
u32 cr3;
asm("movl %%cr3, %%eax"
FlatPtr cr3;
#if ARCH(I386)
asm("mov %%cr3, %%eax"
: "=a"(cr3));
#else
asm("mov %%cr3, %%rax"
: "=a"(cr3));
#endif
return cr3;
}
void write_cr3(u32 cr3)
void write_cr3(FlatPtr cr3)
{
// NOTE: If you're here from a GPF crash, it's very likely that a PDPT entry is incorrect, not this!
asm volatile("movl %%eax, %%cr3" ::"a"(cr3)
#if ARCH(I386)
asm volatile("mov %%eax, %%cr3" ::"a"(cr3)
: "memory");
#else
asm volatile("mov %%rax, %%cr3" ::"a"(cr3)
: "memory");
#endif
}
u32 read_cr4()
FlatPtr read_cr4()
{
u32 cr4;
asm("movl %%cr4, %%eax"
FlatPtr cr4;
#if ARCH(I386)
asm("mov %%cr4, %%eax"
: "=a"(cr4));
#else
asm("mov %%cr4, %%rax"
: "=a"(cr4));
#endif
return cr4;
}
u32 read_dr6()
FlatPtr read_dr6()
{
u32 dr6;
asm("movl %%dr6, %%eax"
FlatPtr dr6;
#if ARCH(I386)
asm("mov %%dr6, %%eax"
: "=a"(dr6));
#else
asm("mov %%dr6, %%rax"
: "=a"(dr6));
#endif
return dr6;
}
@ -1291,6 +1327,7 @@ void Processor::switch_context(Thread*& from_thread, Thread*& to_thread)
dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context --> switching out of: {} {}", VirtualAddress(from_thread), *from_thread);
from_thread->save_critical(m_in_critical);
#if ARCH(I386)
// clang-format off
// Switch to new thread context, passing from_thread and to_thread
// through to the new context using registers edx and eax
@ -1333,6 +1370,9 @@ void Processor::switch_context(Thread*& from_thread, Thread*& to_thread)
: "memory"
);
// clang-format on
#else
PANIC("Context switching not implemented.");
#endif
dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {}", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread);
@ -1576,6 +1616,7 @@ UNMAP_AFTER_INIT void Processor::initialize_context_switching(Thread& initial_th
m_scheduler_initialized = true;
#if ARCH(I386)
// clang-format off
asm volatile(
"movl %[new_esp], %%esp \n" // switch to new stack
@ -1601,6 +1642,7 @@ UNMAP_AFTER_INIT void Processor::initialize_context_switching(Thread& initial_th
[cpu] "c" (id())
);
// clang-format on
#endif
VERIFY_NOT_REACHED();
}

View file

@ -296,9 +296,9 @@ void load_task_register(u16 selector);
#define sti() asm volatile("sti" :: \
: "memory")
inline u32 cpu_flags()
inline FlatPtr cpu_flags()
{
u32 flags;
FlatPtr flags;
asm volatile(
"pushf\n"
"pop %0\n"
@ -441,29 +441,36 @@ private:
};
struct [[gnu::packed]] RegisterState {
u32 ss;
u32 gs;
u32 fs;
u32 es;
u32 ds;
u32 edi;
u32 esi;
u32 ebp;
u32 esp;
u32 ebx;
u32 edx;
u32 ecx;
u32 eax;
FlatPtr ss;
FlatPtr gs;
FlatPtr fs;
FlatPtr es;
FlatPtr ds;
FlatPtr edi;
FlatPtr esi;
FlatPtr ebp;
FlatPtr esp;
FlatPtr ebx;
FlatPtr edx;
FlatPtr ecx;
FlatPtr eax;
u16 exception_code;
u16 isr_number;
u32 eip;
u32 cs;
u32 eflags;
u32 userspace_esp;
u32 userspace_ss;
#if ARCH(X86_64)
u32 padding;
#endif
FlatPtr eip;
FlatPtr cs;
FlatPtr eflags;
FlatPtr userspace_esp;
FlatPtr userspace_ss;
};
#define REGISTER_STATE_SIZE (19 * 4)
#if ARCH(I386)
# define REGISTER_STATE_SIZE (19 * 4)
#else
# define REGISTER_STATE_SIZE (19 * 8)
#endif
static_assert(REGISTER_STATE_SIZE == sizeof(RegisterState));
void copy_kernel_registers_into_ptrace_registers(PtraceRegisters&, const RegisterState&);
@ -494,16 +501,16 @@ inline FlatPtr offset_in_page(const void* address)
return offset_in_page((FlatPtr)address);
}
u32 read_cr0();
u32 read_cr2();
u32 read_cr3();
u32 read_cr4();
FlatPtr read_cr0();
FlatPtr read_cr2();
FlatPtr read_cr3();
FlatPtr read_cr4();
void write_cr0(u32);
void write_cr3(u32);
void write_cr4(u32);
void write_cr0(FlatPtr);
void write_cr3(FlatPtr);
void write_cr4(FlatPtr);
u32 read_dr6();
FlatPtr read_dr6();
static inline bool is_kernel_mode()
{
@ -1071,7 +1078,12 @@ struct TrapFrame {
TrapFrame& operator=(TrapFrame&&) = delete;
};
#define TRAP_FRAME_SIZE (3 * sizeof(FlatPtr))
#if ARCH(I386)
# define TRAP_FRAME_SIZE (3 * 4)
#else
# define TRAP_FRAME_SIZE (3 * 8)
#endif
static_assert(TRAP_FRAME_SIZE == sizeof(TrapFrame));
extern "C" void enter_trap_no_irq(TrapFrame*);

View file

@ -130,7 +130,9 @@ static SlabAllocator<32> s_slab_allocator_32;
static SlabAllocator<64> s_slab_allocator_64;
static SlabAllocator<128> s_slab_allocator_128;
#if ARCH(I386)
static_assert(sizeof(Region) <= s_slab_allocator_128.slab_size());
#endif
template<typename Callback>
void for_each_allocator(Callback callback)

View file

@ -48,7 +48,7 @@ static u8 parse_hex_digit(char nibble)
return 10 + (nibble - 'a');
}
u32 address_for_kernel_symbol(const StringView& name)
FlatPtr address_for_kernel_symbol(const StringView& name)
{
for (size_t i = 0; i < s_symbol_count; ++i) {
if (!strncmp(name.characters_without_null_termination(), s_symbols[i].name, name.length()))
@ -57,7 +57,7 @@ u32 address_for_kernel_symbol(const StringView& name)
return 0;
}
const KernelSymbol* symbolicate_kernel_address(u32 address)
const KernelSymbol* symbolicate_kernel_address(FlatPtr address)
{
if (address < g_lowest_kernel_symbol_address || address > g_highest_kernel_symbol_address)
return nullptr;
@ -147,7 +147,7 @@ NEVER_INLINE static void dump_backtrace_impl(FlatPtr base_pointer, bool use_ksym
FlatPtr* stack_ptr = (FlatPtr*)base_pointer;
while (stack_ptr && safe_memcpy(copied_stack_ptr, stack_ptr, sizeof(copied_stack_ptr), fault_at)) {
FlatPtr retaddr = copied_stack_ptr[1];
dbgln("{:p} (next: {:p})", retaddr, stack_ptr ? (u32*)copied_stack_ptr[0] : 0);
dbgln("{:p} (next: {:p})", retaddr, stack_ptr ? (FlatPtr*)copied_stack_ptr[0] : 0);
stack_ptr = (FlatPtr*)copied_stack_ptr[0];
}
return;

View file

@ -31,12 +31,12 @@
namespace Kernel {
struct KernelSymbol {
u32 address;
FlatPtr address;
const char* name;
};
u32 address_for_kernel_symbol(const StringView& name);
const KernelSymbol* symbolicate_kernel_address(u32 address);
FlatPtr address_for_kernel_symbol(const StringView& name);
const KernelSymbol* symbolicate_kernel_address(FlatPtr);
void load_kernel_symbol_table();
extern bool g_kernel_symbols_available;

View file

@ -260,6 +260,7 @@ Process::~Process()
extern void signal_trampoline_dummy();
void signal_trampoline_dummy()
{
#if ARCH(I386)
// The trampoline preserves the current eax, pushes the signal code and
// then calls the signal handler. We do this because, when interrupting a
// blocking syscall, that syscall may return some special error code in eax;
@ -280,6 +281,9 @@ void signal_trampoline_dummy()
"int 0x82\n" // sigreturn syscall
"asm_signal_trampoline_end:\n"
".att_syntax" ::"i"(Syscall::SC_sigreturn));
#else
// FIXME: Implement trampoline for other architectures.
#endif
}
extern "C" void asm_signal_trampoline(void);

View file

@ -251,8 +251,8 @@ public:
[[noreturn]] void sys$exit(int status);
int sys$sigreturn(RegisterState& registers);
pid_t sys$waitid(Userspace<const Syscall::SC_waitid_params*>);
void* sys$mmap(Userspace<const Syscall::SC_mmap_params*>);
void* sys$mremap(Userspace<const Syscall::SC_mremap_params*>);
FlatPtr sys$mmap(Userspace<const Syscall::SC_mmap_params*>);
FlatPtr sys$mremap(Userspace<const Syscall::SC_mremap_params*>);
int sys$munmap(void*, size_t size);
int sys$set_mmap_name(Userspace<const Syscall::SC_set_mmap_name_params*>);
int sys$mprotect(void*, size_t, int prot);
@ -352,7 +352,7 @@ public:
int sys$recvfd(int sockfd, int options);
long sys$sysconf(int name);
int sys$disown(ProcessID);
void* sys$allocate_tls(size_t);
FlatPtr sys$allocate_tls(size_t);
int sys$prctl(int option, FlatPtr arg1, FlatPtr arg2);
int sys$set_coredump_metadata(Userspace<const Syscall::SC_set_coredump_metadata_params*>);
void sys$abort();

View file

@ -137,13 +137,13 @@ static bool validate_inode_mmap_prot(const Process& process, int prot, const Ino
return true;
}
void* Process::sys$mmap(Userspace<const Syscall::SC_mmap_params*> user_params)
FlatPtr Process::sys$mmap(Userspace<const Syscall::SC_mmap_params*> user_params)
{
REQUIRE_PROMISE(stdio);
Syscall::SC_mmap_params params;
if (!copy_from_user(&params, user_params))
return (void*)-EFAULT;
return -EFAULT;
void* addr = (void*)params.addr;
size_t size = params.size;
@ -162,27 +162,27 @@ void* Process::sys$mmap(Userspace<const Syscall::SC_mmap_params*> user_params)
}
if (alignment & ~PAGE_MASK)
return (void*)-EINVAL;
return -EINVAL;
if (page_round_up_would_wrap(size))
return (void*)-EINVAL;
return -EINVAL;
if (!is_user_range(VirtualAddress(addr), page_round_up(size)))
return (void*)-EFAULT;
return -EFAULT;
String name;
if (params.name.characters) {
if (params.name.length > PATH_MAX)
return (void*)-ENAMETOOLONG;
return -ENAMETOOLONG;
name = copy_string_from_user(params.name);
if (name.is_null())
return (void*)-EFAULT;
return -EFAULT;
}
if (size == 0)
return (void*)-EINVAL;
return -EINVAL;
if ((FlatPtr)addr & ~PAGE_MASK)
return (void*)-EINVAL;
return -EINVAL;
bool map_shared = flags & MAP_SHARED;
bool map_anonymous = flags & MAP_ANONYMOUS;
@ -193,19 +193,19 @@ void* Process::sys$mmap(Userspace<const Syscall::SC_mmap_params*> user_params)
bool map_randomized = flags & MAP_RANDOMIZED;
if (map_shared && map_private)
return (void*)-EINVAL;
return -EINVAL;
if (!map_shared && !map_private)
return (void*)-EINVAL;
return -EINVAL;
if (map_fixed && map_randomized)
return (void*)-EINVAL;
return -EINVAL;
if (!validate_mmap_prot(prot, map_stack, map_anonymous))
return (void*)-EINVAL;
return -EINVAL;
if (map_stack && (!map_private || !map_anonymous))
return (void*)-EINVAL;
return -EINVAL;
Region* region = nullptr;
Optional<Range> range;
@ -223,44 +223,44 @@ void* Process::sys$mmap(Userspace<const Syscall::SC_mmap_params*> user_params)
}
if (!range.has_value())
return (void*)-ENOMEM;
return -ENOMEM;
if (map_anonymous) {
auto strategy = map_noreserve ? AllocationStrategy::None : AllocationStrategy::Reserve;
auto region_or_error = space().allocate_region(range.value(), !name.is_null() ? name : "mmap", prot, strategy);
if (region_or_error.is_error())
return (void*)region_or_error.error().error();
return region_or_error.error().error();
region = region_or_error.value();
} else {
if (offset < 0)
return (void*)-EINVAL;
return -EINVAL;
if (static_cast<size_t>(offset) & ~PAGE_MASK)
return (void*)-EINVAL;
return -EINVAL;
auto description = file_description(fd);
if (!description)
return (void*)-EBADF;
return -EBADF;
if (description->is_directory())
return (void*)-ENODEV;
return -ENODEV;
// Require read access even when read protection is not requested.
if (!description->is_readable())
return (void*)-EACCES;
return -EACCES;
if (map_shared) {
if ((prot & PROT_WRITE) && !description->is_writable())
return (void*)-EACCES;
return -EACCES;
}
if (description->inode()) {
if (!validate_inode_mmap_prot(*this, prot, *description->inode(), map_shared))
return (void*)-EACCES;
return -EACCES;
}
auto region_or_error = description->mmap(*this, range.value(), static_cast<size_t>(offset), prot, map_shared);
if (region_or_error.is_error())
return (void*)region_or_error.error().error();
return region_or_error.error().error();
region = region_or_error.value();
}
if (!region)
return (void*)-ENOMEM;
return -ENOMEM;
region->set_mmap(true);
if (map_shared)
region->set_shared(true);
@ -268,7 +268,7 @@ void* Process::sys$mmap(Userspace<const Syscall::SC_mmap_params*> user_params)
region->set_stack(true);
if (!name.is_null())
region->set_name(name);
return region->vaddr().as_ptr();
return region->vaddr().get();
}
static KResultOr<Range> expand_range_to_page_boundaries(FlatPtr address, size_t size)
@ -494,26 +494,26 @@ int Process::sys$munmap(void* addr, size_t size)
return -EINVAL;
}
void* Process::sys$mremap(Userspace<const Syscall::SC_mremap_params*> user_params)
FlatPtr Process::sys$mremap(Userspace<const Syscall::SC_mremap_params*> user_params)
{
REQUIRE_PROMISE(stdio);
Syscall::SC_mremap_params params {};
if (!copy_from_user(&params, user_params))
return (void*)-EFAULT;
return -EFAULT;
auto range_or_error = expand_range_to_page_boundaries((FlatPtr)params.old_address, params.old_size);
if (range_or_error.is_error())
return (void*)range_or_error.error().error();
return range_or_error.error().error();
auto old_range = range_or_error.value();
auto* old_region = space().find_region_from_range(old_range);
if (!old_region)
return (void*)-EINVAL;
return -EINVAL;
if (!old_region->is_mmap())
return (void*)-EPERM;
return -EPERM;
if (old_region->vmobject().is_shared_inode() && params.flags & MAP_PRIVATE && !(params.flags & (MAP_ANONYMOUS | MAP_NORESERVE))) {
auto range = old_region->range();
@ -529,28 +529,28 @@ void* Process::sys$mremap(Userspace<const Syscall::SC_mremap_params*> user_param
auto new_region_or_error = space().allocate_region_with_vmobject(range, new_vmobject, 0, old_name, old_prot, false);
if (new_region_or_error.is_error())
return (void*)new_region_or_error.error().error();
return new_region_or_error.error().error();
auto& new_region = *new_region_or_error.value();
new_region.set_mmap(true);
return new_region.vaddr().as_ptr();
return new_region.vaddr().get();
}
dbgln("sys$mremap: Unimplemented remap request (flags={})", params.flags);
return (void*)-ENOTIMPL;
return -ENOTIMPL;
}
void* Process::sys$allocate_tls(size_t size)
FlatPtr Process::sys$allocate_tls(size_t size)
{
REQUIRE_PROMISE(stdio);
if (!size)
return (void*)-EINVAL;
return -EINVAL;
if (!m_master_tls_region.is_null())
return (void*)-EEXIST;
return -EEXIST;
if (thread_count() != 1)
return (void*)-EFAULT;
return -EFAULT;
Thread* main_thread = nullptr;
for_each_thread([&main_thread](auto& thread) {
@ -561,11 +561,11 @@ void* Process::sys$allocate_tls(size_t size)
auto range = space().allocate_range({}, size);
if (!range.has_value())
return (void*)-ENOMEM;
return -ENOMEM;
auto region_or_error = space().allocate_region(range.value(), String(), PROT_READ | PROT_WRITE);
if (region_or_error.is_error())
return (void*)region_or_error.error().error();
return region_or_error.error().error();
m_master_tls_region = region_or_error.value()->make_weak_ptr();
m_master_tls_size = size;
@ -573,13 +573,13 @@ void* Process::sys$allocate_tls(size_t size)
auto tsr_result = main_thread->make_thread_specific_region({});
if (tsr_result.is_error())
return (void*)-EFAULT;
return -EFAULT;
auto& tls_descriptor = Processor::current().get_gdt_entry(GDT_SELECTOR_TLS);
tls_descriptor.set_base(main_thread->thread_specific_data());
tls_descriptor.set_limit(main_thread->thread_specific_region_size());
return m_master_tls_region.unsafe_ptr()->vaddr().as_ptr();
return m_master_tls_region.unsafe_ptr()->vaddr().get();
}
int Process::sys$msyscall(void* address)

View file

@ -87,11 +87,11 @@ int Process::sys$module_load(Userspace<const char*> user_path, size_t path_lengt
case R_386_PC32: {
// PC-relative relocation
dbgln("PC-relative relocation: {}", relocation.symbol().name());
u32 symbol_address = address_for_kernel_symbol(relocation.symbol().name());
auto symbol_address = address_for_kernel_symbol(relocation.symbol().name());
if (symbol_address == 0)
missing_symbols = true;
dbgln(" Symbol address: {:p}", symbol_address);
ptrdiff_t relative_offset = (char*)symbol_address - ((char*)&patch_ptr + 4);
ptrdiff_t relative_offset = (FlatPtr)symbol_address - ((FlatPtr)&patch_ptr + 4);
patch_ptr = relative_offset;
break;
}

View file

@ -83,7 +83,7 @@ int Process::sys$create_thread(void* (*entry)(void*), Userspace<const Syscall::S
tss.eip = (FlatPtr)entry;
tss.eflags = 0x0202;
tss.cr3 = space().page_directory().cr3();
tss.esp = (u32)user_stack_address;
tss.esp = (FlatPtr)user_stack_address;
auto tsr_result = thread->make_thread_specific_region({});
if (tsr_result.is_error())

View file

@ -666,10 +666,10 @@ bool Thread::has_signal_handler(u8 signal) const
return !action.handler_or_sigaction.is_null();
}
static bool push_value_on_user_stack(u32* stack, u32 data)
static bool push_value_on_user_stack(FlatPtr* stack, FlatPtr data)
{
*stack -= 4;
return copy_to_user((u32*)*stack, &data);
*stack -= sizeof(FlatPtr);
return copy_to_user((FlatPtr*)*stack, &data);
}
void Thread::resume_from_stopped()
@ -792,19 +792,24 @@ DispatchSignalResult Thread::dispatch_signal(u8 signal)
m_have_any_unmasked_pending_signals.store(m_pending_signals & ~m_signal_mask, AK::memory_order_release);
auto setup_stack = [&](RegisterState& state) {
u32* stack = &state.userspace_esp;
u32 old_esp = *stack;
u32 ret_eip = state.eip;
u32 ret_eflags = state.eflags;
#if ARCH(I386)
FlatPtr* stack = &state.userspace_esp;
#elif ARCH(X86_64)
FlatPtr* stack = &state.userspace_esp;
#endif
FlatPtr old_esp = *stack;
FlatPtr ret_eip = state.eip;
FlatPtr ret_eflags = state.eflags;
#if SIGNAL_DEBUG
klog() << "signal: setting up user stack to return to eip: " << String::format("%p", (void*)ret_eip) << " esp: " << String::format("%p", (void*)old_esp);
#endif
#if ARCH(I386)
// Align the stack to 16 bytes.
// Note that we push 56 bytes (4 * 14) on to the stack,
// so we need to account for this here.
u32 stack_alignment = (*stack - 56) % 16;
FlatPtr stack_alignment = (*stack - 56) % 16;
*stack -= stack_alignment;
push_value_on_user_stack(stack, ret_eflags);
@ -819,6 +824,10 @@ DispatchSignalResult Thread::dispatch_signal(u8 signal)
push_value_on_user_stack(stack, state.esi);
push_value_on_user_stack(stack, state.edi);
#elif ARCH(X86_64)
// FIXME
#endif
// PUSH old_signal_mask
push_value_on_user_stack(stack, old_signal_mask);
@ -941,7 +950,7 @@ void Thread::set_state(State new_state, u8 stop_signal)
}
struct RecognizedSymbol {
u32 address;
FlatPtr address;
const KernelSymbol* symbol { nullptr };
};