bhyve: Move most early initialization into an MD routine

Prior to initializing PCI devices, main() calls a number of
initialization routines, many of which are amd64-specific.  Move this
list of calls to bhyverun_machdep.c.  Similarly, add an MD function to
handle late initialization.

No functional change intended.

Reviewed by:	corvink, jhb
MFC after:	1 week
Sponsored by:	Innovate UK
Differential Revision:	https://reviews.freebsd.org/D40989

usr.sbin/bhyve/amd64/bhyverun_machdep.c

@@ -33,9 +33,21 @@
 
 #include <vmmapi.h>
 
+#include "acpi.h"
+#include "atkbdc.h"
 #include "bhyverun.h"
 #include "config.h"
+#include "e820.h"
+#include "fwctl.h"
+#include "ioapic.h"
+#include "inout.h"
+#include "kernemu_dev.h"
+#include "mptbl.h"
+#include "pci_irq.h"
 #include "pci_lpc.h"
+#include "rtc.h"
+#include "smbiostbl.h"
+#include "xmsr.h"
 
 void
 bhyve_init_config(void)
@@ -123,3 +135,53 @@ bhyve_start_vcpu(struct vcpu *vcpu, bool bsp)
 
 	fbsdrun_addcpu(vcpu_id(vcpu));
 }
+
+int
+bhyve_init_platform(struct vmctx *ctx, struct vcpu *bsp __unused)
+{
+	int error;
+
+	error = init_msr();
+	if (error != 0)
+		return (error);
+	init_inout();
+	kernemu_dev_init();
+	atkbdc_init(ctx);
+	pci_irq_init(ctx);
+	ioapic_init(ctx);
+	rtc_init(ctx);
+	sci_init(ctx);
+	error = e820_init(ctx);
+	if (error != 0)
+		return (error);
+
+	return (0);
+}
+
+int
+bhyve_init_platform_late(struct vmctx *ctx, struct vcpu *bsp __unused)
+{
+	int error;
+
+	if (get_config_bool_default("x86.mptable", true)) {
+		error = mptable_build(ctx, guest_ncpus);
+		if (error != 0)
+			return (error);
+	}
+	error = smbios_build(ctx);
+	if (error != 0)
+		return (error);
+	error = e820_finalize();
+	if (error != 0)
+		return (error);
+
+	if (lpc_bootrom() && strcmp(lpc_fwcfg(), "bhyve") == 0)
+		fwctl_init();
+
+	if (get_config_bool("acpi_tables")) {
+		error = acpi_build(ctx, guest_ncpus);
+		assert(error == 0);
+	}
+
+	return (0);
+}

usr.sbin/bhyve/amd64/xmsr.c

@@ -33,12 +33,13 @@
 #include <machine/vmm.h>
 #include <machine/specialreg.h>
 
-#include <vmmapi.h>
-
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#include <vmmapi.h>
+
 #include "debug.h"
 #include "xmsr.h"
 
@@ -236,7 +237,7 @@ init_msr(void)
 		cpu_vendor_intel = 1;
 	} else {
 		EPRINTLN("Unknown cpu vendor \"%s\"", cpu_vendor);
-		error = -1;
+		error = ENOENT;
 	}
 	return (error);
 }

usr.sbin/bhyve/bhyverun.c

@@ -71,52 +71,27 @@
 
 #include <vmmapi.h>
 
-#include "bhyverun.h"
 #include "acpi.h"
-#ifdef __amd64__
-#include "amd64/atkbdc.h"
-#endif
+#include "bhyverun.h"
 #include "bootrom.h"
 #include "config.h"
-#ifdef __amd64__
-#include "amd64/inout.h"
-#endif
 #include "debug.h"
-#ifdef __amd64__
-#include "amd64/e820.h"
-#include "amd64/fwctl.h"
-#endif
 #ifdef BHYVE_GDB
 #include "gdb.h"
 #endif
-#ifdef __amd64__
-#include "amd64/ioapic.h"
-#include "amd64/kernemu_dev.h"
-#endif
 #include "mem.h"
 #include "mevent.h"
-#ifdef __amd64__
-#include "amd64/mptbl.h"
-#endif
 #include "pci_emul.h"
 #ifdef __amd64__
-#include "amd64/pci_irq.h"
 #include "amd64/pci_lpc.h"
 #endif
 #include "qemu_fwcfg.h"
-#include "smbiostbl.h"
 #ifdef BHYVE_SNAPSHOT
 #include "snapshot.h"
 #endif
 #include "tpm_device.h"
-#ifdef __amd64__
-#include "amd64/rtc.h"
-#endif
 #include "vmgenc.h"
 #include "vmexit.h"
-#ifdef __amd64__
-#include "amd64/xmsr.h"
-#endif
 
 #define MB		(1024UL * 1024)
 #define GB		(1024UL * MB)
@@ -970,30 +945,10 @@ main(int argc, char *argv[])
 		exit(4);
 	}
 
-#ifdef __amd64__
-	error = init_msr();
-	if (error) {
-		fprintf(stderr, "init_msr error %d", error);
-		exit(4);
-	}
-#endif
-
 	init_mem(guest_ncpus);
-#ifdef __amd64__
-	init_inout();
-	kernemu_dev_init();
-#endif
 	init_bootrom(ctx);
-#ifdef __amd64__
-	atkbdc_init(ctx);
-	pci_irq_init(ctx);
-	ioapic_init(ctx);
-#endif
-
-#ifdef __amd64__
-	rtc_init(ctx);
-	sci_init(ctx);
-#endif
+	if (bhyve_init_platform(ctx, bsp) != 0)
+		exit(4);
 
 	if (qemu_fwcfg_init(ctx) != 0) {
 		fprintf(stderr, "qemu fwcfg initialization error");
@@ -1006,13 +961,6 @@ main(int argc, char *argv[])
 		exit(4);
 	}
 
-#ifdef __amd64__
-	if (e820_init(ctx) != 0) {
-		fprintf(stderr, "Unable to setup E820");
-		exit(4);
-	}
-#endif
-
 	/*
 	 * Exit if a device emulation finds an error in its initialization
 	 */
@@ -1076,37 +1024,9 @@ main(int argc, char *argv[])
 	}
 #endif
 
-#ifdef __amd64__
-	if (get_config_bool_default("x86.mptable", true)) {
-		error = mptable_build(ctx, guest_ncpus);
-		if (error) {
-			perror("error to build the guest tables");
-			exit(4);
-		}
-	}
-#endif
-
-	error = smbios_build(ctx);
-	if (error != 0)
+	if (bhyve_init_platform_late(ctx, bsp) != 0)
 		exit(4);
 
-	if (get_config_bool("acpi_tables")) {
-		error = acpi_build(ctx, guest_ncpus);
-		assert(error == 0);
-	}
-
-#ifdef __amd64__
-	error = e820_finalize();
-	if (error != 0)
-		exit(4);
-#endif
-
-#ifdef __amd64__
-	if (lpc_bootrom() && strcmp(lpc_fwcfg(), "bhyve") == 0) {
-		fwctl_init();
-	}
-#endif
-
 	/*
 	 * Change the proc title to include the VM name.
 	 */

usr.sbin/bhyve/bhyverun.h

@@ -60,5 +60,7 @@ typedef int (*vmexit_handler_t)(struct vmctx *, struct vcpu *, struct vm_run *);
 void bhyve_init_config(void);
 void bhyve_init_vcpu(struct vcpu *vcpu);
 void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp);
+int bhyve_init_platform(struct vmctx *ctx, struct vcpu *bsp);
+int bhyve_init_platform_late(struct vmctx *ctx, struct vcpu *bsp);
 
 #endif