Partially revert VNET change and expand VNET structure.

Revert parts of r353274 replacing vnet_state with a shutdown flag.

Not having the state flag for the current SI_SUB_* makes it harder to debug
kernel or module panics related to VNET bringup or teardown.
Not having the state also does not allow us to check for other dependency
levels between components, e.g. for moving interfaces.

Expand the VNET structure with the new boolean flag indicating that we are
doing a shutdown of a given vnet and update the vnet magic cookie for the
change.

Update libkvm to compile with a bool in the kernel struct.

Bump __FreeBSD_version for (external) module builds to more easily detect
the change.

Reviewed by:	hselasky
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D23097

UPDATING

@@ -26,6 +26,11 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 13.x IS SLOW:
 	disable the most expensive debugging functionality run
 	"ln -s 'abort:false,junk:false' /etc/malloc.conf".)
 
+20200217:
+	The size of struct vnet and the magic cookie have changed.
+	Users need to recompile libkvm and all modules using VIMAGE
+	together with their new kernel.
+
 20200212:
 	Defining the long deprecated NO_CTF, NO_DEBUG_FILES, NO_INSTALLLIB,
 	NO_MAN, NO_PROFILE, and NO_WARNS variables is now an error.  Update

lib/libkvm/kvm.c

@@ -49,6 +49,7 @@ __SCCSID("@(#)kvm.c	8.2 (Berkeley) 2/13/94");
 #include <sys/sysctl.h>
 #include <sys/mman.h>
 
+#include <stdbool.h>
 #include <net/vnet.h>
 
 #include <fcntl.h>

lib/libkvm/kvm_private.c

@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 #include <sys/mman.h>
 
+#include <stdbool.h>
 #include <net/vnet.h>
 
 #include <assert.h>

lib/libkvm/kvm_vnet.c

@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/types.h>
 
+#include <stdbool.h>
 #include <net/vnet.h>
 
 #include <kvm.h>

sys/net/if.c

@@ -322,6 +322,11 @@ SX_SYSINIT_FLAGS(ifnet_sx, &ifnet_sxlock, "ifnet_sx", SX_RECURSE);
  */
 #define	IFNET_HOLD	(void *)(uintptr_t)(-1)
 
+#ifdef VIMAGE
+#define	VNET_IS_SHUTTING_DOWN(_vnet)					\
+    ((_vnet)->vnet_shutdown && (_vnet)->vnet_state < SI_SUB_VNET_DONE)
+#endif
+
 static	if_com_alloc_t *if_com_alloc[256];
 static	if_com_free_t *if_com_free[256];
 
@@ -1080,7 +1085,7 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp)
 #ifdef VIMAGE
 	bool shutdown;
 
-	shutdown = ifp->if_vnet->vnet_shutdown;
+	shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet);
 #endif
 	IFNET_WLOCK();
 	CK_STAILQ_FOREACH(iter, &V_ifnet, if_link)
@@ -1339,6 +1344,7 @@ if_vmove_loan(struct thread *td, struct ifnet *ifp, char *ifname, int jid)
 	struct prison *pr;
 	struct ifnet *difp;
 	int error;
+	bool shutdown;
 
 	/* Try to find the prison within our visibility. */
 	sx_slock(&allprison_lock);
@@ -1366,7 +1372,8 @@ if_vmove_loan(struct thread *td, struct ifnet *ifp, char *ifname, int jid)
 	}
 
 	/* Make sure the VNET is stable. */
-	if (ifp->if_vnet->vnet_shutdown) {
+	shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet);
+	if (shutdown) {
 		CURVNET_RESTORE();
 		prison_free(pr);
 		return (EBUSY);
@@ -1391,6 +1398,7 @@ if_vmove_reclaim(struct thread *td, char *ifname, int jid)
 	struct vnet *vnet_dst;
 	struct ifnet *ifp;
 	int error;
+ 	bool shutdown;
 
 	/* Try to find the prison within our visibility. */
 	sx_slock(&allprison_lock);
@@ -1419,7 +1427,8 @@ if_vmove_reclaim(struct thread *td, char *ifname, int jid)
 	}
 
 	/* Make sure the VNET is stable. */
-	if (ifp->if_vnet->vnet_shutdown) {
+	shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet);
+	if (shutdown) {
 		CURVNET_RESTORE();
 		prison_free(pr);
 		return (EBUSY);
@@ -2950,11 +2959,15 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 	struct ifreq *ifr;
 	int error;
 	int oif_flags;
+#ifdef VIMAGE
+	bool shutdown;
+#endif
 
 	CURVNET_SET(so->so_vnet);
 #ifdef VIMAGE
 	/* Make sure the VNET is stable. */
-	if (so->so_vnet->vnet_shutdown) {
+	shutdown = VNET_IS_SHUTTING_DOWN(so->so_vnet);
+	if (shutdown) {
 		CURVNET_RESTORE();
 		return (EBUSY);
 	}

sys/net/vnet.c

@@ -279,6 +279,9 @@ vnet_destroy(struct vnet *vnet)
 	LIST_REMOVE(vnet, vnet_le);
 	VNET_LIST_WUNLOCK();
 
+	/* Signal that VNET is being shutdown. */
+	vnet->vnet_shutdown = true;
+
 	CURVNET_SET_QUIET(vnet);
 	vnet_sysuninit();
 	CURVNET_RESTORE();
@@ -350,15 +353,15 @@ vnet_data_startup(void *dummy __unused)
 }
 SYSINIT(vnet_data, SI_SUB_KLD, SI_ORDER_FIRST, vnet_data_startup, NULL);
 
+/* Dummy VNET_SYSINIT to make sure we always reach the final end state. */
 static void
-vnet_sysuninit_shutdown(void *unused __unused)
+vnet_sysinit_done(void *unused __unused)
 {
 
-	/* Signal that VNET is being shutdown. */
-	curvnet->vnet_shutdown = 1;
+	return;
 }
-VNET_SYSUNINIT(vnet_sysuninit_shutdown, SI_SUB_VNET_DONE, SI_ORDER_FIRST,
-    vnet_sysuninit_shutdown, NULL);
+VNET_SYSINIT(vnet_sysinit_done, SI_SUB_VNET_DONE, SI_ORDER_ANY,
+    vnet_sysinit_done, NULL);
 
 /*
  * When a module is loaded and requires storage for a virtualized global
@@ -572,8 +575,10 @@ vnet_sysinit(void)
 	struct vnet_sysinit *vs;
 
 	VNET_SYSINIT_RLOCK();
-	TAILQ_FOREACH(vs, &vnet_constructors, link)
+	TAILQ_FOREACH(vs, &vnet_constructors, link) {
+		curvnet->vnet_state = vs->subsystem;
 		vs->func(vs->arg);
+	}
 	VNET_SYSINIT_RUNLOCK();
 }
 
@@ -589,8 +594,10 @@ vnet_sysuninit(void)
 
 	VNET_SYSINIT_RLOCK();
 	TAILQ_FOREACH_REVERSE(vs, &vnet_destructors, vnet_sysuninit_head,
-	    link)
+	    link) {
+		curvnet->vnet_state = vs->subsystem;
 		vs->func(vs->arg);
+	}
 	VNET_SYSINIT_RUNLOCK();
 }
 
@@ -704,7 +711,8 @@ db_vnet_print(struct vnet *vnet)
 	db_printf(" vnet_data_mem  = %p\n", vnet->vnet_data_mem);
 	db_printf(" vnet_data_base = %#jx\n",
 	    (uintmax_t)vnet->vnet_data_base);
-	db_printf(" vnet_shutdown  = %#08x\n", vnet->vnet_shutdown);
+	db_printf(" vnet_state     = %#08x\n", vnet->vnet_state);
+	db_printf(" vnet_shutdown  = %#03x\n", vnet->vnet_shutdown);
 	db_printf("\n");
 }
 

sys/net/vnet.h

@@ -72,11 +72,12 @@ struct vnet {
 	u_int			 vnet_magic_n;
 	u_int			 vnet_ifcnt;
 	u_int			 vnet_sockcnt;
-	u_int			 vnet_shutdown; /* Shutdown in progress. */
+	u_int			 vnet_state;	/* SI_SUB_* */
 	void			*vnet_data_mem;
 	uintptr_t		 vnet_data_base;
-};
-#define	VNET_MAGIC_N	0x3e0d8f29
+	bool			 vnet_shutdown;	/* Shutdown in progress. */
+} __aligned(CACHE_LINE_SIZE);
+#define	VNET_MAGIC_N	0x5e4a6f28
 
 /*
  * These two virtual network stack allocator definitions are also required

sys/sys/param.h

@@ -60,7 +60,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1300077	/* Master, propagated to newvers */
+#define __FreeBSD_version 1300078	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,