- Remove 'struct vmmeter' from 'struct pcpu', leaving only global vmmeter

in place. To do per-cpu stats, convert all fields that previously were maintained in the vmmeters that sit in pcpus to counter(9). - Since some vmmeter stats may be touched at very early stages of boot, before we have set up UMA and we can do counter_u64_alloc(), provide an early counter mechanism: o Leave one spare uint64_t in struct pcpu, named pc_early_dummy_counter. o Point counter(9) fields of vmmeter to pcpu[0].pc_early_dummy_counter, so that at early stages of boot, before counters are allocated we already point to a counter that can be safely written to. o For sparc64 that required a whole dummy pcpu[MAXCPU] array. Further related changes: - Don't include vmmeter.h into pcpu.h. - vm.stats.vm.v_swappgsout and vm.stats.vm.v_swappgsin changed to 64-bit, to match kernel representation. - struct vmmeter hidden under _KERNEL, and only vmstat(1) is an exclusion. This is based on benno@'s 4-year old patch: https://lists.freebsd.org/pipermail/freebsd-arch/2013-July/014471.html Reviewed by: kib, gallatin, marius, lidl Differential Revision: https://reviews.freebsd.org/D10156
svn path=/head/; revision=317061
2024-10-06 16:40:47 +00:00 · 2017-04-17 17:34:47 +00:00 · 2017-04-17 17:34:47 +00:00 · 83c9dea1ba · 2020-12-20 02:59:44 +00:00
parent 21d5d37ba4
commit 83c9dea1ba
56 changed files with 375 additions and 322 deletions
--- a/libexec/rpc.rstatd/rstat_proc.c
+++ b/libexec/rpc.rstatd/rstat_proc.c
@ -170,6 +170,7 @@ updatestat(void)
 	struct timeval tm, btm;
 	int mib[6];
 	size_t len;
+	uint64_t val;
 	int ifcount;

 #ifdef DEBUG
@ -229,11 +230,12 @@ updatestat(void)
 #endif

 #define	FETCH_CNT(stat, cnt) do {					\
-	len = sizeof((stat));						\
-	if (sysctlbyname("vm.stats." #cnt , &(stat), &len, 0, 0) < 0) { \
-		syslog(LOG_ERR, "sysctl(vm.stats." #cnt "): %m"); \
+	len = sizeof(uint64_t);						\
+	if (sysctlbyname("vm.stats." #cnt , &val, &len, NULL, 0) < 0) {	\
+		syslog(LOG_ERR, "sysctl(vm.stats." #cnt "): %m");	\
 		exit(1);						\
 	}								\
+	stat = val;							\
 } while (0)

 	FETCH_CNT(stats_all.s1.v_pgpgin, vm.v_vnodepgsin);
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@ -176,7 +176,7 @@ trap(struct trapframe *frame)
 	register_t addr = 0;
 	ksiginfo_t ksi;

-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);
 	type = frame->tf_trapno;

 #ifdef SMP
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@ -348,7 +348,7 @@ atomic_testandclear_long(volatile u_long *p, u_int v)
 * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
 * An assertion in amd64/vm_machdep.c ensures that the value is correct.
 */
-#define	OFFSETOF_MONITORBUF	0x180
+#define	OFFSETOF_MONITORBUF	0x100

 #if defined(SMP)
 static __inline void
--- a/sys/amd64/include/counter.h
+++ b/sys/amd64/include/counter.h
@ -31,7 +31,9 @@

 #include <sys/pcpu.h>

-extern struct pcpu __pcpu[1];
+extern struct pcpu __pcpu[];
+
+#define	EARLY_COUNTER	&__pcpu[0].pc_early_dummy_counter

 #define	counter_enter()	do {} while (0)
 #define	counter_exit()	do {} while (0)
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@ -66,7 +66,7 @@
 	uint32_t pc_pcid_next;						\
 	uint32_t pc_pcid_gen;						\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
-	char	__pad[145]		/* be divisor of PAGE_SIZE	\
+	char	__pad[384]		/* be divisor of PAGE_SIZE	\
 					   after cache alignment */

 #define	PC_DBREG_CMD_NONE	0
--- a/sys/arm/arm/intr.c
+++ b/sys/arm/arm/intr.c
@ -185,7 +185,7 @@ intr_irq_handler(struct trapframe *frame)
 	struct intr_event *event;
 	int i;

-	PCPU_INC(cnt.v_intr);
+	VM_CNT_INC(v_intr);
 	i = -1;
 	while ((i = arm_get_next_irq(i)) != -1) {
 		intrcnt[i]++;
--- a/sys/arm/arm/trap-v4.c
+++ b/sys/arm/arm/trap-v4.c
@ -204,7 +204,7 @@ abort_handler(struct trapframe *tf, int type)
 	td = curthread;
 	p = td->td_proc;

-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);
 	/* Data abort came from user mode? */
 	user = TRAP_USERMODE(tf);

@ -615,7 +615,7 @@ prefetch_abort_handler(struct trapframe *tf)

 	td = curthread;
 	p = td->td_proc;
-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);

 	if (TRAP_USERMODE(tf)) {
 		td->td_frame = tf;
--- a/sys/arm/arm/trap-v6.c
+++ b/sys/arm/arm/trap-v6.c
@ -291,7 +291,7 @@ abort_handler(struct trapframe *tf, int prefetch)
 	void *onfault;
 #endif

-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);
 	td = curthread;

 	fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get();
--- a/sys/arm/arm/undefined.c
+++ b/sys/arm/arm/undefined.c
@ -201,7 +201,7 @@ undefinedinstruction(struct trapframe *frame)
 	if (__predict_true(frame->tf_spsr & PSR_F) == 0)
 		enable_interrupts(PSR_F);

-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);

 	fault_pc = frame->tf_pc;

--- a/sys/arm/include/counter.h
+++ b/sys/arm/include/counter.h
@ -32,6 +32,10 @@
 #include <sys/pcpu.h>
 #include <machine/atomic.h>

+extern struct pcpu __pcpu[];
+
+#define	EARLY_COUNTER	&__pcpu[0].pc_early_dummy_counter
+
 #define	counter_enter()	do {} while (0)
 #define	counter_exit()	do {} while (0)

--- a/sys/arm/include/pcpu.h
+++ b/sys/arm/include/pcpu.h
@ -57,10 +57,10 @@ struct vmspace;
 	void *pc_qmap_pte2p;						\
 	unsigned int pc_dbreg[32];					\
 	int pc_dbreg_cmd;						\
-	char __pad[27]
+	char __pad[155]
 #else
 #define PCPU_MD_FIELDS							\
-	char __pad[157]
+	char __pad[93]
 #endif

 #ifdef _KERNEL
--- a/sys/arm64/include/counter.h
+++ b/sys/arm64/include/counter.h
@ -32,6 +32,10 @@
 #include <sys/pcpu.h>
 #include <machine/atomic.h>

+extern struct pcpu __pcpu[];
+ 
+#define	EARLY_COUNTER	&__pcpu[0].pc_early_dummy_counter
+
 #define	counter_enter()	do {} while (0)
 #define	counter_exit()	do {} while (0)

--- a/sys/arm64/include/pcpu.h
+++ b/sys/arm64/include/pcpu.h
@ -39,7 +39,7 @@
 	u_int	pc_acpi_id;	/* ACPI CPU id */		\
 	u_int	pc_midr;	/* stored MIDR value */	\
 	uint64_t pc_clock;						\
-	char __pad[113]
+	char __pad[241]

 #ifdef _KERNEL

--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
@ -4575,8 +4575,8 @@ zfs_getpages(struct vnode *vp, vm_page_t *m, int count, int *rbehind,
 		return (zfs_vm_pagerret_bad);
 	}

-	PCPU_INC(cnt.v_vnodein);
-	PCPU_ADD(cnt.v_vnodepgsin, count);
+	VM_CNT_INC(v_vnodein);
+	VM_CNT_ADD(v_vnodepgsin, count);

 	lsize = PAGE_SIZE;
 	if (IDX_TO_OFF(mlast->pindex) + lsize > object->un_pager.vnp.vnp_size)
@ -4758,8 +4758,8 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags,
 			vm_page_undirty(ma[i]);
 		}
 		zfs_vmobject_wunlock(object);
-		PCPU_INC(cnt.v_vnodeout);
-		PCPU_ADD(cnt.v_vnodepgsout, ncount);
+		VM_CNT_INC(v_vnodeout);
+		VM_CNT_ADD(v_vnodepgsout, ncount);
 	}
 	dmu_tx_commit(tx);

--- a/sys/compat/linprocfs/linprocfs.c
+++ b/sys/compat/linprocfs/linprocfs.c
@ -425,17 +425,17 @@ linprocfs_dostat(PFS_FILL_ARGS)
 	}
 	sbuf_printf(sb,
 	    "disk 0 0 0 0\n"
-	    "page %u %u\n"
-	    "swap %u %u\n"
-	    "intr %u\n"
-	    "ctxt %u\n"
+	    "page %ju %ju\n"
+	    "swap %ju %ju\n"
+	    "intr %ju\n"
+	    "ctxt %ju\n"
 	    "btime %lld\n",
-	    vm_cnt.v_vnodepgsin,
-	    vm_cnt.v_vnodepgsout,
-	    vm_cnt.v_swappgsin,
-	    vm_cnt.v_swappgsout,
-	    vm_cnt.v_intr,
-	    vm_cnt.v_swtch,
+	    (uintmax_t)VM_CNT_FETCH(v_vnodepgsin),
+	    (uintmax_t)VM_CNT_FETCH(v_vnodepgsout),
+	    (uintmax_t)VM_CNT_FETCH(v_swappgsin),
+	    (uintmax_t)VM_CNT_FETCH(v_swappgsout),
+	    (uintmax_t)VM_CNT_FETCH(v_intr),   
+	    (uintmax_t)VM_CNT_FETCH(v_swtch),
 	    (long long)boottime.tv_sec);
 	return (0);
 }
--- a/sys/fs/fuse/fuse_vnops.c
+++ b/sys/fs/fuse/fuse_vnops.c
@ -1793,8 +1793,8 @@ fuse_vnop_getpages(struct vop_getpages_args *ap)

 	kva = (vm_offset_t)bp->b_data;
 	pmap_qenter(kva, pages, npages);
-	PCPU_INC(cnt.v_vnodein);
-	PCPU_ADD(cnt.v_vnodepgsin, npages);
+	VM_CNT_INC(v_vnodein);
+	VM_CNT_ADD(v_vnodepgsin, npages);

 	count = npages << PAGE_SHIFT;
 	iov.iov_base = (caddr_t)kva;
@ -1927,8 +1927,8 @@ fuse_vnop_putpages(struct vop_putpages_args *ap)

 	kva = (vm_offset_t)bp->b_data;
 	pmap_qenter(kva, pages, npages);
-	PCPU_INC(cnt.v_vnodeout);
-	PCPU_ADD(cnt.v_vnodepgsout, count);
+	VM_CNT_INC(v_vnodeout);
+	VM_CNT_ADD(v_vnodepgsout, count);

 	iov.iov_base = (caddr_t)kva;
 	iov.iov_len = count;
--- a/sys/fs/nfsclient/nfs_clbio.c
+++ b/sys/fs/nfsclient/nfs_clbio.c
@ -184,8 +184,8 @@ ncl_getpages(struct vop_getpages_args *ap)

 	kva = (vm_offset_t) bp->b_data;
 	pmap_qenter(kva, pages, npages);
-	PCPU_INC(cnt.v_vnodein);
-	PCPU_ADD(cnt.v_vnodepgsin, npages);
+	VM_CNT_INC(v_vnodein);
+	VM_CNT_ADD(v_vnodepgsin, npages);

 	count = npages << PAGE_SHIFT;
 	iov.iov_base = (caddr_t) kva;
@ -320,8 +320,8 @@ ncl_putpages(struct vop_putpages_args *ap)
 	}
 	mtx_unlock(&np->n_mtx);

-	PCPU_INC(cnt.v_vnodeout);
-	PCPU_ADD(cnt.v_vnodepgsout, count);
+	VM_CNT_INC(v_vnodeout);
+	VM_CNT_ADD(v_vnodepgsout, count);

 	iov.iov_base = unmapped_buf;
 	iov.iov_len = count;
--- a/sys/fs/smbfs/smbfs_io.c
+++ b/sys/fs/smbfs/smbfs_io.c
@ -470,8 +470,8 @@ smbfs_getpages(ap)

 	kva = (vm_offset_t) bp->b_data;
 	pmap_qenter(kva, pages, npages);
-	PCPU_INC(cnt.v_vnodein);
-	PCPU_ADD(cnt.v_vnodepgsin, npages);
+	VM_CNT_INC(v_vnodein);
+	VM_CNT_ADD(v_vnodepgsin, npages);

 	count = npages << PAGE_SHIFT;
 	iov.iov_base = (caddr_t) kva;
@ -595,8 +595,8 @@ smbfs_putpages(ap)

 	kva = (vm_offset_t) bp->b_data;
 	pmap_qenter(kva, pages, npages);
-	PCPU_INC(cnt.v_vnodeout);
-	PCPU_ADD(cnt.v_vnodepgsout, count);
+	VM_CNT_INC(v_vnodeout);
+	VM_CNT_ADD(v_vnodepgsout, count);

 	iov.iov_base = (caddr_t) kva;
 	iov.iov_len = count;
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@ -192,7 +192,7 @@ trap(struct trapframe *frame)
 	static int lastalert = 0;
 #endif

-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);
 	type = frame->tf_trapno;

 #ifdef SMP
--- a/sys/i386/include/atomic.h
+++ b/sys/i386/include/atomic.h
@ -45,7 +45,7 @@
 * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
 * An assertion in i386/vm_machdep.c ensures that the value is correct.
 */
-#define	__OFFSETOF_MONITORBUF	0x180
+#define	__OFFSETOF_MONITORBUF	0x80

 static __inline void
 __mbk(void)
--- a/sys/i386/include/counter.h
+++ b/sys/i386/include/counter.h
@ -36,6 +36,10 @@
 #include <machine/md_var.h>
 #include <machine/specialreg.h>

+extern struct pcpu __pcpu[];
+
+#define	EARLY_COUNTER	&__pcpu[0].pc_early_dummy_counter
+
 #define	counter_enter()	do {				\
 	if ((cpu_feature & CPUID_CX8) == 0)		\
 		critical_enter();			\
@ -46,8 +50,6 @@
 		critical_exit();			\
 } while (0)

-extern struct pcpu __pcpu[MAXCPU];
-
 static inline void
 counter_64_inc_8b(uint64_t *p, int64_t inc)
 {
--- a/sys/i386/include/pcpu.h
+++ b/sys/i386/include/pcpu.h
@ -68,7 +68,7 @@
 	caddr_t	pc_cmap_addr2;						\
 	vm_offset_t pc_qmap_addr;	/* KVA for temporary mappings */\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
-	char	__pad[189]
+	char	__pad[445]

 #ifdef _KERNEL

--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@ -664,20 +664,20 @@ do_fork(struct thread *td, struct fork_req *fr, struct proc *p2, struct thread *
 	vm_forkproc(td, p2, td2, vm2, fr->fr_flags);

 	if (fr->fr_flags == (RFFDG | RFPROC)) {
-		PCPU_INC(cnt.v_forks);
-		PCPU_ADD(cnt.v_forkpages, p2->p_vmspace->vm_dsize +
+		VM_CNT_INC(v_forks);
+		VM_CNT_ADD(v_forkpages, p2->p_vmspace->vm_dsize +
 		    p2->p_vmspace->vm_ssize);
 	} else if (fr->fr_flags == (RFFDG | RFPROC | RFPPWAIT | RFMEM)) {
-		PCPU_INC(cnt.v_vforks);
-		PCPU_ADD(cnt.v_vforkpages, p2->p_vmspace->vm_dsize +
+		VM_CNT_INC(v_vforks);
+		VM_CNT_ADD(v_vforkpages, p2->p_vmspace->vm_dsize +
 		    p2->p_vmspace->vm_ssize);
 	} else if (p1 == &proc0) {
-		PCPU_INC(cnt.v_kthreads);
-		PCPU_ADD(cnt.v_kthreadpages, p2->p_vmspace->vm_dsize +
+		VM_CNT_INC(v_kthreads);
+		VM_CNT_ADD(v_kthreadpages, p2->p_vmspace->vm_dsize +
 		    p2->p_vmspace->vm_ssize);
 	} else {
-		PCPU_INC(cnt.v_rforks);
-		PCPU_ADD(cnt.v_rforkpages, p2->p_vmspace->vm_dsize +
+		VM_CNT_INC(v_rforks);
+		VM_CNT_ADD(v_rforkpages, p2->p_vmspace->vm_dsize +
 		    p2->p_vmspace->vm_ssize);
 	}

--- a/sys/kern/kern_intr.c
+++ b/sys/kern/kern_intr.c
@ -1156,7 +1156,7 @@ swi_sched(void *cookie, int flags)
 	ih->ih_need = 1;

 	if (!(flags & SWI_DELAY)) {
-		PCPU_INC(cnt.v_soft);
+		VM_CNT_INC(v_soft);
 #ifdef INTR_FILTER
 		error = intr_event_schedule_thread(ie, ie->ie_thread);
 #else
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@ -426,7 +426,7 @@ mi_switch(int flags, struct thread *newtd)
 	td->td_incruntime += runtime;
 	PCPU_SET(switchtime, new_switchtime);
 	td->td_generation++;	/* bump preempt-detect counter */
-	PCPU_INC(cnt.v_swtch);
+	VM_CNT_INC(v_swtch);
 	PCPU_SET(switchticks, ticks);
 	CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)",
 	    td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name);
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@ -546,7 +546,7 @@ thread_exit(void)
 	td->td_incruntime += runtime;
 	PCPU_SET(switchtime, new_switchtime);
 	PCPU_SET(switchticks, ticks);
-	PCPU_INC(cnt.v_swtch);
+	VM_CNT_INC(v_swtch);

 	/* Save our resource usage in our process. */
 	td->td_ru.ru_nvcsw++;
--- a/sys/kern/subr_intr.c
+++ b/sys/kern/subr_intr.c
@ -292,7 +292,7 @@ intr_irq_handler(struct trapframe *tf)

 	KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__));

-	PCPU_INC(cnt.v_intr);
+	VM_CNT_INC(v_intr);
 	critical_enter();
 	td = curthread;
 	oldframe = td->td_intr_frame;
--- a/sys/kern/subr_syscall.c
+++ b/sys/kern/subr_syscall.c
@ -58,7 +58,7 @@ syscallenter(struct thread *td, struct syscall_args *sa)
 	struct proc *p;
 	int error, traced;

-	PCPU_INC(cnt.v_syscall);
+	VM_CNT_INC(v_syscall);
 	p = td->td_proc;

 	td->td_pticks = 0;
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@ -234,7 +234,7 @@ ast(struct trapframe *framep)
 	td->td_flags &= ~(TDF_ASTPENDING | TDF_NEEDSIGCHK | TDF_NEEDSUSPCHK |
 	    TDF_NEEDRESCHED | TDF_ALRMPEND | TDF_PROFPEND | TDF_MACPEND);
 	thread_unlock(td);
-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);

 	if (td->td_cowgen != p->p_cowgen)
 		thread_cow_update(td);
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@ -4769,8 +4769,8 @@ vfs_bio_getpages(struct vnode *vp, vm_page_t *ma, int count,
 	pgsin += pgsin_a;
 	if (rahead != NULL)
 		*rahead = pgsin_a;
-	PCPU_INC(cnt.v_vnodein);
-	PCPU_ADD(cnt.v_vnodepgsin, pgsin);
+	VM_CNT_INC(v_vnodein);
+	VM_CNT_ADD(v_vnodepgsin, pgsin);

 	br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS)
 	    != 0) ? GB_UNMAPPED : 0;
--- a/sys/mips/include/counter.h
+++ b/sys/mips/include/counter.h
@ -34,6 +34,8 @@
 #include <sys/proc.h>
 #endif

+#define	EARLY_COUNTER	&((struct pcpu *)pcpu_space)->pc_early_dummy_counter
+
 #define	counter_enter()	critical_enter()
 #define	counter_exit()	critical_exit()

--- a/sys/mips/include/intr_machdep.h
+++ b/sys/mips/include/intr_machdep.h
@ -71,6 +71,6 @@ mips_intrcnt_inc(mips_intrcnt_t counter)
 {
 	if (counter)
 		atomic_add_long(counter, 1);
-	PCPU_INC(cnt.v_intr);
+	VM_CNT_INC(v_intr);
 }
 #endif /* !_MACHINE_INTR_MACHDEP_H_ */
--- a/sys/mips/include/pcpu.h
+++ b/sys/mips/include/pcpu.h
@ -45,11 +45,11 @@
 #ifdef	__mips_n64
 #define	PCPU_MD_MIPS64_FIELDS						\
 	PCPU_MD_COMMON_FIELDS						\
-	char		__pad[53]
+	char		__pad[245]
 #else
 #define	PCPU_MD_MIPS32_FIELDS						\
 	PCPU_MD_COMMON_FIELDS						\
-	char		__pad[189]
+	char		__pad[125]
 #endif

 #ifdef	__mips_n64
--- a/sys/powerpc/include/counter.h
+++ b/sys/powerpc/include/counter.h
@ -34,6 +34,10 @@
 #include <sys/proc.h>
 #endif

+extern struct pcpu __pcpu[];
+
+#define	EARLY_COUNTER	&__pcpu[0].pc_early_dummy_counter
+
 #ifdef __powerpc64__

 #define	counter_enter()	do {} while (0)
@ -79,8 +83,6 @@ counter_u64_zero_inline(counter_u64_t c)

 #define	counter_u64_add_protected(c, i)	counter_u64_add(c, i)

-extern struct pcpu __pcpu[MAXCPU];
-
 static inline void
 counter_u64_add(counter_u64_t c, int64_t inc)
 {
--- a/sys/powerpc/include/pcpu.h
+++ b/sys/powerpc/include/pcpu.h
@ -57,7 +57,7 @@ struct pvo_entry;
 	vm_offset_t	pc_qmap_addr;					\
 	struct pvo_entry *pc_qmap_pvo;					\
 	struct mtx	pc_qmap_lock;					\
-	/* char		__pad[0] */
+	char		__pad[128]

 #define PCPU_MD_AIM64_FIELDS						\
 	struct slb	pc_slb[64];					\
@ -67,7 +67,7 @@ struct pvo_entry;
 	vm_offset_t	pc_qmap_addr;					\
 	struct pvo_entry *pc_qmap_pvo;					\
 	struct mtx	pc_qmap_lock;					\
-	char		__pad[1121 - sizeof(struct mtx)]
+	char		__pad[1345]

 #ifdef __powerpc64__
 #define PCPU_MD_AIM_FIELDS	PCPU_MD_AIM64_FIELDS
@ -81,9 +81,9 @@ struct pvo_entry;
 #define	BOOKE_TLBSAVE_LEN	(BOOKE_TLB_SAVELEN * BOOKE_TLB_MAXNEST)

 #ifdef __powerpc64__
-#define	BOOKE_PCPU_PAD	773
+#define	BOOKE_PCPU_PAD	901
 #else
-#define	BOOKE_PCPU_PAD	173
+#define	BOOKE_PCPU_PAD	429
 #endif
 #define PCPU_MD_BOOKE_FIELDS						\
 	register_t	pc_booke_critsave[BOOKE_CRITSAVE_LEN];		\
--- a/sys/powerpc/powerpc/trap.c
+++ b/sys/powerpc/powerpc/trap.c
@ -171,7 +171,7 @@ trap(struct trapframe *frame)
 	u_int		ucode;
 	ksiginfo_t	ksi;

-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);

 	td = curthread;
 	p = td->td_proc;
--- a/sys/sparc64/include/counter.h
+++ b/sys/sparc64/include/counter.h
@ -34,6 +34,9 @@
 #include <sys/proc.h>
 #endif

+extern struct pcpu dummy_pcpu[];
+#define	EARLY_COUNTER	&dummy_pcpu[0].pc_early_dummy_counter
+
 #define	counter_enter()	critical_enter()
 #define	counter_exit()	critical_exit()

--- a/sys/sparc64/include/pcpu.h
+++ b/sys/sparc64/include/pcpu.h
@ -62,7 +62,7 @@ struct pmap;
 	u_int	pc_tlb_ctx;						\
 	u_int	pc_tlb_ctx_max;						\
 	u_int	pc_tlb_ctx_min;						\
-	char	__pad[397]
+	char	__pad[653]

 #ifdef _KERNEL

--- a/sys/sparc64/sparc64/exception.S
+++ b/sys/sparc64/sparc64/exception.S
@ -2476,9 +2476,8 @@ ENTRY(tl0_intr)
 	inc	%l1
 	stx	%l1, [%l0]

-	lduw	[PCPU(CNT) + V_INTR], %l0
-	inc	%l0
-	stw	%l0, [PCPU(CNT) + V_INTR]
+	call	counter_intr_inc
+	 nop

 	ba,a	%xcc, tl0_ret
 	 nop
@ -2989,11 +2988,8 @@ ENTRY(tl1_intr)
 	add	%l5, %l4, %l4
 	ldx	[%l4], %l5
 	inc	%l5
-	stx	%l5, [%l4]
-
-	lduw	[PCPU(CNT) + V_INTR], %l4
-	inc	%l4
-	stw	%l4, [PCPU(CNT) + V_INTR]
+	call	counter_intr_inc
+	 stx	%l5, [%l4]

 	ldx	[%sp + SPOFF + CCFSZ + TF_Y], %l4

--- a/sys/sparc64/sparc64/genassym.c
+++ b/sys/sparc64/sparc64/genassym.c
@ -134,7 +134,6 @@ ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
 ASSYM(PC_IRHEAD, offsetof(struct pcpu, pc_irhead));
 ASSYM(PC_IRTAIL, offsetof(struct pcpu, pc_irtail));
 ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree));
-ASSYM(PC_CNT, offsetof(struct pcpu, pc_cnt));
 ASSYM(PC_SIZEOF, sizeof(struct pcpu));

 ASSYM(PC_CACHE, offsetof(struct pcpu, pc_cache));
--- a/sys/sparc64/sparc64/intr_machdep.c
+++ b/sys/sparc64/sparc64/intr_machdep.c
@ -123,6 +123,7 @@ static void intr_stray_level(struct trapframe *);
 static void intr_stray_vector(void *);
 static int intrcnt_setname(const char *, int);
 static void intrcnt_updatename(int, const char *, int);
+void counter_intr_inc(void);

 static void
 intrcnt_updatename(int vec, const char *name, int ispil)
@ -451,6 +452,19 @@ intr_describe(int vec, void *ih, const char *descr)
 	return (error);
 }

+/*
+ * Do VM_CNT_INC(intr), being in the interrupt context already. This is
+ * called from assembly.
+ * To avoid counter_enter() and appropriate assertion, unwrap VM_CNT_INC()
+ * and hardcode the actual increment.
+ */
+void
+counter_intr_inc(void)
+{
+
+	*(uint64_t *)zpcpu_get(vm_cnt.v_intr) += 1;
+}
+
 #ifdef SMP
 /*
 * Support for balancing interrupt sources across CPUs.  For now we just
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@ -126,6 +126,7 @@ long realmem;

 void *dpcpu0;
 char pcpu0[PCPU_PAGES * PAGE_SIZE];
+struct pcpu dummy_pcpu[MAXCPU];
 struct trapframe frame0;

 vm_offset_t kstack0;
--- a/sys/sparc64/sparc64/trap.c
+++ b/sys/sparc64/sparc64/trap.c
@ -267,7 +267,7 @@ trap(struct trapframe *tf)
 	    trap_msg[tf->tf_type & ~T_KERNEL],
 	    (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil));

-	PCPU_INC(cnt.v_trap);
+	VM_CNT_INC(v_trap);

 	if ((tf->tf_tstate & TSTATE_PRIV) == 0) {
 		KASSERT(td != NULL, ("trap: curthread NULL"));
--- a/sys/sys/pcpu.h
+++ b/sys/sys/pcpu.h
@ -43,7 +43,6 @@
 #include <sys/_sx.h>
 #include <sys/queue.h>
 #include <sys/_rmlock.h>
-#include <sys/vmmeter.h>
 #include <sys/resource.h>
 #include <machine/pcpu.h>

@ -158,7 +157,6 @@ struct pcpu {
 	u_int		pc_cpuid;		/* This cpu number */
 	STAILQ_ENTRY(pcpu) pc_allcpu;
 	struct lock_list_entry *pc_spinlocks;
-	struct vmmeter	pc_cnt;			/* VM stats counters */
 	long		pc_cp_time[CPUSTATES];	/* statclock ticks */
 	struct device	*pc_device;
 	void		*pc_netisr;		/* netisr SWI cookie */
@ -166,6 +164,7 @@ struct pcpu {
 	int		pc_domain;		/* Memory domain. */
 	struct rm_queue	pc_rm_queue;		/* rmlock list of trackers */
 	uintptr_t	pc_dynamic;		/* Dynamic per-cpu data area */
+	uint64_t	pc_early_dummy_counter;	/* Startup time counter(9) */

 	/*
 	 * Keep MD fields last, so that CPU-specific variations on a
--- a/sys/sys/vmmeter.h
+++ b/sys/sys/vmmeter.h
@ -39,50 +39,84 @@
 */
 #define	MAXSLP			20

+/* Systemwide totals computed every five seconds. */
+struct vmtotal {
+	int16_t	t_rq;		/* length of the run queue */
+	int16_t	t_dw;		/* jobs in ``disk wait'' (neg priority) */
+	int16_t	t_pw;		/* jobs in page wait */
+	int16_t	t_sl;		/* jobs sleeping in core */
+	int16_t	t_sw;		/* swapped out runnable/short block jobs */
+	int32_t	t_vm;		/* total virtual memory */
+	int32_t	t_avm;		/* active virtual memory */
+	int32_t	t_rm;		/* total real memory in use */
+	int32_t	t_arm;		/* active real memory */
+	int32_t	t_vmshr;	/* shared virtual memory */
+	int32_t	t_avmshr;	/* active shared virtual memory */
+	int32_t	t_rmshr;	/* shared real memory */
+	int32_t	t_armshr;	/* active shared real memory */
+	int32_t	t_free;		/* free memory pages */
+};
+
+#if defined(_KERNEL) || defined(_WANT_VMMETER)
+#include <sys/counter.h>
+
 /*
 * System wide statistics counters.
 * Locking:
 *      a - locked by atomic operations
 *      c - constant after initialization
 *      f - locked by vm_page_queue_free_mtx
- *      p - locked by being in the PCPU and atomicity respect to interrupts
+ *      p - uses counter(9)
 *      q - changes are synchronized by the corresponding vm_pagequeue lock
 */
 struct vmmeter {
 	/*
 	 * General system activity.
 	 */
-	u_int v_swtch;		/* (p) context switches */
-	u_int v_trap;		/* (p) calls to trap */
-	u_int v_syscall;	/* (p) calls to syscall() */
-	u_int v_intr;		/* (p) device interrupts */
-	u_int v_soft;		/* (p) software interrupts */
+	counter_u64_t v_swtch;		/* (p) context switches */
+	counter_u64_t v_trap;		/* (p) calls to trap */
+	counter_u64_t v_syscall;	/* (p) calls to syscall() */
+	counter_u64_t v_intr;		/* (p) device interrupts */
+	counter_u64_t v_soft;		/* (p) software interrupts */
 	/*
 	 * Virtual memory activity.
 	 */
-	u_int v_vm_faults;	/* (p) address memory faults */
-	u_int v_io_faults;	/* (p) page faults requiring I/O */
-	u_int v_cow_faults;	/* (p) copy-on-writes faults */
-	u_int v_cow_optim;	/* (p) optimized copy-on-writes faults */
-	u_int v_zfod;		/* (p) pages zero filled on demand */
-	u_int v_ozfod;		/* (p) optimized zero fill pages */
-	u_int v_swapin;		/* (p) swap pager pageins */
-	u_int v_swapout;	/* (p) swap pager pageouts */
-	u_int v_swappgsin;	/* (p) swap pager pages paged in */
-	u_int v_swappgsout;	/* (p) swap pager pages paged out */
-	u_int v_vnodein;	/* (p) vnode pager pageins */
-	u_int v_vnodeout;	/* (p) vnode pager pageouts */
-	u_int v_vnodepgsin;	/* (p) vnode_pager pages paged in */
-	u_int v_vnodepgsout;	/* (p) vnode pager pages paged out */
-	u_int v_intrans;	/* (p) intransit blocking page faults */
-	u_int v_reactivated;	/* (p) pages reactivated by the pagedaemon */
-	u_int v_pdwakeups;	/* (p) times daemon has awaken from sleep */
-	u_int v_pdpages;	/* (p) pages analyzed by daemon */
-	u_int v_pdshortfalls;	/* (p) page reclamation shortfalls */
+	counter_u64_t v_vm_faults;	/* (p) address memory faults */
+	counter_u64_t v_io_faults;	/* (p) page faults requiring I/O */
+	counter_u64_t v_cow_faults;	/* (p) copy-on-writes faults */
+	counter_u64_t v_cow_optim;	/* (p) optimized COW faults */
+	counter_u64_t v_zfod;		/* (p) pages zero filled on demand */
+	counter_u64_t v_ozfod;		/* (p) optimized zero fill pages */
+	counter_u64_t v_swapin;		/* (p) swap pager pageins */
+	counter_u64_t v_swapout;	/* (p) swap pager pageouts */
+	counter_u64_t v_swappgsin;	/* (p) swap pager pages paged in */
+	counter_u64_t v_swappgsout;	/* (p) swap pager pages paged out */
+	counter_u64_t v_vnodein;	/* (p) vnode pager pageins */
+	counter_u64_t v_vnodeout;	/* (p) vnode pager pageouts */
+	counter_u64_t v_vnodepgsin;	/* (p) vnode_pager pages paged in */
+	counter_u64_t v_vnodepgsout;	/* (p) vnode pager pages paged out */
+	counter_u64_t v_intrans;	/* (p) intransit blocking page faults */
+	counter_u64_t v_reactivated;	/* (p) reactivated by the pagedaemon */
+	counter_u64_t v_pdwakeups;	/* (p) times daemon has awaken */
+	counter_u64_t v_pdpages;	/* (p) pages analyzed by daemon */
+	counter_u64_t v_pdshortfalls;	/* (p) page reclamation shortfalls */

-	u_int v_dfree;		/* (p) pages freed by daemon */
-	u_int v_pfree;		/* (p) pages freed by exiting processes */
-	u_int v_tfree;		/* (p) total pages freed */
+	counter_u64_t v_dfree;		/* (p) pages freed by daemon */
+	counter_u64_t v_pfree;		/* (p) pages freed by processes */
+	counter_u64_t v_tfree;		/* (p) total pages freed */
+	/*
+	 * Fork/vfork/rfork activity.
+	 */
+	counter_u64_t v_forks;		/* (p) fork() calls */
+	counter_u64_t v_vforks;		/* (p) vfork() calls */
+	counter_u64_t v_rforks;		/* (p) rfork() calls */
+	counter_u64_t v_kthreads;	/* (p) fork() calls by kernel */
+	counter_u64_t v_forkpages;	/* (p) pages affected by fork() */
+	counter_u64_t v_vforkpages;	/* (p) pages affected by vfork() */
+	counter_u64_t v_rforkpages;	/* (p) pages affected by rfork() */
+	counter_u64_t v_kthreadpages;	/* (p) ... and by kernel fork() */
+#define	VM_METER_NCOUNTERS	\
+	(offsetof(struct vmmeter, v_page_size) / sizeof(counter_u64_t))
 	/*
 	 * Distribution of page usages.
 	 */
@ -100,24 +134,18 @@ struct vmmeter {
 	u_int v_pageout_free_min;   /* (c) min pages reserved for kernel */
 	u_int v_interrupt_free_min; /* (c) reserved pages for int code */
 	u_int v_free_severe;	/* (c) severe page depletion point */
-	/*
-	 * Fork/vfork/rfork activity.
-	 */
-	u_int v_forks;		/* (p) fork() calls */
-	u_int v_vforks;		/* (p) vfork() calls */
-	u_int v_rforks;		/* (p) rfork() calls */
-	u_int v_kthreads;	/* (p) fork() calls by kernel */
-	u_int v_forkpages;	/* (p) VM pages affected by fork() */
-	u_int v_vforkpages;	/* (p) VM pages affected by vfork() */
-	u_int v_rforkpages;	/* (p) VM pages affected by rfork() */
-	u_int v_kthreadpages;	/* (p) VM pages affected by fork() by kernel */
 };
+#endif /* _KERNEL || _WANT_VMMETER */
+
 #ifdef _KERNEL

 extern struct vmmeter vm_cnt;
-
 extern u_int vm_pageout_wakeup_thresh;

+#define	VM_CNT_ADD(var, x)	counter_u64_add(vm_cnt.var, x)
+#define	VM_CNT_INC(var)		VM_CNT_ADD(var, 1)
+#define	VM_CNT_FETCH(var)	counter_u64_fetch(vm_cnt.var)
+
 /*
 * Return TRUE if we are under our severe low-free-pages threshold
 *
@ -189,33 +217,5 @@ vm_laundry_target(void)

 	return (vm_paging_target());
 }
-
-/*
- * Obtain the value of a per-CPU counter.
- */
-#define	VM_METER_PCPU_CNT(member)					\
-	vm_meter_cnt(__offsetof(struct vmmeter, member))
-
-u_int	vm_meter_cnt(size_t);
-
-#endif
-
-/* systemwide totals computed every five seconds */
-struct vmtotal {
-	int16_t	t_rq;		/* length of the run queue */
-	int16_t	t_dw;		/* jobs in ``disk wait'' (neg priority) */
-	int16_t	t_pw;		/* jobs in page wait */
-	int16_t	t_sl;		/* jobs sleeping in core */
-	int16_t	t_sw;		/* swapped out runnable/short block jobs */
-	int32_t	t_vm;		/* total virtual memory */
-	int32_t	t_avm;		/* active virtual memory */
-	int32_t	t_rm;		/* total real memory in use */
-	int32_t	t_arm;		/* active real memory */
-	int32_t	t_vmshr;	/* shared virtual memory */
-	int32_t	t_avmshr;	/* active shared virtual memory */
-	int32_t	t_rmshr;	/* shared real memory */
-	int32_t	t_armshr;	/* active shared real memory */
-	int32_t	t_free;		/* free memory pages */
-};
-
-#endif
+#endif	/* _KERNEL */
+#endif	/* _SYS_VMMETER_H_ */
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@ -1180,8 +1180,8 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
 	bp->b_pgbefore = rbehind != NULL ? *rbehind : 0;
 	bp->b_pgafter = rahead != NULL ? *rahead : 0;

-	PCPU_INC(cnt.v_swapin);
-	PCPU_ADD(cnt.v_swappgsin, count);
+	VM_CNT_INC(v_swapin);
+	VM_CNT_ADD(v_swappgsin, count);

 	/*
 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
@ -1205,7 +1205,7 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
 	VM_OBJECT_WLOCK(object);
 	while ((m[0]->oflags & VPO_SWAPINPROG) != 0) {
 		m[0]->oflags |= VPO_SWAPSLEEP;
-		PCPU_INC(cnt.v_intrans);
+		VM_CNT_INC(v_intrans);
 		if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
 		    "swread", hz * 20)) {
 			printf(
@ -1393,8 +1393,8 @@ swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 		bp->b_dirtyoff = 0;
 		bp->b_dirtyend = bp->b_bcount;

-		PCPU_INC(cnt.v_swapout);
-		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
+		VM_CNT_INC(v_swapout);
+		VM_CNT_ADD(v_swappgsout, bp->b_npages);

 		/*
 		 * We unconditionally set rtvals[] to VM_PAGER_PEND so that we
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@ -497,7 +497,7 @@ vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
 	boolean_t wired;	/* Passed by reference. */
 	bool dead, growstack, hardfault, is_first_object_locked;

-	PCPU_INC(cnt.v_vm_faults);
+	VM_CNT_INC(v_vm_faults);
 	fs.vp = NULL;
 	faultcount = 0;
 	nera = -1;
@ -673,7 +673,7 @@ RetryFault:;
 				}
 				vm_object_pip_wakeup(fs.object);
 				VM_OBJECT_WUNLOCK(fs.object);
-				PCPU_INC(cnt.v_intrans);
+				VM_CNT_INC(v_intrans);
 				vm_object_deallocate(fs.first_object);
 				goto RetryFault;
 			}
@ -999,9 +999,9 @@ RetryFault:;
 			if ((fs.m->flags & PG_ZERO) == 0) {
 				pmap_zero_page(fs.m);
 			} else {
-				PCPU_INC(cnt.v_ozfod);
+				VM_CNT_INC(v_ozfod);
 			}
-			PCPU_INC(cnt.v_zfod);
+			VM_CNT_INC(v_zfod);
 			fs.m->valid = VM_PAGE_BITS_ALL;
 			/* Don't try to prefault neighboring pages. */
 			faultcount = 1;
@ -1095,7 +1095,7 @@ RetryFault:;
 				vm_page_xbusy(fs.m);
 				fs.first_m = fs.m;
 				fs.m = NULL;
-				PCPU_INC(cnt.v_cow_optim);
+				VM_CNT_INC(v_cow_optim);
 			} else {
 				/*
 				 * Oh, well, lets copy it.
@ -1131,7 +1131,7 @@ RetryFault:;
 			fs.m = fs.first_m;
 			if (!is_first_object_locked)
 				VM_OBJECT_WLOCK(fs.object);
-			PCPU_INC(cnt.v_cow_faults);
+			VM_CNT_INC(v_cow_faults);
 			curthread->td_cow++;
 		} else {
 			prot &= ~VM_PROT_WRITE;
@ -1246,7 +1246,7 @@ RetryFault:;
 	 */
 	unlock_and_deallocate(&fs);
 	if (hardfault) {
-		PCPU_INC(cnt.v_io_faults);
+		VM_CNT_INC(v_io_faults);
 		curthread->td_ru.ru_majflt++;
 #ifdef RACCT
 		if (racct_enable && fs.object->type == OBJT_VNODE) {
--- a/sys/vm/vm_meter.c
+++ b/sys/vm/vm_meter.c
@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
 #include <sys/resource.h>
@ -55,7 +56,52 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_object.h>
 #include <sys/sysctl.h>

-struct vmmeter vm_cnt;
+struct vmmeter vm_cnt = {
+	.v_swtch = EARLY_COUNTER,
+	.v_trap = EARLY_COUNTER,
+	.v_syscall = EARLY_COUNTER,
+	.v_intr = EARLY_COUNTER,
+	.v_soft = EARLY_COUNTER,
+	.v_vm_faults = EARLY_COUNTER,
+	.v_io_faults = EARLY_COUNTER,
+	.v_cow_faults = EARLY_COUNTER,
+	.v_cow_optim = EARLY_COUNTER,
+	.v_zfod = EARLY_COUNTER,
+	.v_ozfod = EARLY_COUNTER,
+	.v_swapin = EARLY_COUNTER,
+	.v_swapout = EARLY_COUNTER,
+	.v_swappgsin = EARLY_COUNTER,
+	.v_swappgsout = EARLY_COUNTER,
+	.v_vnodein = EARLY_COUNTER,
+	.v_vnodeout = EARLY_COUNTER,
+	.v_vnodepgsin = EARLY_COUNTER,
+	.v_vnodepgsout = EARLY_COUNTER,
+	.v_intrans = EARLY_COUNTER,
+	.v_reactivated = EARLY_COUNTER,
+	.v_pdwakeups = EARLY_COUNTER,
+	.v_pdpages = EARLY_COUNTER,
+	.v_pdshortfalls = EARLY_COUNTER,
+	.v_dfree = EARLY_COUNTER,
+	.v_pfree = EARLY_COUNTER,
+	.v_tfree = EARLY_COUNTER,
+	.v_forks = EARLY_COUNTER,
+	.v_vforks = EARLY_COUNTER,
+	.v_rforks = EARLY_COUNTER,
+	.v_kthreads = EARLY_COUNTER,
+	.v_forkpages = EARLY_COUNTER,
+	.v_vforkpages = EARLY_COUNTER,
+	.v_rforkpages = EARLY_COUNTER,
+	.v_kthreadpages = EARLY_COUNTER,
+};
+
+static void
+vmcounter_startup(void)
+{
+	counter_u64_t *cnt = (counter_u64_t *)&vm_cnt;
+
+	COUNTER_ARRAY_ALLOC(cnt, VM_METER_NCOUNTERS, M_WAITOK);
+}
+SYSINIT(counter, SI_SUB_CPU, SI_ORDER_FOURTH + 1, vmcounter_startup, NULL);

 SYSCTL_UINT(_vm, VM_V_FREE_MIN, v_free_min,
 	CTLFLAG_RW, &vm_cnt.v_free_min, 0, "Minimum low-free-pages threshold");
@ -210,40 +256,6 @@ vmtotal(SYSCTL_HANDLER_ARGS)
 	return (sysctl_handle_opaque(oidp, &total, sizeof(total), req));
 }

-/*
- * vm_meter_cnt() -	accumulate statistics from all cpus and the global cnt
- *			structure.
- *
- *	The vmmeter structure is now per-cpu as well as global.  Those
- *	statistics which can be kept on a per-cpu basis (to avoid cache
- *	stalls between cpus) can be moved to the per-cpu vmmeter.  Remaining
- *	statistics, such as v_free_reserved, are left in the global
- *	structure.
- */
-u_int
-vm_meter_cnt(size_t offset)
-{
-	struct pcpu *pcpu;
-	u_int count;
-	int i;
-
-	count = *(u_int *)((char *)&vm_cnt + offset);
-	CPU_FOREACH(i) {
-		pcpu = pcpu_find(i);
-		count += *(u_int *)((char *)&pcpu->pc_cnt + offset);
-	}
-	return (count);
-}
-
-static int
-cnt_sysctl(SYSCTL_HANDLER_ARGS)
-{
-	u_int count;
-
-	count = vm_meter_cnt((char *)arg1 - (char *)&vm_cnt);
-	return (SYSCTL_OUT(req, &count, sizeof(count)));
-}
-
 SYSCTL_PROC(_vm, VM_TOTAL, vmtotal, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE,
    0, sizeof(struct vmtotal), vmtotal, "S,vmtotal", 
    "System virtual memory statistics");
@ -255,9 +267,7 @@ static SYSCTL_NODE(_vm_stats, OID_AUTO, vm, CTLFLAG_RW, 0,
 SYSCTL_NODE(_vm_stats, OID_AUTO, misc, CTLFLAG_RW, 0, "VM meter misc stats");

 #define	VM_STATS(parent, var, descr) \
-	SYSCTL_PROC(parent, OID_AUTO, var, \
-	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0,	\
-	    cnt_sysctl, "IU", descr)
+    SYSCTL_COUNTER_U64(parent, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, descr)
 #define	VM_STATS_VM(var, descr)		VM_STATS(_vm_stats_vm, var, descr)
 #define	VM_STATS_SYS(var, descr)	VM_STATS(_vm_stats_sys, var, descr)

@ -288,19 +298,6 @@ VM_STATS_VM(v_pdshortfalls, "Page reclamation shortfalls");
 VM_STATS_VM(v_dfree, "Pages freed by pagedaemon");
 VM_STATS_VM(v_pfree, "Pages freed by exiting processes");
 VM_STATS_VM(v_tfree, "Total pages freed");
-VM_STATS_VM(v_page_size, "Page size in bytes");
-VM_STATS_VM(v_page_count, "Total number of pages in system");
-VM_STATS_VM(v_free_reserved, "Pages reserved for deadlock");
-VM_STATS_VM(v_free_target, "Pages desired free");
-VM_STATS_VM(v_free_min, "Minimum low-free-pages threshold");
-VM_STATS_VM(v_free_count, "Free pages");
-VM_STATS_VM(v_wire_count, "Wired pages");
-VM_STATS_VM(v_active_count, "Active pages");
-VM_STATS_VM(v_inactive_target, "Desired inactive pages");
-VM_STATS_VM(v_inactive_count, "Inactive pages");
-VM_STATS_VM(v_laundry_count, "Pages eligible for laundering");
-VM_STATS_VM(v_pageout_free_min, "Min pages reserved for kernel");
-VM_STATS_VM(v_interrupt_free_min, "Reserved pages for interrupt code");
 VM_STATS_VM(v_forks, "Number of fork() calls");
 VM_STATS_VM(v_vforks, "Number of vfork() calls");
 VM_STATS_VM(v_rforks, "Number of rfork() calls");
@ -310,6 +307,23 @@ VM_STATS_VM(v_vforkpages, "VM pages affected by vfork()");
 VM_STATS_VM(v_rforkpages, "VM pages affected by rfork()");
 VM_STATS_VM(v_kthreadpages, "VM pages affected by fork() by kernel");

+#define	VM_STATS_UINT(var, descr)	\
+    SYSCTL_UINT(_vm_stats_vm, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, 0, descr)
+VM_STATS_UINT(v_page_size, "Page size in bytes");
+VM_STATS_UINT(v_page_count, "Total number of pages in system");
+VM_STATS_UINT(v_free_reserved, "Pages reserved for deadlock");
+VM_STATS_UINT(v_free_target, "Pages desired free");
+VM_STATS_UINT(v_free_min, "Minimum low-free-pages threshold");
+VM_STATS_UINT(v_free_count, "Free pages");
+VM_STATS_UINT(v_wire_count, "Wired pages");
+VM_STATS_UINT(v_active_count, "Active pages");
+VM_STATS_UINT(v_inactive_target, "Desired inactive pages");
+VM_STATS_UINT(v_inactive_count, "Inactive pages");
+VM_STATS_UINT(v_laundry_count, "Pages eligible for laundering");
+VM_STATS_UINT(v_pageout_free_min, "Min pages reserved for kernel");
+VM_STATS_UINT(v_interrupt_free_min, "Reserved pages for interrupt code");
+VM_STATS_UINT(v_free_severe, "Severe page depletion point");
+
 #ifdef COMPAT_FREEBSD11
 /*
 * Provide compatibility sysctls for the benefit of old utilities which exit
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@ -765,7 +765,7 @@ vm_object_terminate(vm_object_t object)
 		p->object = NULL;
 		if (p->wire_count == 0) {
 			vm_page_free(p);
-			PCPU_INC(cnt.v_pfree);
+			VM_CNT_INC(v_pfree);
 		}
 		vm_page_unlock(p);
 	}
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@ -2748,7 +2748,7 @@ vm_page_free_toq(vm_page_t m)
 	} else
 		KASSERT(m->queue == PQ_NONE,
 		    ("vm_page_free_toq: unmanaged page %p is queued", m));
-	PCPU_INC(cnt.v_tfree);
+	VM_CNT_INC(v_tfree);

 	if (vm_page_sbusied(m))
 		panic("vm_page_free: freeing busy page %p", m);
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@ -667,7 +667,7 @@ vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
 				goto unlock_return;
 			if (vm_page_busied(p))
 				continue;
-			PCPU_INC(cnt.v_pdpages);
+			VM_CNT_INC(v_pdpages);
 			vm_page_lock(p);
 			if (p->wire_count != 0 || p->hold_count != 0 ||
 			    !pmap_page_exists_quick(pmap, p)) {
@ -1003,7 +1003,7 @@ vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall)
 		}
 		if (act_delta != 0) {
 			if (object->ref_count != 0) {
-				PCPU_INC(cnt.v_reactivated);
+				VM_CNT_INC(v_reactivated);
 				vm_page_activate(m);

 				/*
@ -1052,7 +1052,7 @@ vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall)
 		if (m->dirty == 0) {
 free_page:
 			vm_page_free(m);
-			PCPU_INC(cnt.v_dfree);
+			VM_CNT_INC(v_dfree);
 		} else if ((object->flags & OBJ_DEAD) == 0) {
 			if (object->type != OBJT_SWAP &&
 			    object->type != OBJT_DEFAULT)
@ -1187,7 +1187,7 @@ vm_pageout_laundry_worker(void *arg)
 		KASSERT(shortfall_cycle >= 0,
 		    ("negative cycle %d", shortfall_cycle));
 		launder = 0;
-		wakeups = VM_METER_PCPU_CNT(v_pdwakeups);
+		wakeups = VM_CNT_FETCH(v_pdwakeups);

 		/*
 		 * First determine whether we need to launder pages to meet a
@ -1378,7 +1378,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		KASSERT(queue_locked, ("unlocked inactive queue"));
 		KASSERT(vm_page_inactive(m), ("Inactive queue %p", m));

-		PCPU_INC(cnt.v_pdpages);
+		VM_CNT_INC(v_pdpages);
 		next = TAILQ_NEXT(m, plinks.q);

 		/*
@ -1476,7 +1476,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		}
 		if (act_delta != 0) {
 			if (object->ref_count != 0) {
-				PCPU_INC(cnt.v_reactivated);
+				VM_CNT_INC(v_reactivated);
 				vm_page_activate(m);

 				/*
@ -1521,7 +1521,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		if (m->dirty == 0) {
 free_page:
 			vm_page_free(m);
-			PCPU_INC(cnt.v_dfree);
+			VM_CNT_INC(v_dfree);
 			--page_shortage;
 		} else if ((object->flags & OBJ_DEAD) == 0)
 			vm_page_launder(m);
@ -1551,7 +1551,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		if (pq->pq_cnt > 0 || atomic_load_acq_int(&swapdev_enabled)) {
 			if (page_shortage > 0) {
 				vm_laundry_request = VM_LAUNDRY_SHORTFALL;
-				PCPU_INC(cnt.v_pdshortfalls);
+				VM_CNT_INC(v_pdshortfalls);
 			} else if (vm_laundry_request != VM_LAUNDRY_SHORTFALL)
 				vm_laundry_request = VM_LAUNDRY_BACKGROUND;
 			wakeup(&vm_laundry_request);
@ -1635,7 +1635,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		 * The count for page daemon pages is updated after checking
 		 * the page for eligibility.
 		 */
-		PCPU_INC(cnt.v_pdpages);
+		VM_CNT_INC(v_pdpages);

 		/*
 		 * Check to see "how much" the page has been used.
@ -2036,7 +2036,7 @@ vm_pageout_worker(void *arg)
 			if (mtx_sleep(&vm_pageout_wanted,
 			    &vm_page_queue_free_mtx, PDROP | PVM, "psleep",
 			    hz) == 0) {
-				PCPU_INC(cnt.v_pdwakeups);
+				VM_CNT_INC(v_pdwakeups);
 				pass = 1;
 			} else
 				pass = 0;
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@ -799,8 +799,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
 		relpbuf(bp, freecnt);
 		VM_OBJECT_WLOCK(object);
 		for (i = 0; i < count; i++) {
-			PCPU_INC(cnt.v_vnodein);
-			PCPU_INC(cnt.v_vnodepgsin);
+			VM_CNT_INC(v_vnodein);
+			VM_CNT_INC(v_vnodepgsin);
 			error = vnode_pager_input_old(object, m[i]);
 			if (error)
 				break;
@ -819,8 +819,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
 	if (pagesperblock == 0) {
 		relpbuf(bp, freecnt);
 		for (i = 0; i < count; i++) {
-			PCPU_INC(cnt.v_vnodein);
-			PCPU_INC(cnt.v_vnodepgsin);
+			VM_CNT_INC(v_vnodein);
+			VM_CNT_INC(v_vnodepgsin);
 			error = vnode_pager_input_smlfs(object, m[i]);
 			if (error)
 				break;
@ -1030,8 +1030,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
 	    (uintmax_t)blkno0, (uintmax_t)bp->b_blkno));

 	atomic_add_long(&runningbufspace, bp->b_runningbufspace);
-	PCPU_INC(cnt.v_vnodein);
-	PCPU_ADD(cnt.v_vnodepgsin, bp->b_npages);
+	VM_CNT_INC(v_vnodein);
+	VM_CNT_ADD(v_vnodepgsin, bp->b_npages);

 	if (iodone != NULL) { /* async */
 		bp->b_pgiodone = iodone;
@ -1276,8 +1276,8 @@ vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount,
 	auio.uio_td = (struct thread *) 0;
 	error = VOP_WRITE(vp, &auio, vnode_pager_putpages_ioflags(flags),
 	    curthread->td_ucred);
-	PCPU_INC(cnt.v_vnodeout);
-	PCPU_ADD(cnt.v_vnodepgsout, ncount);
+	VM_CNT_INC(v_vnodeout);
+	VM_CNT_ADD(v_vnodepgsout, ncount);

 	ppscheck = 0;
 	if (error) {
--- a/sys/x86/acpica/srat.c
+++ b/sys/x86/acpica/srat.c
@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_vm.h"

 #include <sys/param.h>
+#include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
--- a/sys/x86/x86/intr_machdep.c
+++ b/sys/x86/x86/intr_machdep.c
@ -246,7 +246,7 @@ intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame)
 	 * processed too.
 	 */
 	(*isrc->is_count)++;
-	PCPU_INC(cnt.v_intr);
+	VM_CNT_INC(v_intr);

 	ie = isrc->is_event;

--- a/usr.bin/top/machine.c
+++ b/usr.bin/top/machine.c
@ -516,7 +516,7 @@ get_system_info(struct system_info *si)
 		static int swapavail = 0;
 		static int swapfree = 0;
 		static long bufspace = 0;
-		static int nspgsin, nspgsout;
+		static uint64_t nspgsin, nspgsout;

 		GETSYSCTL("vfs.bufspace", bufspace);
 		GETSYSCTL("vm.stats.vm.v_active_count", memory_stats[0]);
--- a/usr.bin/vmstat/vmstat.c
+++ b/usr.bin/vmstat/vmstat.c
@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/time.h>
 #include <sys/user.h>
+#define	_WANT_VMMETER
 #include <sys/vmmeter.h>
 #include <sys/pcpu.h>

@ -127,7 +128,57 @@ static long select_generation;
 static char **specified_devices;
 static devstat_select_mode select_mode;

-static struct	vmmeter sum, osum;
+static struct __vmmeter {
+	uint64_t v_swtch;
+	uint64_t v_trap;
+	uint64_t v_syscall;
+	uint64_t v_intr;
+	uint64_t v_soft;
+	uint64_t v_vm_faults;
+	uint64_t v_io_faults;
+	uint64_t v_cow_faults;
+	uint64_t v_cow_optim;
+	uint64_t v_zfod;
+	uint64_t v_ozfod;
+	uint64_t v_swapin;
+	uint64_t v_swapout;
+	uint64_t v_swappgsin;
+	uint64_t v_swappgsout;
+	uint64_t v_vnodein;
+	uint64_t v_vnodeout;
+	uint64_t v_vnodepgsin;
+	uint64_t v_vnodepgsout;
+	uint64_t v_intrans;
+	uint64_t v_reactivated;
+	uint64_t v_pdwakeups;
+	uint64_t v_pdpages;
+	uint64_t v_pdshortfalls;
+	uint64_t v_dfree;
+	uint64_t v_pfree;
+	uint64_t v_tfree;
+	uint64_t v_forks;
+	uint64_t v_vforks;
+	uint64_t v_rforks;
+	uint64_t v_kthreads;
+	uint64_t v_forkpages;
+	uint64_t v_vforkpages;
+	uint64_t v_rforkpages;
+	uint64_t v_kthreadpages;
+	u_int v_page_size;
+	u_int v_page_count;
+	u_int v_free_reserved;
+	u_int v_free_target;
+	u_int v_free_min;
+	u_int v_free_count;
+	u_int v_wire_count;
+	u_int v_active_count;
+	u_int v_inactive_target;
+	u_int v_inactive_count;
+	u_int v_laundry_count;
+	u_int v_pageout_free_min;
+	u_int v_interrupt_free_min;
+	u_int v_free_severe;
+} sum, osum;

 #define	VMSTAT_DEFAULT_LINES	20	/* Default number of `winlines'. */
 volatile sig_atomic_t wresized;		/* Tty resized, when non-zero. */
@ -448,93 +499,50 @@ getuptime(void)
 }

 static void
-fill_pcpu(struct pcpu ***pcpup, int* maxcpup)
-{
-	struct pcpu **pcpu;
-	
-	int maxcpu, i;
-
-	*pcpup = NULL;
-	
-	if (kd == NULL)
-		return;
-
-	maxcpu = kvm_getmaxcpu(kd);
-	if (maxcpu < 0)
-		xo_errx(1, "kvm_getmaxcpu: %s", kvm_geterr(kd));
-
-	pcpu = calloc(maxcpu, sizeof(struct pcpu *));
-	if (pcpu == NULL)
-		xo_err(1, "calloc");
-
-	for (i = 0; i < maxcpu; i++) {
-		pcpu[i] = kvm_getpcpu(kd, i);
-		if (pcpu[i] == (struct pcpu *)-1)
-			xo_errx(1, "kvm_getpcpu: %s", kvm_geterr(kd));
-	}
-
-	*maxcpup = maxcpu;
-	*pcpup = pcpu;
-}
-
-static void
-free_pcpu(struct pcpu **pcpu, int maxcpu)
-{
-	int i;
-
-	for (i = 0; i < maxcpu; i++)
-		free(pcpu[i]);
-	free(pcpu);
-}
-
-static void
-fill_vmmeter(struct vmmeter *vmmp)
+fill_vmmeter(struct __vmmeter *vmmp)
 {
 	struct pcpu **pcpu;
 	int maxcpu, i;

 	if (kd != NULL) {
-		kread(X_SUM, vmmp, sizeof(*vmmp));
-		fill_pcpu(&pcpu, &maxcpu);
-		for (i = 0; i < maxcpu; i++) {
-			if (pcpu[i] == NULL)
-				continue;
-#define ADD_FROM_PCPU(i, name) \
-			vmmp->name += pcpu[i]->pc_cnt.name
-			ADD_FROM_PCPU(i, v_swtch);
-			ADD_FROM_PCPU(i, v_trap);
-			ADD_FROM_PCPU(i, v_syscall);
-			ADD_FROM_PCPU(i, v_intr);
-			ADD_FROM_PCPU(i, v_soft);
-			ADD_FROM_PCPU(i, v_vm_faults);
-			ADD_FROM_PCPU(i, v_io_faults);
-			ADD_FROM_PCPU(i, v_cow_faults);
-			ADD_FROM_PCPU(i, v_cow_optim);
-			ADD_FROM_PCPU(i, v_zfod);
-			ADD_FROM_PCPU(i, v_ozfod);
-			ADD_FROM_PCPU(i, v_swapin);
-			ADD_FROM_PCPU(i, v_swapout);
-			ADD_FROM_PCPU(i, v_swappgsin);
-			ADD_FROM_PCPU(i, v_swappgsout);
-			ADD_FROM_PCPU(i, v_vnodein);
-			ADD_FROM_PCPU(i, v_vnodeout);
-			ADD_FROM_PCPU(i, v_vnodepgsin);
-			ADD_FROM_PCPU(i, v_vnodepgsout);
-			ADD_FROM_PCPU(i, v_intrans);
-			ADD_FROM_PCPU(i, v_tfree);
-			ADD_FROM_PCPU(i, v_forks);
-			ADD_FROM_PCPU(i, v_vforks);
-			ADD_FROM_PCPU(i, v_rforks);
-			ADD_FROM_PCPU(i, v_kthreads);
-			ADD_FROM_PCPU(i, v_forkpages);
-			ADD_FROM_PCPU(i, v_vforkpages);
-			ADD_FROM_PCPU(i, v_rforkpages);
-			ADD_FROM_PCPU(i, v_kthreadpages);
-#undef ADD_FROM_PCPU
-		}
-		free_pcpu(pcpu, maxcpu);
+		struct vmmeter vm_cnt;
+
+		kread(X_SUM, &vm_cnt, sizeof(vm_cnt));
+#define	GET_COUNTER(name) \
+		vmmp->name = kvm_counter_u64_fetch(kd, (u_long)vm_cnt.name)
+		GET_COUNTER(v_swtch);
+		GET_COUNTER(v_trap);
+		GET_COUNTER(v_syscall);
+		GET_COUNTER(v_intr);
+		GET_COUNTER(v_soft);
+		GET_COUNTER(v_vm_faults);
+		GET_COUNTER(v_io_faults);
+		GET_COUNTER(v_cow_faults);
+		GET_COUNTER(v_cow_optim);
+		GET_COUNTER(v_zfod);
+		GET_COUNTER(v_ozfod);
+		GET_COUNTER(v_swapin);
+		GET_COUNTER(v_swapout);
+		GET_COUNTER(v_swappgsin);
+		GET_COUNTER(v_swappgsout);
+		GET_COUNTER(v_vnodein);
+		GET_COUNTER(v_vnodeout);
+		GET_COUNTER(v_vnodepgsin);
+		GET_COUNTER(v_vnodepgsout);
+		GET_COUNTER(v_intrans);
+		GET_COUNTER(v_tfree);
+		GET_COUNTER(v_forks);
+		GET_COUNTER(v_vforks);
+		GET_COUNTER(v_rforks);
+		GET_COUNTER(v_kthreads);
+		GET_COUNTER(v_forkpages);
+		GET_COUNTER(v_vforkpages);
+		GET_COUNTER(v_rforkpages);
+		GET_COUNTER(v_kthreadpages);
+#undef GET_COUNTER
 	} else {
-		size_t size = sizeof(unsigned int);
+		size_t size = sizeof(uint64_t);
+
 #define GET_VM_STATS(cat, name) \
 	mysysctl("vm.stats." #cat "." #name, &vmmp->name, &size, NULL, 0)
 		/* sys */