vm_page: Consolidate page busy sleep mechanisms

- Modify vm_page_busy_sleep() and vm_page_busy_sleep_unlocked() to take
  a VM_ALLOC_* flag indicating whether to sleep on shared-busy, and fix
  up callers.
- Modify vm_page_busy_sleep() to return a status indicating whether the
  object lock was dropped, and fix up callers.
- Convert callers of vm_page_sleep_if_busy() to use vm_page_busy_sleep()
  instead.
- Remove vm_page_sleep_if_(x)busy().

No functional change intended.

Obtained from:	jeff (object_concurrency patches)
Reviewed by:	kib
MFC after:	2 weeks
Differential Revision:	https://reviews.freebsd.org/D32947

ObsoleteFiles.inc

@@ -40,6 +40,9 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20211115: vm_page_sleep_if_busy removed
+OLD_FILES+=share/man/man9/vm_page_sleep_if_busy.9.gz
+
 # 20211113: new clang import which bumps version from 12.0.1 to 13.0.0.
 OLD_FILES+=usr/lib/clang/12.0.1/include/cuda_wrappers/algorithm
 OLD_FILES+=usr/lib/clang/12.0.1/include/cuda_wrappers/complex

share/man/man9/Makefile

@@ -2354,7 +2354,6 @@ MLINKS+=vm_page_busy.9 vm_page_busied.9 \
 	vm_page_busy.9 vm_page_busy_sleep.9 \
 	vm_page_busy.9 vm_page_sbusied.9 \
 	vm_page_busy.9 vm_page_sbusy.9 \
-	vm_page_busy.9 vm_page_sleep_if_busy.9 \
 	vm_page_busy.9 vm_page_sunbusy.9 \
 	vm_page_busy.9 vm_page_trysbusy.9 \
 	vm_page_busy.9 vm_page_tryxbusy.9 \

share/man/man9/vm_page_busy.9

@@ -24,7 +24,7 @@
 .\" DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd August 07, 2013
+.Dd November 11, 2021
 .Dt VM_PAGE_BUSY 9
 .Os
 .Sh NAME
@@ -33,7 +33,6 @@
 .Nm vm_page_busy_sleep ,
 .Nm vm_page_sbusied ,
 .Nm vm_page_sbusy ,
-.Nm vm_page_sleep_if_busy ,
 .Nm vm_page_sunbusy ,
 .Nm vm_page_trysbusy ,
 .Nm vm_page_tryxbusy ,
@@ -52,14 +51,12 @@
 .Fn vm_page_busied "vm_page_t m"
 .Ft void
 .Fn vm_page_busy_downgrade "vm_page_t m"
-.Ft void
-.Fn vm_page_busy_sleep "vm_page_t m" "const char *msg"
+.Ft bool
+.Fn vm_page_busy_sleep "vm_page_t m" "const char *msg" "int allocflags"
 .Ft int
 .Fn vm_page_sbusied "vm_page_t m"
 .Ft void
 .Fn vm_page_sbusy "vm_page_t m"
-.Ft int
-.Fn vm_page_sleep_if_busy "vm_page_t m" "const char *msg"
 .Ft void
 .Fn vm_page_sunbusy "vm_page_t m"
 .Ft int
@@ -114,8 +111,23 @@ from an exclusive busy state to a shared busy state.
 .Pp
 The
 .Fn vm_page_busy_sleep
-function puts the invoking thread to sleep using the appropriate
-waitchannels for the busy mechanism.
+checks the busy state of the page
+.Fa m
+and puts the invoking thread to sleep if the page is busy.
+The VM object for the page must be locked.
+The
+.Fa allocflags
+parameter must be either
+.Dv 0 ,
+in which case the function will sleep if the page is busied,
+or
+.Dv VM_ALLOC_IGN_SBUSY ,
+in which case the function will sleep only if the page is exclusively
+busied.
+A return value of true indicates that the invoking thread was put to
+sleep and that the object was unlocked.
+A return value of false indicates that the invoking thread did not sleep
+and the object remains locked.
 The parameter
 .Fa msg
 is a string describing the sleep condition for userland tools.
@@ -133,18 +145,6 @@ function shared busies
 .Fa m .
 .Pp
 The
-.Fn vm_page_sleep_if_busy
-function puts the invoking thread to sleep, using the appropriate
-waitchannels for the busy mechanism, if
-.Fa m .
-is busied in either exclusive or shared mode.
-If the invoking thread slept a non-zero value is returned, otherwise
-0 is returned.
-The parameter
-.Fa msg
-is a string describing the sleep condition for userland tools.
-.Pp
-The
 .Fn vm_page_sunbusy
 function shared unbusies
 .Fa m .

sys/vm/vm_fault.c

@@ -1294,9 +1294,8 @@ vm_fault_busy_sleep(struct faultstate *fs)
 	}
 	vm_object_pip_wakeup(fs->object);
 	unlock_map(fs);
-	if (fs->m == vm_page_lookup(fs->object, fs->pindex))
-		vm_page_busy_sleep(fs->m, "vmpfw", false);
-	else
+	if (fs->m != vm_page_lookup(fs->object, fs->pindex) ||
+	    !vm_page_busy_sleep(fs->m, "vmpfw", 0))
 		VM_OBJECT_WUNLOCK(fs->object);
 	VM_CNT_INC(v_intrans);
 	vm_object_deallocate(fs->first_object);

sys/vm/vm_object.c

@@ -1397,7 +1397,8 @@ vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
 				 */
 				vm_page_aflag_set(tm, PGA_REFERENCED);
 			}
-			vm_page_busy_sleep(tm, "madvpo", false);
+			if (!vm_page_busy_sleep(tm, "madvpo", 0))
+				VM_OBJECT_WUNLOCK(tobject);
   			goto relookup;
 		}
 		vm_page_advise(tm, advice);
@@ -1581,7 +1582,8 @@ vm_object_split(vm_map_entry_t entry)
 		 */
 		if (vm_page_tryxbusy(m) == 0) {
 			VM_OBJECT_WUNLOCK(new_object);
-			vm_page_sleep_if_busy(m, "spltwt");
+			if (vm_page_busy_sleep(m, "spltwt", 0))
+				VM_OBJECT_WLOCK(orig_object);
 			VM_OBJECT_WLOCK(new_object);
 			goto retry;
 		}
@@ -1667,14 +1669,17 @@ vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
 		VM_OBJECT_WUNLOCK(object);
 		VM_OBJECT_WUNLOCK(backing_object);
 		vm_radix_wait();
+		VM_OBJECT_WLOCK(object);
+	} else if (p->object == object) {
+		VM_OBJECT_WUNLOCK(backing_object);
+		if (vm_page_busy_sleep(p, "vmocol", 0))
+			VM_OBJECT_WLOCK(object);
 	} else {
-		if (p->object == object)
+		VM_OBJECT_WUNLOCK(object);
+		if (!vm_page_busy_sleep(p, "vmocol", 0))
 			VM_OBJECT_WUNLOCK(backing_object);
-		else
-			VM_OBJECT_WUNLOCK(object);
-		vm_page_busy_sleep(p, "vmocol", false);
+		VM_OBJECT_WLOCK(object);
 	}
-	VM_OBJECT_WLOCK(object);
 	VM_OBJECT_WLOCK(backing_object);
 	return (TAILQ_FIRST(&backing_object->memq));
 }
@@ -2103,7 +2108,8 @@ vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
 		 * not specified.
 		 */
 		if (vm_page_tryxbusy(p) == 0) {
-			vm_page_sleep_if_busy(p, "vmopar");
+			if (vm_page_busy_sleep(p, "vmopar", 0))
+				VM_OBJECT_WLOCK(object);
 			goto again;
 		}
 		if (vm_page_wired(p)) {
@@ -2407,7 +2413,10 @@ vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
 					VM_OBJECT_RUNLOCK(tobject);
 				tobject = t1object;
 			}
-			vm_page_busy_sleep(tm, "unwbo", true);
+			tobject = tm->object;
+			if (!vm_page_busy_sleep(tm, "unwbo",
+			    VM_ALLOC_IGN_SBUSY))
+				VM_OBJECT_RUNLOCK(tobject);
 			goto again;
 		}
 		vm_page_unwire(tm, queue);

sys/vm/vm_page.c

@@ -1000,24 +1000,26 @@ vm_page_sunbusy(vm_page_t m)
  *	vm_page_busy_sleep:
  *
  *	Sleep if the page is busy, using the page pointer as wchan.
- *	This is used to implement the hard-path of busying mechanism.
+ *	This is used to implement the hard-path of the busying mechanism.
  *
- *	If nonshared is true, sleep only if the page is xbusy.
+ *	If VM_ALLOC_IGN_SBUSY is specified in allocflags, the function
+ *	will not sleep if the page is shared-busy.
  *
- *	The object lock must be held on entry and will be released on exit.
+ *	The object lock must be held on entry.
+ *
+ *	Returns true if it slept and dropped the object lock, or false
+ *	if there was no sleep and the lock is still held.
  */
-void
-vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
+bool
+vm_page_busy_sleep(vm_page_t m, const char *wmesg, int allocflags)
 {
 	vm_object_t obj;
 
 	obj = m->object;
 	VM_OBJECT_ASSERT_LOCKED(obj);
-	vm_page_lock_assert(m, MA_NOTOWNED);
 
-	if (!_vm_page_busy_sleep(obj, m, m->pindex, wmesg,
-	    nonshared ? VM_ALLOC_SBUSY : 0 , true))
-		VM_OBJECT_DROP(obj);
+	return (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, allocflags,
+	    true));
 }
 
 /*
@@ -1026,21 +1028,19 @@ vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
  *	Sleep if the page is busy, using the page pointer as wchan.
  *	This is used to implement the hard-path of busying mechanism.
  *
- *	If nonshared is true, sleep only if the page is xbusy.
+ *	If VM_ALLOC_IGN_SBUSY is specified in allocflags, the function
+ *	will not sleep if the page is shared-busy.
  *
  *	The object lock must not be held on entry.  The operation will
  *	return if the page changes identity.
  */
 void
 vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
-    const char *wmesg, bool nonshared)
+    const char *wmesg, int allocflags)
 {
-
 	VM_OBJECT_ASSERT_UNLOCKED(obj);
-	vm_page_lock_assert(m, MA_NOTOWNED);
 
-	_vm_page_busy_sleep(obj, m, pindex, wmesg,
-	    nonshared ? VM_ALLOC_SBUSY : 0, false);
+	(void)_vm_page_busy_sleep(obj, m, pindex, wmesg, allocflags, false);
 }
 
 /*
@@ -1050,6 +1050,8 @@ vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
  *	lockstate against parameters.  Returns true if it sleeps and
  *	false otherwise.
  *
+ *	allocflags uses VM_ALLOC_* flags to specify the lock required.
+ *
  *	If locked is true the lock will be dropped for any true returns
  *	and held for any false returns.
  */
@@ -1395,71 +1397,6 @@ vm_page_free_invalid(vm_page_t m)
 		vm_page_free(m);
 }
 
-/*
- *	vm_page_sleep_if_busy:
- *
- *	Sleep and release the object lock if the page is busied.
- *	Returns TRUE if the thread slept.
- *
- *	The given page must be unlocked and object containing it must
- *	be locked.
- */
-int
-vm_page_sleep_if_busy(vm_page_t m, const char *wmesg)
-{
-	vm_object_t obj;
-
-	vm_page_lock_assert(m, MA_NOTOWNED);
-	VM_OBJECT_ASSERT_WLOCKED(m->object);
-
-	/*
-	 * The page-specific object must be cached because page
-	 * identity can change during the sleep, causing the
-	 * re-lock of a different object.
-	 * It is assumed that a reference to the object is already
-	 * held by the callers.
-	 */
-	obj = m->object;
-	if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, 0, true)) {
-		VM_OBJECT_WLOCK(obj);
-		return (TRUE);
-	}
-	return (FALSE);
-}
-
-/*
- *	vm_page_sleep_if_xbusy:
- *
- *	Sleep and release the object lock if the page is xbusied.
- *	Returns TRUE if the thread slept.
- *
- *	The given page must be unlocked and object containing it must
- *	be locked.
- */
-int
-vm_page_sleep_if_xbusy(vm_page_t m, const char *wmesg)
-{
-	vm_object_t obj;
-
-	vm_page_lock_assert(m, MA_NOTOWNED);
-	VM_OBJECT_ASSERT_WLOCKED(m->object);
-
-	/*
-	 * The page-specific object must be cached because page
-	 * identity can change during the sleep, causing the
-	 * re-lock of a different object.
-	 * It is assumed that a reference to the object is already
-	 * held by the callers.
-	 */
-	obj = m->object;
-	if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, VM_ALLOC_SBUSY,
-	    true)) {
-		VM_OBJECT_WLOCK(obj);
-		return (TRUE);
-	}
-	return (FALSE);
-}
-
 /*
  *	vm_page_dirty_KBI:		[ internal use only ]
  *

sys/vm/vm_page.h

@@ -599,9 +599,9 @@ malloc2vm_flags(int malloc_flags)
 bool vm_page_busy_acquire(vm_page_t m, int allocflags);
 void vm_page_busy_downgrade(vm_page_t m);
 int vm_page_busy_tryupgrade(vm_page_t m);
-void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared);
+bool vm_page_busy_sleep(vm_page_t m, const char *msg, int allocflags);
 void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m,
-    vm_pindex_t pindex, const char *wmesg, bool nonshared);
+    vm_pindex_t pindex, const char *wmesg, int allocflags);
 void vm_page_free(vm_page_t m);
 void vm_page_free_zero(vm_page_t m);
 
@@ -685,8 +685,6 @@ vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start,
     vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options);
 vm_page_bits_t vm_page_set_dirty(vm_page_t m);
 void vm_page_set_valid_range(vm_page_t m, int base, int size);
-int vm_page_sleep_if_busy(vm_page_t m, const char *msg);
-int vm_page_sleep_if_xbusy(vm_page_t m, const char *msg);
 vm_offset_t vm_page_startup(vm_offset_t vaddr);
 void vm_page_sunbusy(vm_page_t m);
 bool vm_page_try_remove_all(vm_page_t m);