rangeset: speed up range traversal

For rangeset-next search, use exact search rather than greater-than search.

Move a bit of the testing logic from the pmap code to the common rangeset code.

Reviewed by:	kib (previous version)
Tested by:	pho (previous version)
Differential Revision:	https://reviews.freebsd.org/D46314

sys/amd64/amd64/pmap.c

@@ -11468,7 +11468,7 @@ pmap_pkru_deassign_all(pmap_t pmap)
 static bool
 pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)
 {
-	struct pmap_pkru_range *next_ppr, *ppr;
+	struct pmap_pkru_range *ppr;
 	vm_offset_t va;
 	u_int keyidx;
 
@@ -11480,20 +11480,14 @@ pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)
 	    sva >= VM_MAXUSER_ADDRESS)
 		return (true);
 	MPASS(eva <= VM_MAXUSER_ADDRESS);
-	ppr = rangeset_lookup(&pmap->pm_pkru, sva);
-	if (ppr == NULL) {
-		ppr = rangeset_next(&pmap->pm_pkru, sva);
-		return (ppr == NULL ||
-		    ppr->pkru_rs_el.re_start >= eva);
-	}
+	ppr = rangeset_containing(&pmap->pm_pkru, sva);
+	if (ppr == NULL)
+		return (rangeset_empty(&pmap->pm_pkru, sva, eva));
 	keyidx = ppr->pkru_keyidx;
 	while ((va = ppr->pkru_rs_el.re_end) < eva) {
-		next_ppr = rangeset_next(&pmap->pm_pkru, va);
-		if (next_ppr == NULL ||
-		    va != next_ppr->pkru_rs_el.re_start ||
-		    keyidx != next_ppr->pkru_keyidx)
+		if ((ppr = rangeset_beginning(&pmap->pm_pkru, va)) == NULL ||
+		    keyidx != ppr->pkru_keyidx)
 			return (false);
-		ppr = next_ppr;
 	}
 	*pte |= X86_PG_PKU(keyidx);
 	return (true);
@@ -11509,7 +11503,7 @@ pmap_pkru_get(pmap_t pmap, vm_offset_t va)
 	    (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0 ||
 	    va >= VM_MAXUSER_ADDRESS)
 		return (0);
-	ppr = rangeset_lookup(&pmap->pm_pkru, va);
+	ppr = rangeset_containing(&pmap->pm_pkru, va);
 	if (ppr != NULL)
 		return (X86_PG_PKU(ppr->pkru_keyidx));
 	return (0);

sys/arm64/arm64/pmap.c

@@ -9370,7 +9370,7 @@ pmap_bti_deassign_all(pmap_t pmap)
 static bool
 pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)
 {
-	struct rs_el *next_rs, *rs;
+	struct rs_el *rs;
 	vm_offset_t va;
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
@@ -9388,18 +9388,12 @@ pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)
 	if (pmap->pm_bti == NULL)
 		return (true);
 	PMAP_ASSERT_STAGE1(pmap);
-	rs = rangeset_lookup(pmap->pm_bti, sva);
-	if (rs == NULL) {
-		rs = rangeset_next(pmap->pm_bti, sva);
-		return (rs == NULL ||
-			rs->re_start >= eva);
-	}
+	rs = rangeset_containing(pmap->pm_bti, sva);
+	if (rs == NULL)
+		return (rangeset_empty(pmap->pm_bti, sva, eva));
 	while ((va = rs->re_end) < eva) {
-		next_rs = rangeset_next(pmap->pm_bti, va);
-		if (next_rs == NULL ||
-		    va != next_rs->re_start)
+		if ((rs = rangeset_beginning(pmap->pm_bti, va)) == NULL)
 			return (false);
-		rs = next_rs;
 	}
 	*pte |= ATTR_S1_GP;
 	return (true);
@@ -9415,7 +9409,8 @@ pmap_pte_bti(pmap_t pmap, vm_offset_t va)
 		return (0);
 	if (pmap == kernel_pmap)
 		return (ATTR_KERN_GP);
-	if (pmap->pm_bti != NULL && rangeset_lookup(pmap->pm_bti, va) != NULL)
+	if (pmap->pm_bti != NULL &&
+	    rangeset_containing(pmap->pm_bti, va) != NULL)
 		return (ATTR_S1_GP);
 	return (0);
 }

sys/kern/subr_rangeset.c

@@ -248,25 +248,32 @@ rangeset_remove_all(struct rangeset *rs)
 }
 
 void *
-rangeset_lookup(struct rangeset *rs, uint64_t place)
+rangeset_containing(struct rangeset *rs, uint64_t place)
 {
 	struct rs_el *r;
 
 	rangeset_check(rs);
 	r = RANGESET_PCTRIE_LOOKUP_LE(&rs->rs_trie, place);
-	if (r == NULL)
-		return (NULL);
-	if (r->re_end <= place)
-		return (NULL);
-	return (r);
+	if (r != NULL && place < r->re_end)
+		return (r);
+	return (NULL);
+}
+
+bool
+rangeset_empty(struct rangeset *rs, uint64_t start, uint64_t end)
+{
+	struct rs_el *r;
+
+	r = RANGESET_PCTRIE_LOOKUP_GE(&rs->rs_trie, start + 1);
+	return (r == NULL || r->re_start >= end);
 }
 
 void *
-rangeset_next(struct rangeset *rs, uint64_t place)
+rangeset_beginning(struct rangeset *rs, uint64_t place)
 {
 
 	rangeset_check(rs);
-	return (RANGESET_PCTRIE_LOOKUP_GE(&rs->rs_trie, place));
+	return (RANGESET_PCTRIE_LOOKUP(&rs->rs_trie, place));
 }
 
 int

sys/sys/rangeset.h

@@ -69,15 +69,19 @@ int	rangeset_remove_pred(struct rangeset *rs, uint64_t start,
 	    uint64_t end, rs_pred_t pred);
 
 /*
- * Really returns the pointer to the data with struct rs_el embedded
- * at the beginning.
+ * Finds the range that contains place, if any.
  */
-void	*rangeset_lookup(struct rangeset *rs, uint64_t place);
+void	*rangeset_containing(struct rangeset *rs, uint64_t place);
 
 /*
- * Finds the first range that begins at or after place.
+ * Report whether no range begins between start and end.
  */
-void	*rangeset_next(struct rangeset *rs, uint64_t place);
+bool	rangeset_empty(struct rangeset *rs, uint64_t start, uint64_t end);
+
+/*
+ * Finds the range that begins at place, if any.
+ */
+void	*rangeset_beginning(struct rangeset *rs, uint64_t place);
 
 /*
  * Copies src_rs entries into dst_rs.  dst_rs must be empty.