Make withering water tight.

When we orphan/wither a provider, an attached geom+consumer could
end up being withered as a result and it may be in front of us in
the normal object scanning order so we need to do multi-pass.  On
the other hand, there may be withering stuff we can't get rid off
(yet), so we need to keep track of both the existence of withering
stuff and if there is more we can do at this time.

sys/geom/geom_event.c

@@ -59,6 +59,7 @@ static struct event_tailq_head g_events = TAILQ_HEAD_INITIALIZER(g_events);
 static u_int g_pending_events;
 static TAILQ_HEAD(,g_provider) g_doorstep = TAILQ_HEAD_INITIALIZER(g_doorstep);
 static struct mtx g_eventlock;
+static int g_wither_work;
 
 #define G_N_EVENTREFS		20
 
@@ -194,9 +195,18 @@ one_event(void)
 void
 g_run_events()
 {
+	int i;
 
 	while (one_event())
 		;
+	g_topology_lock();
+	i = g_wither_work;
+	while (i) {
+		i = g_wither_washer();
+		g_wither_work = i & 1;
+		i &= 2;
+	}
+	g_topology_unlock();
 }
 
 void
@@ -286,6 +296,12 @@ g_post_event(g_event_t *func, void *arg, int flag, ...)
 	return (i);
 }
 
+void
+g_do_wither() {
+
+	g_wither_work = 1;
+	wakeup(&g_wait_event);
+}
 
 /*
  * XXX: It might actually be useful to call this function with topology held.

sys/geom/geom_int.h

@@ -68,10 +68,12 @@ void g_conftxt(void *, int flag);
 /* geom_event.c */
 void g_event_init(void);
 void g_run_events(void);
+void g_do_wither(void);
 
 /* geom_subr.c */
 extern struct class_list_head g_classes;
 extern char *g_wait_event, *g_wait_sim, *g_wait_up, *g_wait_down;
+int g_wither_washer(void);
 
 /* geom_io.c */
 void g_io_init(void);

sys/geom/geom_subr.c

@@ -263,15 +263,10 @@ g_destroy_geom(struct g_geom *gp)
 void
 g_wither_geom(struct g_geom *gp, int error)
 {
-	struct g_provider *pp, *pp2;
-	struct g_consumer *cp, *cp2;
-	static int once_is_enough;
+	struct g_provider *pp;
 
 	g_topology_assert();
 	G_VALID_GEOM(gp);
-	if (once_is_enough)
-		return;
-	once_is_enough = 1;
 	g_trace(G_T_TOPOLOGY, "g_wither_geom(%p(%s))", gp, gp->name);
 	if (!(gp->flags & G_GEOM_WITHER)) {
 		gp->flags |= G_GEOM_WITHER;
@@ -279,22 +274,61 @@ g_wither_geom(struct g_geom *gp, int error)
 			if (!(pp->flags & G_PF_ORPHAN))
 				g_orphan_provider(pp, error);
 	}
-	for (pp = LIST_FIRST(&gp->provider); pp != NULL; pp = pp2) {
-		pp2 = LIST_NEXT(pp, provider);
-		if (!LIST_EMPTY(&pp->consumers))
-			continue;
-		g_destroy_provider(pp);
+	g_do_wither();
+}
+
+/*
+ * This function is called (repeatedly) until we cant wash away more
+ * withered bits at present.  Return value contains two bits.  Bit 0
+ * set means "withering stuff we can't wash now", bit 1 means "call
+ * me again, there may be stuff I didn't get the first time around.
+ */
+int
+g_wither_washer()
+{
+	struct g_class *mp;
+	struct g_geom *gp, *gp2;
+	struct g_provider *pp, *pp2;
+	struct g_consumer *cp, *cp2;
+	int result;
+
+	result = 0;
+	g_topology_assert();
+	LIST_FOREACH(mp, &g_classes, class) {
+		LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
+			LIST_FOREACH_SAFE(pp, &gp->provider, provider, pp2) {
+				if (!(pp->flags & G_PF_WITHER))
+					continue;
+				if (LIST_EMPTY(&pp->consumers))
+					g_destroy_provider(pp);
+				else
+					result |= 1;
+			}
+			if (!(gp->flags & G_GEOM_WITHER))
+				continue;
+			LIST_FOREACH_SAFE(pp, &gp->provider, provider, pp2) {
+				if (LIST_EMPTY(&pp->consumers))
+					g_destroy_provider(pp);
+				else
+					result |= 1;
+			}
+			LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp2) {
+				if (cp->acr || cp->acw || cp->ace) {
+					result |= 1;
+					continue;
+				}
+				g_detach(cp);
+				g_destroy_consumer(cp);
+				result |= 2;
+			}
+			if (LIST_EMPTY(&gp->provider) &&
+			    LIST_EMPTY(&gp->consumer))
+				g_destroy_geom(gp);
+			else
+				result |= 1;
+		}
 	}
-	for (cp = LIST_FIRST(&gp->consumer); cp != NULL; cp = cp2) {
-		cp2 = LIST_NEXT(cp, consumer);
-		if (cp->acr || cp->acw || cp->ace)
-			continue;
-		g_detach(cp);
-		g_destroy_consumer(cp);
-	}
-	if (LIST_EMPTY(&gp->provider) && LIST_EMPTY(&gp->consumer))
-		g_destroy_geom(gp);
-	once_is_enough = 0;
+	return (result);
 }
 
 struct g_consumer *
@@ -337,7 +371,7 @@ g_destroy_consumer(struct g_consumer *cp)
 	devstat_remove_entry(cp->stat);
 	g_free(cp);
 	if (gp->flags & G_GEOM_WITHER)
-		g_wither_geom(gp, 0);
+		g_do_wither();
 }
 
 static void
@@ -451,7 +485,7 @@ g_destroy_provider(struct g_provider *pp)
 	devstat_remove_entry(pp->stat);
 	g_free(pp);
 	if ((gp->flags & G_GEOM_WITHER))
-		g_wither_geom(gp, 0);
+		g_do_wither();
 }
 
 /*
@@ -561,9 +595,9 @@ g_detach(struct g_consumer *cp)
 	LIST_REMOVE(cp, consumers);
 	cp->provider = NULL;
 	if (pp->geom->flags & G_GEOM_WITHER)
-		g_wither_geom(pp->geom, 0);
+		g_do_wither();
 	else if (pp->flags & G_PF_WITHER)
-		g_destroy_provider(pp);
+		g_do_wither();
 	redo_rank(cp->geom);
 }