Implement checkparity/rebuildparity.

svn path=/head/; revision=138110

sbin/gvinum/gvinum.c

@@ -55,6 +55,7 @@ void	gvinum_create(int, char **);
 void	gvinum_help(void);
 void	gvinum_init(int, char **);
 void	gvinum_list(int, char **);
+void	gvinum_parityop(int, char **, int);
 void	gvinum_printconfig(int, char **);
 void	gvinum_rm(int, char **);
 void	gvinum_saveconfig(void);
@@ -551,6 +552,89 @@ gvinum_printconfig(int argc, char **argv)
 	printconfig(stdout, "");
 }
 
+void
+gvinum_parityop(int argc, char **argv, int rebuild)
+{
+	struct gctl_req *req;
+	int flags, i, rv;
+	off_t offset;
+	const char *errstr;
+	char *op, *msg;
+
+	if (rebuild) {
+		op = "rebuildparity";
+		msg = "Rebuilding";
+	} else {
+		op = "checkparity";
+		msg = "Checking";
+	}
+
+	optreset = 1;
+	optind = 1;
+	flags = 0;
+	while ((i = getopt(argc, argv, "fv")) != -1) {
+		switch (i) {
+		case 'f':
+			flags |= GV_FLAG_F;
+			break;
+		case 'v':
+			flags |= GV_FLAG_V;
+			break;
+		case '?':
+		default:
+			warnx("invalid flag '%c'", i);
+			return;
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc != 1) {
+		warn("usage: %s [-f] [-v] <plex>", op);
+		return;
+	}
+
+	do {
+		rv = 0;
+		req = gctl_get_handle();
+		gctl_ro_param(req, "class", -1, "VINUM");
+		gctl_ro_param(req, "verb", -1, "parityop");
+		gctl_ro_param(req, "flags", sizeof(int), &flags);
+		gctl_ro_param(req, "rebuild", sizeof(int), &rebuild);
+		gctl_rw_param(req, "rv", sizeof(int), &rv);
+		gctl_rw_param(req, "offset", sizeof(off_t), &offset);
+		gctl_ro_param(req, "plex", -1, argv[0]);
+		errstr = gctl_issue(req);
+		if (errstr) {
+			warnx("%s\n", errstr);
+			gctl_free(req);
+			break;
+		}
+		gctl_free(req);
+		if (flags & GV_FLAG_V) {
+			printf("\r%s at %s ... ", msg,
+			    gv_roughlength(offset, 1));
+		}
+		if (rv == 1) {
+			printf("Parity incorrect at offset 0x%jx\n",
+			    (intmax_t)offset);
+			if (!rebuild)
+				break;
+		}
+		fflush(stdout);
+
+		/* Clear the -f flag. */
+		flags &= ~GV_FLAG_F;
+	} while (rv >= 0);
+
+	if ((rv == 2) && (flags & GV_FLAG_V)) {
+		if (rebuild)
+			printf("Rebuilt parity on %s\n", argv[0]);
+		else
+			printf("%s has correct parity\n", argv[0]);
+	}
+}
+
 void
 gvinum_rm(int argc, char **argv)
 {
@@ -721,6 +805,10 @@ parseline(int argc, char **argv)
 		gvinum_start(argc, argv);
 	else if (!strcmp(argv[0], "stop"))
 		gvinum_stop(argc, argv);
+	else if (!strcmp(argv[0], "checkparity"))
+		gvinum_parityop(argc, argv, 0);
+	else if (!strcmp(argv[0], "rebuildparity"))
+		gvinum_parityop(argc, argv, 1);
 	else
 		printf("unknown command '%s'\n", argv[0]);
 

sys/geom/vinum/geom_vinum.c

@@ -503,6 +503,9 @@ gv_config(struct gctl_req *req, struct g_class *mp, char const *verb)
 	} else if (!strcmp(verb, "create")) {
 		gv_create(gp, req);
 
+	} else if (!strcmp(verb, "parityop")) {
+		gv_parityop(gp, req);
+
 	} else if (!strcmp(verb, "remove")) {
 		gv_remove(gp, req);
 

sys/geom/vinum/geom_vinum.h

@@ -38,6 +38,7 @@ void	gv_save_config(struct g_consumer *, struct gv_drive *,
 	    struct gv_softc *);
 
 /* geom_vinum_init.c */
+void	gv_parityop(struct g_geom *, struct gctl_req *);
 void	gv_start_obj(struct g_geom *, struct gctl_req *);
 
 /* geom_vinum_list.c */

sys/geom/vinum/geom_vinum_init.c

@@ -57,6 +57,125 @@ struct gv_sync_args {
 	off_t syncsize;
 };
 
+void
+gv_parityop(struct g_geom *gp, struct gctl_req *req)
+{
+	struct gv_softc *sc;
+	struct gv_plex *p;
+	struct bio *bp;
+	struct g_consumer *cp;
+	int error, *flags, type, *rebuild, rv;
+	char *plex;
+
+	rv = -1;
+
+	plex = gctl_get_param(req, "plex", NULL);
+	if (plex == NULL) {
+		gctl_error(req, "no plex given");
+		goto out;
+	}
+
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	if (flags == NULL) {
+		gctl_error(req, "no flags given");
+		goto out;
+	}
+
+	rebuild = gctl_get_paraml(req, "rebuild", sizeof(*rebuild));
+	if (rebuild == NULL) {
+		gctl_error(req, "no rebuild op given");
+		goto out;
+	}
+
+	sc = gp->softc;
+	type = gv_object_type(sc, plex);
+	switch (type) {
+	case GV_TYPE_PLEX:
+		break;
+	case GV_TYPE_VOL:
+	case GV_TYPE_SD:
+	case GV_TYPE_DRIVE:
+	default:
+		gctl_error(req, "'%s' is not a plex", plex);
+		goto out;
+	}
+
+	p = gv_find_plex(sc, plex);
+	if (p->state != GV_PLEX_UP) {
+		gctl_error(req, "plex %s is not completely accessible",
+		    p->name);
+		goto out;
+	}
+
+	cp = p->consumer;
+	error = g_access(cp, 1, 1, 0);
+	if (error) {
+		gctl_error(req, "cannot access consumer");
+		goto out;
+	}
+	g_topology_unlock();
+
+	/* Reset the check pointer when using -f. */
+	if (*flags & GV_FLAG_F)
+		p->synced = 0;
+
+	bp = g_new_bio();
+	if (bp == NULL) {
+		gctl_error(req, "cannot create BIO - out of memory");
+		g_topology_lock();
+		error = g_access(cp, -1, -1, 0);
+		goto out;
+	}
+	bp->bio_cmd = BIO_WRITE;
+	bp->bio_done = NULL;
+	bp->bio_data = g_malloc(p->stripesize, M_WAITOK | M_ZERO);
+	bp->bio_cflags |= GV_BIO_CHECK;
+	if (*rebuild)
+		bp->bio_cflags |= GV_BIO_PARITY;
+	bp->bio_offset = p->synced;
+	bp->bio_length = p->stripesize;
+
+	/* Schedule it down ... */
+	g_io_request(bp, cp);
+
+	/* ... and wait for the result. */
+	error = biowait(bp, "gwrite");
+	g_free(bp->bio_data);
+	g_destroy_bio(bp);
+
+	if (error) {
+		/* Incorrect parity. */
+		if (error == EAGAIN)
+			rv = 1;
+
+		/* Some other error happened. */
+		else
+			gctl_error(req, "Parity check failed at offset 0x%jx, "
+			    "errno %d", (intmax_t)p->synced, error);
+
+	/* Correct parity. */
+	} else
+		rv = 0;
+
+	gctl_set_param(req, "offset", &p->synced, sizeof(p->synced));
+
+	/* Advance the checkpointer if there was no error. */
+	if (rv == 0)
+		p->synced += p->stripesize;
+
+	/* End of plex; reset the check pointer and signal it to the caller. */
+	if (p->synced >= p->size) {
+		p->synced = 0;
+		rv = -2;
+	}
+
+	g_topology_lock();
+	error = g_access(cp, -1, -1, 0);
+
+out:
+	gctl_set_param(req, "rv", &rv, sizeof(rv));
+}
+
 void
 gv_start_obj(struct g_geom *gp, struct gctl_req *req)
 {

sys/geom/vinum/geom_vinum_plex.c

@@ -46,6 +46,10 @@ __FBSDID("$FreeBSD$");
 static void gv_plex_completed_request(struct gv_plex *, struct bio *);
 static void gv_plex_normal_request(struct gv_plex *, struct bio *);
 static void gv_plex_worker(void *);
+static int gv_check_parity(struct gv_plex *, struct bio *,
+    struct gv_raid5_packet *);
+static int gv_normal_parity(struct gv_plex *, struct bio *,
+    struct gv_raid5_packet *);
 
 /* XXX: is this the place to catch dying subdisks? */
 static void
@@ -346,6 +350,85 @@ gv_plex_worker(void *arg)
 	kthread_exit(ENXIO);
 }
 
+static int
+gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
+{
+	struct bio *cbp, *pbp;
+	int finished, i;
+
+	finished = 1;
+
+	if (wp->waiting != NULL) {
+		pbp = wp->waiting;
+		wp->waiting = NULL;
+		cbp = wp->parity;
+		for (i = 0; i < wp->length; i++)
+			cbp->bio_data[i] ^= pbp->bio_data[i];
+		g_io_request(pbp, pbp->bio_caller2);
+		finished = 0;
+
+	} else if (wp->parity != NULL) {
+		cbp = wp->parity;
+		wp->parity = NULL;
+		g_io_request(cbp, cbp->bio_caller2);
+		finished = 0;
+	}
+
+	return (finished);
+}
+
+static int
+gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
+{
+	struct bio *cbp, *pbp;
+	int err, finished, i;
+
+	err = 0;
+	finished = 1;
+
+	if (wp->waiting != NULL) {
+		pbp = wp->waiting;
+		wp->waiting = NULL;
+		g_io_request(pbp, pbp->bio_caller2);
+		finished = 0;
+
+	} else if (wp->parity != NULL) {
+		cbp = wp->parity;
+		wp->parity = NULL;
+
+		/* Check if the parity is correct. */
+		for (i = 0; i < wp->length; i++) {
+			if (bp->bio_data[i] != cbp->bio_data[i]) {
+				err = 1;
+				break;
+			}
+		}
+
+		/* The parity is not correct... */
+		if (err) {
+			bp->bio_parent->bio_error = EAGAIN;
+
+			/* ... but we rebuild it. */
+			if (bp->bio_parent->bio_cflags & GV_BIO_PARITY) {
+				g_io_request(cbp, cbp->bio_caller2);
+				finished = 0;
+			}
+		}
+
+		/*
+		 * Clean up the BIO we would have used for rebuilding the
+		 * parity.
+		 */
+		if (finished) {
+			bp->bio_parent->bio_inbed++;
+			g_destroy_bio(cbp);
+		}
+
+	}
+
+	return (finished);
+}
+
 void
 gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
 {
@@ -405,18 +488,13 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
 
 		/* Handle parity data. */
 		if (TAILQ_EMPTY(&wp->bits)) {
-			if (wp->waiting != NULL) {
-				pbp = wp->waiting;
-				wp->waiting = NULL;
-				cbp = wp->parity;
-				for (i = 0; i < wp->length; i++)
-					cbp->bio_data[i] ^= pbp->bio_data[i];
-				g_io_request(pbp, pbp->bio_caller2);
-			} else if (wp->parity != NULL) {
-				cbp = wp->parity;
-				wp->parity = NULL;
-				g_io_request(cbp, cbp->bio_caller2);
-			} else {
+			if (bp->bio_parent->bio_cflags & GV_BIO_CHECK)
+				i = gv_check_parity(p, bp, wp);
+			else
+				i = gv_normal_parity(p, bp, wp);
+
+			/* All of our sub-requests have finished. */
+			if (i) {
 				bp->bio_parent->bio_completed += wp->length;
 				TAILQ_REMOVE(&p->packets, wp, list);
 				/* Bring the waiting bios back into the game. */
@@ -475,6 +553,9 @@ gv_plex_normal_request(struct gv_plex *p, struct bio *bp)
 			if (bp->bio_cflags & GV_BIO_REBUILD)
 				err = gv_rebuild_raid5(p, wp, bp, addr,
 				    boff, bcount);
+			else if (bp->bio_cflags & GV_BIO_CHECK)
+				err = gv_check_raid5(p, wp, bp, addr,
+				    boff, bcount);
 			else
 				err = gv_build_raid5_req(p, wp, bp, addr,
 				    boff, bcount);

sys/geom/vinum/geom_vinum_raid5.c

@@ -80,6 +80,101 @@ gv_stripe_active(struct gv_plex *p, struct bio *bp)
 	return (overlap);
 }
 
+int
+gv_check_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
+    caddr_t addr, off_t boff, off_t bcount)
+{
+	struct gv_sd *parity, *s;
+	struct gv_bioq *bq;
+	struct bio *cbp, *pbp;
+	int i, psdno;
+	off_t real_len, real_off;
+
+	if (p == NULL || LIST_EMPTY(&p->subdisks))
+		return (ENXIO);
+
+	gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, &psdno);
+
+	/* Find the right subdisk. */
+	parity = NULL;
+	i = 0;
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		if (i == psdno) {
+			parity = s;
+			break;
+		}
+		i++;
+	}
+
+	/* Parity stripe not found. */
+	if (parity == NULL)
+		return (ENXIO);
+
+	if (parity->state != GV_SD_UP)
+		return (ENXIO);
+
+	wp->length = real_len;
+	wp->data = addr;
+	wp->lockbase = real_off;
+
+	/* Read all subdisks. */
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		/* Skip the parity subdisk. */
+		if (s == parity)
+			continue;
+
+		cbp = g_clone_bio(bp);
+		if (cbp == NULL)
+			return (ENOMEM);
+		cbp->bio_cmd = BIO_READ;
+		cbp->bio_data = g_malloc(real_len, M_WAITOK);
+		cbp->bio_cflags |= GV_BIO_MALLOC;
+		cbp->bio_offset = real_off;
+		cbp->bio_length = real_len;
+		cbp->bio_done = gv_plex_done;
+		cbp->bio_caller2 = s->consumer;
+		cbp->bio_driver1 = wp;
+
+		GV_ENQUEUE(bp, cbp, pbp);
+
+		bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
+		bq->bp = cbp;
+		TAILQ_INSERT_TAIL(&wp->bits, bq, queue);
+	}
+
+	/* Read the parity data. */
+	cbp = g_clone_bio(bp);
+	if (cbp == NULL)
+		return (ENOMEM);
+	cbp->bio_cmd = BIO_READ;
+	cbp->bio_data = g_malloc(real_len, M_WAITOK | M_ZERO);
+	cbp->bio_cflags |= GV_BIO_MALLOC;
+	cbp->bio_offset = real_off;
+	cbp->bio_length = real_len;
+	cbp->bio_done = gv_plex_done;
+	cbp->bio_caller2 = parity->consumer;
+	cbp->bio_driver1 = wp;
+	wp->waiting = cbp;
+
+	/*
+	 * In case we want to rebuild the parity, create an extra BIO to write
+	 * it out.  It also acts as buffer for the XOR operations.
+	 */
+	cbp = g_clone_bio(bp);
+	if (cbp == NULL)
+		return (ENOMEM);
+	cbp->bio_data = addr;
+	cbp->bio_offset = real_off;
+	cbp->bio_length = real_len;
+	cbp->bio_done = gv_plex_done;
+	cbp->bio_caller2 = parity->consumer;
+	cbp->bio_driver1 = wp;
+	wp->parity = cbp;
+
+	return (0);
+}
+
+/* Rebuild a degraded RAID5 plex. */
 int
 gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
     caddr_t addr, off_t boff, off_t bcount)
@@ -101,7 +196,7 @@ gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
 			broken = s;
 	}
 
-	/* Parity stripe not found. */
+	/* Broken stripe not found. */
 	if (broken == NULL)
 		return (ENXIO);
 

sys/geom/vinum/geom_vinum_raid5.h

@@ -67,6 +67,8 @@ struct gv_raid5_packet {
 int	gv_stripe_active(struct gv_plex *, struct bio *);
 int	gv_build_raid5_req(struct gv_plex *, struct gv_raid5_packet *,
 	    struct bio *, caddr_t, off_t, off_t);
+int	gv_check_raid5(struct gv_plex *, struct gv_raid5_packet *,
+	    struct bio *, caddr_t, off_t, off_t);
 int	gv_rebuild_raid5(struct gv_plex *, struct gv_raid5_packet *,
 	    struct bio *, caddr_t, off_t, off_t);
 void	gv_raid5_worker(void *);

sys/geom/vinum/geom_vinum_var.h

@@ -114,6 +114,8 @@
 #define	GV_BIO_SYNCREQ	0x08
 #define	GV_BIO_SUCCEED	0x10
 #define	GV_BIO_REBUILD	0x20
+#define	GV_BIO_CHECK	0x40
+#define	GV_BIO_PARITY	0x80
 
 /*
  * hostname is 256 bytes long, but we don't need to shlep multiple copies in