From 3c6decc3279a5ca59eb56893dbc03df2ec48e0c6 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Fri, 6 Jan 2006 18:07:32 +0000 Subject: [PATCH] Trim another pointer from struct lock_object (and thus from struct mtx and struct sx). Instead of storing a direct pointer to a our lock_class struct in lock_object, reserve 4 bits in the lo_flags field to serve as an index into a global lock_classes array that contains pointers to the lock classes. Only debugging code such as WITNESS or INVARIANTS checks and KTR logging need to access the lock_class member, so this shouldn't add any overhead to production kernels. It might add some slight overhead to kernels using those debug options however. As with the previous set of changes to lock_object, this is going to completely obliterate the kernel ABI, so be sure to recompile all your modules. --- sys/kern/kern_mutex.c | 43 +++++++++++++++++++----------- sys/kern/kern_sx.c | 4 +-- sys/kern/subr_witness.c | 58 +++++++++++++++++++++++------------------ sys/sys/_lock.h | 1 - sys/sys/lock.h | 22 +++++++++++++--- 5 files changed, 82 insertions(+), 46 deletions(-) diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 9f79ab40d507..9bfe31f18f64 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -278,7 +278,7 @@ _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep, + KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX, ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, @@ -303,7 +303,7 @@ _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep, + KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX, ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); @@ -382,7 +382,7 @@ _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin, + KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SPIN_MUTEX, ("mtx_lock_spin() of sleep mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, @@ -398,7 +398,7 @@ _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin, + KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SPIN_MUTEX, ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); @@ -419,7 +419,7 @@ _mtx_trylock(struct mtx *m, int opts, const char *file, int line) int rval; MPASS(curthread != NULL); - KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep, + KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX, ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); @@ -845,13 +845,13 @@ mtx_init(struct mtx *m, const char *name, const char *type, int opts) ("mutex \"%s\" %p already initialized", name, m)); bzero(m, sizeof(*m)); if (opts & MTX_SPIN) - lock->lo_class = &lock_class_mtx_spin; + lock->lo_flags = LOCK_CLASS_SPIN_MUTEX << LO_CLASSSHIFT; else - lock->lo_class = &lock_class_mtx_sleep; + lock->lo_flags = LOCK_CLASS_SLEEP_MUTEX << LO_CLASSSHIFT; lock->lo_name = name; lock->lo_type = type != NULL ? type : name; if (opts & MTX_QUIET) - lock->lo_flags = LO_QUIET; + lock->lo_flags |= LO_QUIET; if (opts & MTX_RECURSE) lock->lo_flags |= LO_RECURSABLE; if ((opts & MTX_NOWITNESS) == 0) @@ -883,6 +883,10 @@ mtx_destroy(struct mtx *m) else { MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); + /* Perform the non-mtx related part of mtx_unlock_spin(). */ + if (m->mtx_object.lo_class == &lock_class_mtx_spin) + spinlock_exit(); + /* Tell witness this isn't locked to make it happy. */ WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, __LINE__); @@ -916,26 +920,35 @@ mutex_init(void) mtx_lock(&Giant); } +#if LOCK_DEBUG > 0 || defined(DDB) +/* XXX: This is not mutex-specific. */ +struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = { + &lock_class_mtx_spin, + &lock_class_mtx_sleep, + &lock_class_sx, +}; +#endif + #ifdef DDB /* XXX: This function is not mutex-specific. */ DB_SHOW_COMMAND(lock, db_show_lock) { struct lock_object *lock; + struct lock_class *class; if (!have_addr) return; lock = (struct lock_object *)addr; - if (lock->lo_class != &lock_class_mtx_sleep && - lock->lo_class != &lock_class_mtx_spin && - lock->lo_class != &lock_class_sx) { - db_printf("Unknown lock class\n"); + if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) { + db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock)); return; } - db_printf(" class: %s\n", lock->lo_class->lc_name); + class = LOCK_CLASS(lock); + db_printf(" class: %s\n", class->lc_name); db_printf(" name: %s\n", lock->lo_name); if (lock->lo_type && lock->lo_type != lock->lo_name) db_printf(" type: %s\n", lock->lo_type); - lock->lo_class->lc_ddb_show(lock); + class->lc_ddb_show(lock); } void @@ -947,7 +960,7 @@ db_show_mtx(struct lock_object *lock) m = (struct mtx *)lock; db_printf(" flags: {"); - if (m->mtx_object.lo_class == &lock_class_mtx_spin) + if (LO_CLASSINDEX(lock) == LOCK_CLASS_SPIN_MUTEX) db_printf("SPIN"); else db_printf("DEF"); diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c index 7a14de0f89d0..d24e4de31229 100644 --- a/sys/kern/kern_sx.c +++ b/sys/kern/kern_sx.c @@ -83,9 +83,9 @@ sx_init(struct sx *sx, const char *description) KASSERT((lock->lo_flags & LO_INITIALIZED) == 0, ("sx lock %s %p already initialized", description, sx)); bzero(sx, sizeof(*sx)); - lock->lo_class = &lock_class_sx; + lock->lo_flags = LOCK_CLASS_SX << LO_CLASSSHIFT; lock->lo_type = lock->lo_name = description; - lock->lo_flags = LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | + lock->lo_flags |= LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE; sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx); sx->sx_cnt = 0; diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 55e45e572288..db10e8fdb4f9 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -506,7 +506,7 @@ witness_initialize(void *dummy __unused) KASSERT(lock->lo_flags & LO_WITNESS, ("%s: lock %s is on pending list but not LO_WITNESS", __func__, lock->lo_name)); - lock->lo_witness = enroll(lock->lo_type, lock->lo_class); + lock->lo_witness = enroll(lock->lo_type, LOCK_CLASS(lock)); } /* Mark the witness code as being ready for use. */ @@ -542,7 +542,7 @@ witness_init(struct lock_object *lock) struct lock_class *class; /* Various sanity checks. */ - class = lock->lo_class; + class = LOCK_CLASS(lock); if (lock->lo_flags & LO_INITIALIZED) panic("%s: lock (%s) %s is already initialized", __func__, class->lc_name, lock->lo_name); @@ -580,14 +580,16 @@ witness_init(struct lock_object *lock) void witness_destroy(struct lock_object *lock) { + struct lock_class *class; struct witness *w; + class = LOCK_CLASS(lock); if (witness_cold) panic("lock (%s) %s destroyed while witness_cold", - lock->lo_class->lc_name, lock->lo_name); + class->lc_name, lock->lo_name); if ((lock->lo_flags & LO_INITIALIZED) == 0) panic("%s: lock (%s) %s is not initialized", __func__, - lock->lo_class->lc_name, lock->lo_name); + class->lc_name, lock->lo_name); /* XXX: need to verify that no one holds the lock */ if ((lock->lo_flags & (LO_WITNESS | LO_ENROLLPEND)) == LO_WITNESS && @@ -824,7 +826,7 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, __func__); w = lock->lo_witness; - class = lock->lo_class; + class = LOCK_CLASS(lock); td = curthread; file = fixup_filename(file); @@ -1104,7 +1106,7 @@ witness_lock(struct lock_object *lock, int flags, const char *file, int line) file = fixup_filename(file); /* Determine lock list for this lock. */ - if (lock->lo_class->lc_flags & LC_SLEEPLOCK) + if (LOCK_CLASS(lock)->lc_flags & LC_SLEEPLOCK) lock_list = &td->td_sleeplocks; else lock_list = PCPU_PTR(spinlocks); @@ -1157,7 +1159,7 @@ witness_upgrade(struct lock_object *lock, int flags, const char *file, int line) KASSERT(!witness_cold, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) return; - class = lock->lo_class; + class = LOCK_CLASS(lock); file = fixup_filename(file); if ((lock->lo_flags & LO_UPGRADABLE) == 0) panic("upgrade of non-upgradable lock (%s) %s @ %s:%d", @@ -1165,7 +1167,7 @@ witness_upgrade(struct lock_object *lock, int flags, const char *file, int line) if ((flags & LOP_TRYLOCK) == 0) panic("non-try upgrade of lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, file, line); - if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0) + if ((class->lc_flags & LC_SLEEPLOCK) == 0) panic("upgrade of non-sleep lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, file, line); instance = find_instance(curthread->td_sleeplocks, lock); @@ -1192,12 +1194,12 @@ witness_downgrade(struct lock_object *lock, int flags, const char *file, KASSERT(!witness_cold, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) return; - class = lock->lo_class; + class = LOCK_CLASS(lock); file = fixup_filename(file); if ((lock->lo_flags & LO_UPGRADABLE) == 0) panic("downgrade of non-upgradable lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, file, line); - if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0) + if ((class->lc_flags & LC_SLEEPLOCK) == 0) panic("downgrade of non-sleep lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, file, line); instance = find_instance(curthread->td_sleeplocks, lock); @@ -1228,7 +1230,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line) panicstr != NULL) return; td = curthread; - class = lock->lo_class; + class = LOCK_CLASS(lock); file = fixup_filename(file); /* Find lock instance associated with this lock. */ @@ -1739,7 +1741,7 @@ witness_list_lock(struct lock_instance *instance) lock = instance->li_lock; printf("%s %s %s", (instance->li_flags & LI_EXCLUSIVE) != 0 ? - "exclusive" : "shared", lock->lo_class->lc_name, lock->lo_name); + "exclusive" : "shared", LOCK_CLASS(lock)->lc_name, lock->lo_name); if (lock->lo_type != lock->lo_name) printf(" (%s)", lock->lo_type); printf(" r = %d (%p) locked @ %s:%d\n", @@ -1809,11 +1811,13 @@ witness_save(struct lock_object *lock, const char **filep, int *linep) { struct lock_list_entry *lock_list; struct lock_instance *instance; + struct lock_class *class; KASSERT(!witness_cold, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) return; - if (lock->lo_class->lc_flags & LC_SLEEPLOCK) + class = LOCK_CLASS(lock); + if (class->lc_flags & LC_SLEEPLOCK) lock_list = curthread->td_sleeplocks; else { if (witness_skipspin) @@ -1823,7 +1827,7 @@ witness_save(struct lock_object *lock, const char **filep, int *linep) instance = find_instance(lock_list, lock); if (instance == NULL) panic("%s: lock (%s) %s not locked", __func__, - lock->lo_class->lc_name, lock->lo_name); + class->lc_name, lock->lo_name); *filep = instance->li_file; *linep = instance->li_line; } @@ -1833,11 +1837,13 @@ witness_restore(struct lock_object *lock, const char *file, int line) { struct lock_list_entry *lock_list; struct lock_instance *instance; + struct lock_class *class; KASSERT(!witness_cold, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) return; - if (lock->lo_class->lc_flags & LC_SLEEPLOCK) + class = LOCK_CLASS(lock); + if (class->lc_flags & LC_SLEEPLOCK) lock_list = curthread->td_sleeplocks; else { if (witness_skipspin) @@ -1847,7 +1853,7 @@ witness_restore(struct lock_object *lock, const char *file, int line) instance = find_instance(lock_list, lock); if (instance == NULL) panic("%s: lock (%s) %s not locked", __func__, - lock->lo_class->lc_name, lock->lo_name); + class->lc_name, lock->lo_name); lock->lo_witness->w_file = file; lock->lo_witness->w_line = line; instance->li_file = file; @@ -1859,23 +1865,25 @@ witness_assert(struct lock_object *lock, int flags, const char *file, int line) { #ifdef INVARIANT_SUPPORT struct lock_instance *instance; + struct lock_class *class; if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL) return; - if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) != 0) + class = LOCK_CLASS(lock); + if ((class->lc_flags & LC_SLEEPLOCK) != 0) instance = find_instance(curthread->td_sleeplocks, lock); - else if ((lock->lo_class->lc_flags & LC_SPINLOCK) != 0) + else if ((class->lc_flags & LC_SPINLOCK) != 0) instance = find_instance(PCPU_GET(spinlocks), lock); else { panic("Lock (%s) %s is not sleep or spin!", - lock->lo_class->lc_name, lock->lo_name); + class->lc_name, lock->lo_name); } file = fixup_filename(file); switch (flags) { case LA_UNLOCKED: if (instance != NULL) panic("Lock (%s) %s locked @ %s:%d.", - lock->lo_class->lc_name, lock->lo_name, file, line); + class->lc_name, lock->lo_name, file, line); break; case LA_LOCKED: case LA_LOCKED | LA_RECURSED: @@ -1888,25 +1896,25 @@ witness_assert(struct lock_object *lock, int flags, const char *file, int line) case LA_XLOCKED | LA_NOTRECURSED: if (instance == NULL) { panic("Lock (%s) %s not locked @ %s:%d.", - lock->lo_class->lc_name, lock->lo_name, file, line); + class->lc_name, lock->lo_name, file, line); break; } if ((flags & LA_XLOCKED) != 0 && (instance->li_flags & LI_EXCLUSIVE) == 0) panic("Lock (%s) %s not exclusively locked @ %s:%d.", - lock->lo_class->lc_name, lock->lo_name, file, line); + class->lc_name, lock->lo_name, file, line); if ((flags & LA_SLOCKED) != 0 && (instance->li_flags & LI_EXCLUSIVE) != 0) panic("Lock (%s) %s exclusively locked @ %s:%d.", - lock->lo_class->lc_name, lock->lo_name, file, line); + class->lc_name, lock->lo_name, file, line); if ((flags & LA_RECURSED) != 0 && (instance->li_flags & LI_RECURSEMASK) == 0) panic("Lock (%s) %s not recursed @ %s:%d.", - lock->lo_class->lc_name, lock->lo_name, file, line); + class->lc_name, lock->lo_name, file, line); if ((flags & LA_NOTRECURSED) != 0 && (instance->li_flags & LI_RECURSEMASK) != 0) panic("Lock (%s) %s recursed @ %s:%d.", - lock->lo_class->lc_name, lock->lo_name, file, line); + class->lc_name, lock->lo_name, file, line); break; default: panic("Invalid lock assertion at %s:%d.", file, line); diff --git a/sys/sys/_lock.h b/sys/sys/_lock.h index c9da773f0fed..dd0b0715d426 100644 --- a/sys/sys/_lock.h +++ b/sys/sys/_lock.h @@ -32,7 +32,6 @@ #define _SYS__LOCK_H_ struct lock_object { - struct lock_class *lo_class; const char *lo_name; /* Individual lock name. */ const char *lo_type; /* General lock type. */ u_int lo_flags; diff --git a/sys/sys/lock.h b/sys/sys/lock.h index 31b2269088fe..a6f9080918d5 100644 --- a/sys/sys/lock.h +++ b/sys/sys/lock.h @@ -68,6 +68,20 @@ struct lock_class { #define LO_UPGRADABLE 0x00200000 /* Lock may be upgraded/downgraded. */ #define LO_DUPOK 0x00400000 /* Don't check for duplicate acquires */ #define LO_ENROLLPEND 0x00800000 /* On the pending enroll list. */ +#define LO_CLASSMASK 0x0f000000 /* Class index bitmask. */ + +/* + * Lock classes are statically assigned an index into the gobal lock_classes + * array. Debugging code looks up the lock class for a given lock object + * by indexing the array. + */ +#define LO_CLASSSHIFT 24 +#define LO_CLASSINDEX(lock) ((((lock)->lo_flags) & LO_CLASSMASK) >> LO_CLASSSHIFT) +#define LOCK_CLASS(lock) (lock_classes[LO_CLASSINDEX((lock))]) +#define LOCK_CLASS_SPIN_MUTEX 0 +#define LOCK_CLASS_SLEEP_MUTEX 1 +#define LOCK_CLASS_SX 2 +#define LOCK_CLASS_MAX LOCK_CLASS_SX #define LI_RECURSEMASK 0x0000ffff /* Recursion depth of lock instance. */ #define LI_EXCLUSIVE 0x00010000 /* Exclusive lock instance. */ @@ -166,21 +180,21 @@ struct lock_list_entry { #define LOCK_LOG_LOCK(opname, lo, flags, recurse, file, line) do { \ if (LOCK_LOG_TEST((lo), (flags))) \ CTR5(KTR_LOCK, opname " (%s) %s r = %d at %s:%d", \ - (lo)->lo_class->lc_name, (lo)->lo_name, \ + LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \ (u_int)(recurse), (file), (line)); \ } while (0) #define LOCK_LOG_TRY(opname, lo, flags, result, file, line) do { \ if (LOCK_LOG_TEST((lo), (flags))) \ CTR5(KTR_LOCK, "TRY_" opname " (%s) %s result=%d at %s:%d",\ - (lo)->lo_class->lc_name, (lo)->lo_name, \ + LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \ (u_int)(result), (file), (line)); \ } while (0) #define LOCK_LOG_INIT(lo, flags) do { \ if (LOCK_LOG_TEST((lo), (flags))) \ CTR4(KTR_LOCK, "%s: %p (%s) %s", __func__, (lo), \ - (lo)->lo_class->lc_name, (lo)->lo_name); \ + LOCK_CLASS(lo)->lc_name, (lo)->lo_name); \ } while (0) #define LOCK_LOG_DESTROY(lo, flags) LOCK_LOG_INIT(lo, flags) @@ -199,6 +213,8 @@ extern struct lock_class lock_class_mtx_sleep; extern struct lock_class lock_class_mtx_spin; extern struct lock_class lock_class_sx; +extern struct lock_class *lock_classes[]; + void spinlock_enter(void); void spinlock_exit(void); void witness_init(struct lock_object *);