[DLM] cancel in conversion deadlock [4/6]

When conversion deadlock is detected, cancel the conversion and return
EDEADLK to the application.  This is a new default behavior where before
the dlm would allow the deadlock to exist indefinately.

The DLM_LKF_NODLCKWT flag can now be used in a conversion to prevent the
dlm from performing conversion deadlock detection/cancelation on it.
The DLM_LKF_CONVDEADLK flag can continue to be used as before to tell the
dlm to demote the granted mode of the lock being converted if it gets into
a conversion deadlock.

Signed-off-by: David Teigland <teigland@redhat.com>
Signed-off-by: Steven Whitehouse <swhiteho@redhat.com>
This commit is contained in:
David Teigland 2007-05-18 09:01:26 -05:00 committed by Steven Whitehouse
parent d7db923ea4
commit c85d65e914
2 changed files with 138 additions and 63 deletions

View file

@ -1408,10 +1408,8 @@ static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb)
* queue for one resource. The granted mode of each lock blocks the requested
* mode of the other lock."
*
* Part 2: if the granted mode of lkb is preventing the first lkb in the
* convert queue from being granted, then demote lkb (set grmode to NL).
* This second form requires that we check for conv-deadlk even when
* now == 0 in _can_be_granted().
* Part 2: if the granted mode of lkb is preventing an earlier lkb in the
* convert queue from being granted, then deadlk/demote lkb.
*
* Example:
* Granted Queue: empty
@ -1420,41 +1418,52 @@ static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb)
*
* The first lock can't be granted because of the granted mode of the second
* lock and the second lock can't be granted because it's not first in the
* list. We demote the granted mode of the second lock (the lkb passed to this
* function).
* list. We either cancel lkb's conversion (PR->EX) and return EDEADLK, or we
* demote the granted mode of lkb (from PR to NL) if it has the CONVDEADLK
* flag set and return DEMOTED in the lksb flags.
*
* After the resolution, the "grant pending" function needs to go back and try
* to grant locks on the convert queue again since the first lock can now be
* granted.
* Originally, this function detected conv-deadlk in a more limited scope:
* - if !modes_compat(lkb1, lkb2) && !modes_compat(lkb2, lkb1), or
* - if lkb1 was the first entry in the queue (not just earlier), and was
* blocked by the granted mode of lkb2, and there was nothing on the
* granted queue preventing lkb1 from being granted immediately, i.e.
* lkb2 was the only thing preventing lkb1 from being granted.
*
* That second condition meant we'd only say there was conv-deadlk if
* resolving it (by demotion) would lead to the first lock on the convert
* queue being granted right away. It allowed conversion deadlocks to exist
* between locks on the convert queue while they couldn't be granted anyway.
*
* Now, we detect and take action on conversion deadlocks immediately when
* they're created, even if they may not be immediately consequential. If
* lkb1 exists anywhere in the convert queue and lkb2 comes in with a granted
* mode that would prevent lkb1's conversion from being granted, we do a
* deadlk/demote on lkb2 right away and don't let it onto the convert queue.
* I think this means that the lkb_is_ahead condition below should always
* be zero, i.e. there will never be conv-deadlk between two locks that are
* both already on the convert queue.
*/
static int conversion_deadlock_detect(struct dlm_rsb *rsb, struct dlm_lkb *lkb)
static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2)
{
struct dlm_lkb *this, *first = NULL, *self = NULL;
struct dlm_lkb *lkb1;
int lkb_is_ahead = 0;
list_for_each_entry(this, &rsb->res_convertqueue, lkb_statequeue) {
if (!first)
first = this;
if (this == lkb) {
self = lkb;
list_for_each_entry(lkb1, &r->res_convertqueue, lkb_statequeue) {
if (lkb1 == lkb2) {
lkb_is_ahead = 1;
continue;
}
if (!modes_compat(this, lkb) && !modes_compat(lkb, this))
return 1;
if (!lkb_is_ahead) {
if (!modes_compat(lkb2, lkb1))
return 1;
} else {
if (!modes_compat(lkb2, lkb1) &&
!modes_compat(lkb1, lkb2))
return 1;
}
}
/* if lkb is on the convert queue and is preventing the first
from being granted, then there's deadlock and we demote lkb.
multiple converting locks may need to do this before the first
converting lock can be granted. */
if (self && self != first) {
if (!modes_compat(lkb, first) &&
!queue_conflict(&rsb->res_grantqueue, first))
return 1;
}
return 0;
}
@ -1583,42 +1592,57 @@ static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now)
if (!now && !conv && list_empty(&r->res_convertqueue) &&
first_in_list(lkb, &r->res_waitqueue))
return 1;
out:
/*
* The following, enabled by CONVDEADLK, departs from VMS.
*/
if (conv && (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) &&
conversion_deadlock_detect(r, lkb)) {
lkb->lkb_grmode = DLM_LOCK_NL;
lkb->lkb_sbflags |= DLM_SBF_DEMOTED;
}
return 0;
}
/*
* The ALTPR and ALTCW flags aren't traditional lock manager flags, but are a
* simple way to provide a big optimization to applications that can use them.
*/
static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now)
static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now,
int *err)
{
uint32_t flags = lkb->lkb_exflags;
int rv;
int8_t alt = 0, rqmode = lkb->lkb_rqmode;
int8_t is_convert = (lkb->lkb_grmode != DLM_LOCK_IV);
if (err)
*err = 0;
rv = _can_be_granted(r, lkb, now);
if (rv)
goto out;
if (lkb->lkb_sbflags & DLM_SBF_DEMOTED)
goto out;
/*
* The CONVDEADLK flag is non-standard and tells the dlm to resolve
* conversion deadlocks by demoting grmode to NL, otherwise the dlm
* cancels one of the locks.
*/
if (rqmode != DLM_LOCK_PR && flags & DLM_LKF_ALTPR)
if (is_convert && can_be_queued(lkb) &&
conversion_deadlock_detect(r, lkb)) {
if (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) {
lkb->lkb_grmode = DLM_LOCK_NL;
lkb->lkb_sbflags |= DLM_SBF_DEMOTED;
} else if (!(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) {
if (err)
*err = -EDEADLK;
else {
log_print("can_be_granted deadlock %x now %d",
lkb->lkb_id, now);
dlm_dump_rsb(r);
}
}
goto out;
}
/*
* The ALTPR and ALTCW flags are non-standard and tell the dlm to try
* to grant a request in a mode other than the normal rqmode. It's a
* simple way to provide a big optimization to applications that can
* use them.
*/
if (rqmode != DLM_LOCK_PR && (lkb->lkb_exflags & DLM_LKF_ALTPR))
alt = DLM_LOCK_PR;
else if (rqmode != DLM_LOCK_CW && flags & DLM_LKF_ALTCW)
else if (rqmode != DLM_LOCK_CW && (lkb->lkb_exflags & DLM_LKF_ALTCW))
alt = DLM_LOCK_CW;
if (alt) {
@ -1633,10 +1657,20 @@ static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now)
return rv;
}
/* FIXME: I don't think that can_be_granted() can/will demote or find deadlock
for locks pending on the convert list. Once verified (watch for these
log_prints), we should be able to just call _can_be_granted() and not
bother with the demote/deadlk cases here (and there's no easy way to deal
with a deadlk here, we'd have to generate something like grant_lock with
the deadlk error.) */
/* returns the highest requested mode of all blocked conversions */
static int grant_pending_convert(struct dlm_rsb *r, int high)
{
struct dlm_lkb *lkb, *s;
int hi, demoted, quit, grant_restart, demote_restart;
int deadlk;
quit = 0;
restart:
@ -1646,14 +1680,29 @@ static int grant_pending_convert(struct dlm_rsb *r, int high)
list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) {
demoted = is_demoted(lkb);
if (can_be_granted(r, lkb, 0)) {
deadlk = 0;
if (can_be_granted(r, lkb, 0, &deadlk)) {
grant_lock_pending(r, lkb);
grant_restart = 1;
} else {
hi = max_t(int, lkb->lkb_rqmode, hi);
if (!demoted && is_demoted(lkb))
demote_restart = 1;
continue;
}
if (!demoted && is_demoted(lkb)) {
log_print("WARN: pending demoted %x node %d %s",
lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
demote_restart = 1;
continue;
}
if (deadlk) {
log_print("WARN: pending deadlock %x node %d %s",
lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
dlm_dump_rsb(r);
continue;
}
hi = max_t(int, lkb->lkb_rqmode, hi);
}
if (grant_restart)
@ -1671,7 +1720,7 @@ static int grant_pending_wait(struct dlm_rsb *r, int high)
struct dlm_lkb *lkb, *s;
list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) {
if (can_be_granted(r, lkb, 0))
if (can_be_granted(r, lkb, 0, NULL))
grant_lock_pending(r, lkb);
else
high = max_t(int, lkb->lkb_rqmode, high);
@ -2121,7 +2170,7 @@ static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error = 0;
if (can_be_granted(r, lkb, 1)) {
if (can_be_granted(r, lkb, 1, NULL)) {
grant_lock(r, lkb);
queue_cast(r, lkb, 0);
goto out;
@ -2147,16 +2196,32 @@ static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error = 0;
int deadlk = 0;
/* changing an existing lock may allow others to be granted */
if (can_be_granted(r, lkb, 1)) {
if (can_be_granted(r, lkb, 1, &deadlk)) {
grant_lock(r, lkb);
queue_cast(r, lkb, 0);
grant_pending_locks(r);
goto out;
}
/* can_be_granted() detected that this lock would block in a conversion
deadlock, so we leave it on the granted queue and return EDEADLK in
the ast for the convert. */
if (deadlk) {
/* it's left on the granted queue */
log_debug(r->res_ls, "deadlock %x node %d sts%d g%d r%d %s",
lkb->lkb_id, lkb->lkb_nodeid, lkb->lkb_status,
lkb->lkb_grmode, lkb->lkb_rqmode, r->res_name);
revert_lock(r, lkb);
queue_cast(r, lkb, -EDEADLK);
error = -EDEADLK;
goto out;
}
/* is_demoted() means the can_be_granted() above set the grmode
to NL, and left us on the granted queue. This auto-demotion
(due to CONVDEADLK) might mean other locks, and/or this lock, are
@ -2438,7 +2503,7 @@ int dlm_lock(dlm_lockspace_t *lockspace,
out_put:
if (convert || error)
__put_lkb(ls, lkb);
if (error == -EAGAIN)
if (error == -EAGAIN || error == -EDEADLK)
error = 0;
out:
dlm_unlock_recovery(ls);
@ -3312,6 +3377,12 @@ static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
queue_cast(r, lkb, -EAGAIN);
break;
case -EDEADLK:
receive_flags_reply(lkb, ms);
revert_lock_pc(r, lkb);
queue_cast(r, lkb, -EDEADLK);
break;
case -EINPROGRESS:
/* convert was queued on remote master */
receive_flags_reply(lkb, ms);
@ -4284,7 +4355,7 @@ int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
error = convert_lock(ls, lkb, &args);
if (error == -EINPROGRESS || error == -EAGAIN)
if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK)
error = 0;
out_put:
dlm_put_lkb(lkb);

View file

@ -85,7 +85,11 @@
* Only relevant to locks originating in userspace. A persistent lock will not
* be removed if the process holding the lock exits.
*
* DLM_LKF_NODLKWT
* DLM_LKF_NODLCKWT
*
* Do not cancel the lock if it gets into conversion deadlock.
* Exclude this lock from being monitored due to DLM_LSFL_TIMEWARN.
*
* DLM_LKF_NODLCKBLK
*
* net yet implemented