2011-01-17 16:08:14 +00:00
|
|
|
/*
|
|
|
|
* QEMU coroutines
|
|
|
|
*
|
|
|
|
* Copyright IBM, Corp. 2011
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
|
|
|
|
* Kevin Wolf <kwolf@redhat.com>
|
|
|
|
*
|
|
|
|
* This work is licensed under the terms of the GNU LGPL, version 2 or later.
|
|
|
|
* See the COPYING.LIB file in the top-level directory.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "trace.h"
|
|
|
|
#include "qemu-common.h"
|
2013-05-17 13:51:25 +00:00
|
|
|
#include "qemu/thread.h"
|
coroutine: rewrite pool to avoid mutex
This patch removes the mutex by using fancy lock-free manipulation of
the pool. Lock-free stacks and queues are not hard, but they can suffer
from the ABA problem so they are better avoided unless you have some
deferred reclamation scheme like RCU. Otherwise you have to stick
with adding to a list, and emptying it completely. This is what this
patch does, by coupling a lock-free global list of available coroutines
with per-CPU lists that are actually used on coroutine creation.
Whenever the destruction pool is big enough, the next thread that runs
out of coroutines will steal the whole destruction pool. This is positive
in two ways:
1) the allocation does not have to do any atomic operation in the fast
path, it's entirely using thread-local storage. Once every POOL_BATCH_SIZE
allocations it will do a single atomic_xchg. Release does an atomic_cmpxchg
loop, that hopefully doesn't cause any starvation, and an atomic_inc.
A later patch will also remove atomic operations from the release path,
and try to avoid the atomic_xchg altogether---succeeding in doing so if
all devices either use ioeventfd or are not submitting requests actively.
2) in theory this should be completely adaptive. The number of coroutines
around should be a little more than POOL_BATCH_SIZE * number of allocating
threads; so this also empties qemu_coroutine_adjust_pool_size. (The previous
pool size was POOL_BATCH_SIZE * number of block backends, so it was a bit
more generous. But if you actually have many high-iodepth disks, it's better
to put them in different iothreads, which will also use separate thread
pools and aio=native file descriptors).
This speeds up perf/cost (in tests/test-coroutine) by a factor of ~1.33.
No matter if we end with some kind of coroutine bypass scheme or not,
it cannot hurt to optimize hot code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
Message-id: 1417518350-6167-6-git-send-email-pbonzini@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
2014-12-02 11:05:48 +00:00
|
|
|
#include "qemu/atomic.h"
|
2012-12-17 17:19:44 +00:00
|
|
|
#include "block/coroutine.h"
|
|
|
|
#include "block/coroutine_int.h"
|
2011-01-17 16:08:14 +00:00
|
|
|
|
2013-02-19 10:59:09 +00:00
|
|
|
enum {
|
coroutine: rewrite pool to avoid mutex
This patch removes the mutex by using fancy lock-free manipulation of
the pool. Lock-free stacks and queues are not hard, but they can suffer
from the ABA problem so they are better avoided unless you have some
deferred reclamation scheme like RCU. Otherwise you have to stick
with adding to a list, and emptying it completely. This is what this
patch does, by coupling a lock-free global list of available coroutines
with per-CPU lists that are actually used on coroutine creation.
Whenever the destruction pool is big enough, the next thread that runs
out of coroutines will steal the whole destruction pool. This is positive
in two ways:
1) the allocation does not have to do any atomic operation in the fast
path, it's entirely using thread-local storage. Once every POOL_BATCH_SIZE
allocations it will do a single atomic_xchg. Release does an atomic_cmpxchg
loop, that hopefully doesn't cause any starvation, and an atomic_inc.
A later patch will also remove atomic operations from the release path,
and try to avoid the atomic_xchg altogether---succeeding in doing so if
all devices either use ioeventfd or are not submitting requests actively.
2) in theory this should be completely adaptive. The number of coroutines
around should be a little more than POOL_BATCH_SIZE * number of allocating
threads; so this also empties qemu_coroutine_adjust_pool_size. (The previous
pool size was POOL_BATCH_SIZE * number of block backends, so it was a bit
more generous. But if you actually have many high-iodepth disks, it's better
to put them in different iothreads, which will also use separate thread
pools and aio=native file descriptors).
This speeds up perf/cost (in tests/test-coroutine) by a factor of ~1.33.
No matter if we end with some kind of coroutine bypass scheme or not,
it cannot hurt to optimize hot code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
Message-id: 1417518350-6167-6-git-send-email-pbonzini@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
2014-12-02 11:05:48 +00:00
|
|
|
POOL_BATCH_SIZE = 64,
|
2013-02-19 10:59:09 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/** Free list to speed up creation */
|
coroutine: rewrite pool to avoid mutex
This patch removes the mutex by using fancy lock-free manipulation of
the pool. Lock-free stacks and queues are not hard, but they can suffer
from the ABA problem so they are better avoided unless you have some
deferred reclamation scheme like RCU. Otherwise you have to stick
with adding to a list, and emptying it completely. This is what this
patch does, by coupling a lock-free global list of available coroutines
with per-CPU lists that are actually used on coroutine creation.
Whenever the destruction pool is big enough, the next thread that runs
out of coroutines will steal the whole destruction pool. This is positive
in two ways:
1) the allocation does not have to do any atomic operation in the fast
path, it's entirely using thread-local storage. Once every POOL_BATCH_SIZE
allocations it will do a single atomic_xchg. Release does an atomic_cmpxchg
loop, that hopefully doesn't cause any starvation, and an atomic_inc.
A later patch will also remove atomic operations from the release path,
and try to avoid the atomic_xchg altogether---succeeding in doing so if
all devices either use ioeventfd or are not submitting requests actively.
2) in theory this should be completely adaptive. The number of coroutines
around should be a little more than POOL_BATCH_SIZE * number of allocating
threads; so this also empties qemu_coroutine_adjust_pool_size. (The previous
pool size was POOL_BATCH_SIZE * number of block backends, so it was a bit
more generous. But if you actually have many high-iodepth disks, it's better
to put them in different iothreads, which will also use separate thread
pools and aio=native file descriptors).
This speeds up perf/cost (in tests/test-coroutine) by a factor of ~1.33.
No matter if we end with some kind of coroutine bypass scheme or not,
it cannot hurt to optimize hot code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
Message-id: 1417518350-6167-6-git-send-email-pbonzini@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
2014-12-02 11:05:48 +00:00
|
|
|
static QSLIST_HEAD(, Coroutine) release_pool = QSLIST_HEAD_INITIALIZER(pool);
|
|
|
|
static unsigned int release_pool_size;
|
|
|
|
static __thread QSLIST_HEAD(, Coroutine) alloc_pool = QSLIST_HEAD_INITIALIZER(pool);
|
|
|
|
static __thread Notifier coroutine_pool_cleanup_notifier;
|
|
|
|
|
|
|
|
static void coroutine_pool_cleanup(Notifier *n, void *value)
|
|
|
|
{
|
|
|
|
Coroutine *co;
|
|
|
|
Coroutine *tmp;
|
|
|
|
|
|
|
|
QSLIST_FOREACH_SAFE(co, &alloc_pool, pool_next, tmp) {
|
|
|
|
QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
|
|
|
|
qemu_coroutine_delete(co);
|
|
|
|
}
|
|
|
|
}
|
2013-02-19 10:59:09 +00:00
|
|
|
|
2011-01-17 16:08:14 +00:00
|
|
|
Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
|
|
|
|
{
|
2013-09-11 14:42:35 +00:00
|
|
|
Coroutine *co = NULL;
|
2013-02-19 10:59:09 +00:00
|
|
|
|
2013-09-11 14:42:35 +00:00
|
|
|
if (CONFIG_COROUTINE_POOL) {
|
coroutine: rewrite pool to avoid mutex
This patch removes the mutex by using fancy lock-free manipulation of
the pool. Lock-free stacks and queues are not hard, but they can suffer
from the ABA problem so they are better avoided unless you have some
deferred reclamation scheme like RCU. Otherwise you have to stick
with adding to a list, and emptying it completely. This is what this
patch does, by coupling a lock-free global list of available coroutines
with per-CPU lists that are actually used on coroutine creation.
Whenever the destruction pool is big enough, the next thread that runs
out of coroutines will steal the whole destruction pool. This is positive
in two ways:
1) the allocation does not have to do any atomic operation in the fast
path, it's entirely using thread-local storage. Once every POOL_BATCH_SIZE
allocations it will do a single atomic_xchg. Release does an atomic_cmpxchg
loop, that hopefully doesn't cause any starvation, and an atomic_inc.
A later patch will also remove atomic operations from the release path,
and try to avoid the atomic_xchg altogether---succeeding in doing so if
all devices either use ioeventfd or are not submitting requests actively.
2) in theory this should be completely adaptive. The number of coroutines
around should be a little more than POOL_BATCH_SIZE * number of allocating
threads; so this also empties qemu_coroutine_adjust_pool_size. (The previous
pool size was POOL_BATCH_SIZE * number of block backends, so it was a bit
more generous. But if you actually have many high-iodepth disks, it's better
to put them in different iothreads, which will also use separate thread
pools and aio=native file descriptors).
This speeds up perf/cost (in tests/test-coroutine) by a factor of ~1.33.
No matter if we end with some kind of coroutine bypass scheme or not,
it cannot hurt to optimize hot code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
Message-id: 1417518350-6167-6-git-send-email-pbonzini@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
2014-12-02 11:05:48 +00:00
|
|
|
co = QSLIST_FIRST(&alloc_pool);
|
|
|
|
if (!co) {
|
|
|
|
if (release_pool_size > POOL_BATCH_SIZE) {
|
|
|
|
/* Slow path; a good place to register the destructor, too. */
|
|
|
|
if (!coroutine_pool_cleanup_notifier.notify) {
|
|
|
|
coroutine_pool_cleanup_notifier.notify = coroutine_pool_cleanup;
|
|
|
|
qemu_thread_atexit_add(&coroutine_pool_cleanup_notifier);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* This is not exact; there could be a little skew between
|
|
|
|
* release_pool_size and the actual size of release_pool. But
|
|
|
|
* it is just a heuristic, it does not need to be perfect.
|
|
|
|
*/
|
|
|
|
release_pool_size = 0;
|
|
|
|
QSLIST_MOVE_ATOMIC(&alloc_pool, &release_pool);
|
|
|
|
co = QSLIST_FIRST(&alloc_pool);
|
|
|
|
}
|
|
|
|
}
|
2013-09-11 14:42:35 +00:00
|
|
|
if (co) {
|
coroutine: rewrite pool to avoid mutex
This patch removes the mutex by using fancy lock-free manipulation of
the pool. Lock-free stacks and queues are not hard, but they can suffer
from the ABA problem so they are better avoided unless you have some
deferred reclamation scheme like RCU. Otherwise you have to stick
with adding to a list, and emptying it completely. This is what this
patch does, by coupling a lock-free global list of available coroutines
with per-CPU lists that are actually used on coroutine creation.
Whenever the destruction pool is big enough, the next thread that runs
out of coroutines will steal the whole destruction pool. This is positive
in two ways:
1) the allocation does not have to do any atomic operation in the fast
path, it's entirely using thread-local storage. Once every POOL_BATCH_SIZE
allocations it will do a single atomic_xchg. Release does an atomic_cmpxchg
loop, that hopefully doesn't cause any starvation, and an atomic_inc.
A later patch will also remove atomic operations from the release path,
and try to avoid the atomic_xchg altogether---succeeding in doing so if
all devices either use ioeventfd or are not submitting requests actively.
2) in theory this should be completely adaptive. The number of coroutines
around should be a little more than POOL_BATCH_SIZE * number of allocating
threads; so this also empties qemu_coroutine_adjust_pool_size. (The previous
pool size was POOL_BATCH_SIZE * number of block backends, so it was a bit
more generous. But if you actually have many high-iodepth disks, it's better
to put them in different iothreads, which will also use separate thread
pools and aio=native file descriptors).
This speeds up perf/cost (in tests/test-coroutine) by a factor of ~1.33.
No matter if we end with some kind of coroutine bypass scheme or not,
it cannot hurt to optimize hot code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
Message-id: 1417518350-6167-6-git-send-email-pbonzini@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
2014-12-02 11:05:48 +00:00
|
|
|
QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
|
2013-09-11 14:42:35 +00:00
|
|
|
}
|
2013-05-17 13:51:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!co) {
|
2013-02-19 10:59:09 +00:00
|
|
|
co = qemu_coroutine_new();
|
|
|
|
}
|
|
|
|
|
2011-01-17 16:08:14 +00:00
|
|
|
co->entry = entry;
|
2013-05-17 13:51:26 +00:00
|
|
|
QTAILQ_INIT(&co->co_queue_wakeup);
|
2011-01-17 16:08:14 +00:00
|
|
|
return co;
|
|
|
|
}
|
|
|
|
|
2013-02-19 10:59:09 +00:00
|
|
|
static void coroutine_delete(Coroutine *co)
|
|
|
|
{
|
coroutine: rewrite pool to avoid mutex
This patch removes the mutex by using fancy lock-free manipulation of
the pool. Lock-free stacks and queues are not hard, but they can suffer
from the ABA problem so they are better avoided unless you have some
deferred reclamation scheme like RCU. Otherwise you have to stick
with adding to a list, and emptying it completely. This is what this
patch does, by coupling a lock-free global list of available coroutines
with per-CPU lists that are actually used on coroutine creation.
Whenever the destruction pool is big enough, the next thread that runs
out of coroutines will steal the whole destruction pool. This is positive
in two ways:
1) the allocation does not have to do any atomic operation in the fast
path, it's entirely using thread-local storage. Once every POOL_BATCH_SIZE
allocations it will do a single atomic_xchg. Release does an atomic_cmpxchg
loop, that hopefully doesn't cause any starvation, and an atomic_inc.
A later patch will also remove atomic operations from the release path,
and try to avoid the atomic_xchg altogether---succeeding in doing so if
all devices either use ioeventfd or are not submitting requests actively.
2) in theory this should be completely adaptive. The number of coroutines
around should be a little more than POOL_BATCH_SIZE * number of allocating
threads; so this also empties qemu_coroutine_adjust_pool_size. (The previous
pool size was POOL_BATCH_SIZE * number of block backends, so it was a bit
more generous. But if you actually have many high-iodepth disks, it's better
to put them in different iothreads, which will also use separate thread
pools and aio=native file descriptors).
This speeds up perf/cost (in tests/test-coroutine) by a factor of ~1.33.
No matter if we end with some kind of coroutine bypass scheme or not,
it cannot hurt to optimize hot code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
Message-id: 1417518350-6167-6-git-send-email-pbonzini@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
2014-12-02 11:05:48 +00:00
|
|
|
co->caller = NULL;
|
|
|
|
|
2013-09-11 14:42:35 +00:00
|
|
|
if (CONFIG_COROUTINE_POOL) {
|
coroutine: rewrite pool to avoid mutex
This patch removes the mutex by using fancy lock-free manipulation of
the pool. Lock-free stacks and queues are not hard, but they can suffer
from the ABA problem so they are better avoided unless you have some
deferred reclamation scheme like RCU. Otherwise you have to stick
with adding to a list, and emptying it completely. This is what this
patch does, by coupling a lock-free global list of available coroutines
with per-CPU lists that are actually used on coroutine creation.
Whenever the destruction pool is big enough, the next thread that runs
out of coroutines will steal the whole destruction pool. This is positive
in two ways:
1) the allocation does not have to do any atomic operation in the fast
path, it's entirely using thread-local storage. Once every POOL_BATCH_SIZE
allocations it will do a single atomic_xchg. Release does an atomic_cmpxchg
loop, that hopefully doesn't cause any starvation, and an atomic_inc.
A later patch will also remove atomic operations from the release path,
and try to avoid the atomic_xchg altogether---succeeding in doing so if
all devices either use ioeventfd or are not submitting requests actively.
2) in theory this should be completely adaptive. The number of coroutines
around should be a little more than POOL_BATCH_SIZE * number of allocating
threads; so this also empties qemu_coroutine_adjust_pool_size. (The previous
pool size was POOL_BATCH_SIZE * number of block backends, so it was a bit
more generous. But if you actually have many high-iodepth disks, it's better
to put them in different iothreads, which will also use separate thread
pools and aio=native file descriptors).
This speeds up perf/cost (in tests/test-coroutine) by a factor of ~1.33.
No matter if we end with some kind of coroutine bypass scheme or not,
it cannot hurt to optimize hot code.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
Message-id: 1417518350-6167-6-git-send-email-pbonzini@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
2014-12-02 11:05:48 +00:00
|
|
|
if (release_pool_size < POOL_BATCH_SIZE * 2) {
|
|
|
|
QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next);
|
|
|
|
atomic_inc(&release_pool_size);
|
2013-09-11 14:42:35 +00:00
|
|
|
return;
|
|
|
|
}
|
2013-02-19 10:59:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
qemu_coroutine_delete(co);
|
|
|
|
}
|
|
|
|
|
2011-01-17 16:08:14 +00:00
|
|
|
static void coroutine_swap(Coroutine *from, Coroutine *to)
|
|
|
|
{
|
|
|
|
CoroutineAction ret;
|
|
|
|
|
|
|
|
ret = qemu_coroutine_switch(from, to, COROUTINE_YIELD);
|
|
|
|
|
2013-05-17 13:51:26 +00:00
|
|
|
qemu_co_queue_run_restart(to);
|
|
|
|
|
2011-01-17 16:08:14 +00:00
|
|
|
switch (ret) {
|
|
|
|
case COROUTINE_YIELD:
|
|
|
|
return;
|
|
|
|
case COROUTINE_TERMINATE:
|
|
|
|
trace_qemu_coroutine_terminate(to);
|
2013-02-19 10:59:09 +00:00
|
|
|
coroutine_delete(to);
|
2011-01-17 16:08:14 +00:00
|
|
|
return;
|
|
|
|
default:
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void qemu_coroutine_enter(Coroutine *co, void *opaque)
|
|
|
|
{
|
|
|
|
Coroutine *self = qemu_coroutine_self();
|
|
|
|
|
|
|
|
trace_qemu_coroutine_enter(self, co, opaque);
|
|
|
|
|
|
|
|
if (co->caller) {
|
|
|
|
fprintf(stderr, "Co-routine re-entered recursively\n");
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
|
|
|
|
co->caller = self;
|
|
|
|
co->entry_arg = opaque;
|
|
|
|
coroutine_swap(self, co);
|
|
|
|
}
|
|
|
|
|
|
|
|
void coroutine_fn qemu_coroutine_yield(void)
|
|
|
|
{
|
|
|
|
Coroutine *self = qemu_coroutine_self();
|
|
|
|
Coroutine *to = self->caller;
|
|
|
|
|
|
|
|
trace_qemu_coroutine_yield(self, to);
|
|
|
|
|
|
|
|
if (!to) {
|
|
|
|
fprintf(stderr, "Co-routine is yielding to no one\n");
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
|
|
|
|
self->caller = NULL;
|
|
|
|
coroutine_swap(self, to);
|
|
|
|
}
|
2014-07-07 13:15:52 +00:00
|
|
|
|
|
|
|
void qemu_coroutine_adjust_pool_size(int n)
|
|
|
|
{
|
|
|
|
}
|