qemu/block/blkio.c
Alberto Faria e56b0c6631 block/blkio: Set BlockDriver::has_variable_length to false
Setting it to true can cause the device size to be queried from libblkio
in otherwise fast paths, degrading performance. Set it to false and
require users to refresh the device size explicitly instead.

Fixes: 4c8f4fda05 ("block/blkio: Tolerate device size changes")
Suggested-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Alberto Faria <afaria@redhat.com>
Message-Id: <20221108144433.1334074-1-afaria@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2022-11-10 14:52:36 +01:00

1047 lines
30 KiB
C

/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
* libblkio BlockDriver
*
* Copyright Red Hat, Inc.
*
* Author:
* Stefan Hajnoczi <stefanha@redhat.com>
*/
#include "qemu/osdep.h"
#include <blkio.h>
#include "block/block_int.h"
#include "exec/memory.h"
#include "exec/cpu-common.h" /* for qemu_ram_get_fd() */
#include "qapi/error.h"
#include "qemu/error-report.h"
#include "qapi/qmp/qdict.h"
#include "qemu/module.h"
#include "exec/memory.h" /* for ram_block_discard_disable() */
/*
* Keep the QEMU BlockDriver names identical to the libblkio driver names.
* Using macros instead of typing out the string literals avoids typos.
*/
#define DRIVER_IO_URING "io_uring"
#define DRIVER_NVME_IO_URING "nvme-io_uring"
#define DRIVER_VIRTIO_BLK_VFIO_PCI "virtio-blk-vfio-pci"
#define DRIVER_VIRTIO_BLK_VHOST_USER "virtio-blk-vhost-user"
#define DRIVER_VIRTIO_BLK_VHOST_VDPA "virtio-blk-vhost-vdpa"
/*
* Allocated bounce buffers are kept in a list sorted by buffer address.
*/
typedef struct BlkioBounceBuf {
QLIST_ENTRY(BlkioBounceBuf) next;
/* The bounce buffer */
struct iovec buf;
} BlkioBounceBuf;
typedef struct {
/*
* libblkio is not thread-safe so this lock protects ->blkio and
* ->blkioq.
*/
QemuMutex blkio_lock;
struct blkio *blkio;
struct blkioq *blkioq; /* make this multi-queue in the future... */
int completion_fd;
/*
* Polling fetches the next completion into this field.
*
* No lock is necessary since only one thread calls aio_poll() and invokes
* fd and poll handlers.
*/
struct blkio_completion poll_completion;
/*
* Protects ->bounce_pool, ->bounce_bufs, ->bounce_available.
*
* Lock ordering: ->bounce_lock before ->blkio_lock.
*/
CoMutex bounce_lock;
/* Bounce buffer pool */
struct blkio_mem_region bounce_pool;
/* Sorted list of allocated bounce buffers */
QLIST_HEAD(, BlkioBounceBuf) bounce_bufs;
/* Queue for coroutines waiting for bounce buffer space */
CoQueue bounce_available;
/* The value of the "mem-region-alignment" property */
size_t mem_region_alignment;
/* Can we skip adding/deleting blkio_mem_regions? */
bool needs_mem_regions;
/* Are file descriptors necessary for blkio_mem_regions? */
bool needs_mem_region_fd;
/* Are madvise(MADV_DONTNEED)-style operations unavailable? */
bool may_pin_mem_regions;
} BDRVBlkioState;
/* Called with s->bounce_lock held */
static int blkio_resize_bounce_pool(BDRVBlkioState *s, int64_t bytes)
{
/* There can be no allocated bounce buffers during resize */
assert(QLIST_EMPTY(&s->bounce_bufs));
/* Pad size to reduce frequency of resize calls */
bytes += 128 * 1024;
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
int ret;
if (s->bounce_pool.addr) {
blkio_unmap_mem_region(s->blkio, &s->bounce_pool);
blkio_free_mem_region(s->blkio, &s->bounce_pool);
memset(&s->bounce_pool, 0, sizeof(s->bounce_pool));
}
/* Automatically freed when s->blkio is destroyed */
ret = blkio_alloc_mem_region(s->blkio, &s->bounce_pool, bytes);
if (ret < 0) {
return ret;
}
ret = blkio_map_mem_region(s->blkio, &s->bounce_pool);
if (ret < 0) {
blkio_free_mem_region(s->blkio, &s->bounce_pool);
memset(&s->bounce_pool, 0, sizeof(s->bounce_pool));
return ret;
}
}
return 0;
}
/* Called with s->bounce_lock held */
static bool
blkio_do_alloc_bounce_buffer(BDRVBlkioState *s, BlkioBounceBuf *bounce,
int64_t bytes)
{
void *addr = s->bounce_pool.addr;
BlkioBounceBuf *cur = NULL;
BlkioBounceBuf *prev = NULL;
ptrdiff_t space;
/*
* This is just a linear search over the holes between requests. An
* efficient allocator would be nice.
*/
QLIST_FOREACH(cur, &s->bounce_bufs, next) {
space = cur->buf.iov_base - addr;
if (bytes <= space) {
QLIST_INSERT_BEFORE(cur, bounce, next);
bounce->buf.iov_base = addr;
bounce->buf.iov_len = bytes;
return true;
}
addr = cur->buf.iov_base + cur->buf.iov_len;
prev = cur;
}
/* Is there space after the last request? */
space = s->bounce_pool.addr + s->bounce_pool.len - addr;
if (bytes > space) {
return false;
}
if (prev) {
QLIST_INSERT_AFTER(prev, bounce, next);
} else {
QLIST_INSERT_HEAD(&s->bounce_bufs, bounce, next);
}
bounce->buf.iov_base = addr;
bounce->buf.iov_len = bytes;
return true;
}
static int coroutine_fn
blkio_alloc_bounce_buffer(BDRVBlkioState *s, BlkioBounceBuf *bounce,
int64_t bytes)
{
/*
* Ensure fairness: first time around we join the back of the queue,
* subsequently we join the front so we don't lose our place.
*/
CoQueueWaitFlags wait_flags = 0;
QEMU_LOCK_GUARD(&s->bounce_lock);
/* Ensure fairness: don't even try if other requests are already waiting */
if (!qemu_co_queue_empty(&s->bounce_available)) {
qemu_co_queue_wait_flags(&s->bounce_available, &s->bounce_lock,
wait_flags);
wait_flags = CO_QUEUE_WAIT_FRONT;
}
while (true) {
if (blkio_do_alloc_bounce_buffer(s, bounce, bytes)) {
/* Kick the next queued request since there may be space */
qemu_co_queue_next(&s->bounce_available);
return 0;
}
/*
* If there are no in-flight requests then the pool was simply too
* small.
*/
if (QLIST_EMPTY(&s->bounce_bufs)) {
bool ok;
int ret;
ret = blkio_resize_bounce_pool(s, bytes);
if (ret < 0) {
/* Kick the next queued request since that may fail too */
qemu_co_queue_next(&s->bounce_available);
return ret;
}
ok = blkio_do_alloc_bounce_buffer(s, bounce, bytes);
assert(ok); /* must have space this time */
return 0;
}
qemu_co_queue_wait_flags(&s->bounce_available, &s->bounce_lock,
wait_flags);
wait_flags = CO_QUEUE_WAIT_FRONT;
}
}
static void coroutine_fn blkio_free_bounce_buffer(BDRVBlkioState *s,
BlkioBounceBuf *bounce)
{
QEMU_LOCK_GUARD(&s->bounce_lock);
QLIST_REMOVE(bounce, next);
/* Wake up waiting coroutines since space may now be available */
qemu_co_queue_next(&s->bounce_available);
}
/* For async to .bdrv_co_*() conversion */
typedef struct {
Coroutine *coroutine;
int ret;
} BlkioCoData;
static void blkio_completion_fd_read(void *opaque)
{
BlockDriverState *bs = opaque;
BDRVBlkioState *s = bs->opaque;
uint64_t val;
int ret;
/* Polling may have already fetched a completion */
if (s->poll_completion.user_data != NULL) {
BlkioCoData *cod = s->poll_completion.user_data;
cod->ret = s->poll_completion.ret;
/* Clear it in case aio_co_wake() enters a nested event loop */
s->poll_completion.user_data = NULL;
aio_co_wake(cod->coroutine);
}
/* Reset completion fd status */
ret = read(s->completion_fd, &val, sizeof(val));
/* Ignore errors, there's nothing we can do */
(void)ret;
/*
* Reading one completion at a time makes nested event loop re-entrancy
* simple. Change this loop to get multiple completions in one go if it
* becomes a performance bottleneck.
*/
while (true) {
struct blkio_completion completion;
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
ret = blkioq_do_io(s->blkioq, &completion, 0, 1, NULL);
}
if (ret != 1) {
break;
}
BlkioCoData *cod = completion.user_data;
cod->ret = completion.ret;
aio_co_wake(cod->coroutine);
}
}
static bool blkio_completion_fd_poll(void *opaque)
{
BlockDriverState *bs = opaque;
BDRVBlkioState *s = bs->opaque;
int ret;
/* Just in case we already fetched a completion */
if (s->poll_completion.user_data != NULL) {
return true;
}
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
ret = blkioq_do_io(s->blkioq, &s->poll_completion, 0, 1, NULL);
}
return ret == 1;
}
static void blkio_completion_fd_poll_ready(void *opaque)
{
blkio_completion_fd_read(opaque);
}
static void blkio_attach_aio_context(BlockDriverState *bs,
AioContext *new_context)
{
BDRVBlkioState *s = bs->opaque;
aio_set_fd_handler(new_context,
s->completion_fd,
false,
blkio_completion_fd_read,
NULL,
blkio_completion_fd_poll,
blkio_completion_fd_poll_ready,
bs);
}
static void blkio_detach_aio_context(BlockDriverState *bs)
{
BDRVBlkioState *s = bs->opaque;
aio_set_fd_handler(bdrv_get_aio_context(bs),
s->completion_fd,
false, NULL, NULL, NULL, NULL, NULL);
}
/* Call with s->blkio_lock held to submit I/O after enqueuing a new request */
static void blkio_submit_io(BlockDriverState *bs)
{
if (qatomic_read(&bs->io_plugged) == 0) {
BDRVBlkioState *s = bs->opaque;
blkioq_do_io(s->blkioq, NULL, 0, 0, NULL);
}
}
static int coroutine_fn
blkio_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes)
{
BDRVBlkioState *s = bs->opaque;
BlkioCoData cod = {
.coroutine = qemu_coroutine_self(),
};
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
blkioq_discard(s->blkioq, offset, bytes, &cod, 0);
blkio_submit_io(bs);
}
qemu_coroutine_yield();
return cod.ret;
}
static int coroutine_fn
blkio_co_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes,
QEMUIOVector *qiov, BdrvRequestFlags flags)
{
BlkioCoData cod = {
.coroutine = qemu_coroutine_self(),
};
BDRVBlkioState *s = bs->opaque;
bool use_bounce_buffer =
s->needs_mem_regions && !(flags & BDRV_REQ_REGISTERED_BUF);
BlkioBounceBuf bounce;
struct iovec *iov = qiov->iov;
int iovcnt = qiov->niov;
if (use_bounce_buffer) {
int ret = blkio_alloc_bounce_buffer(s, &bounce, bytes);
if (ret < 0) {
return ret;
}
iov = &bounce.buf;
iovcnt = 1;
}
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
blkioq_readv(s->blkioq, offset, iov, iovcnt, &cod, 0);
blkio_submit_io(bs);
}
qemu_coroutine_yield();
if (use_bounce_buffer) {
if (cod.ret == 0) {
qemu_iovec_from_buf(qiov, 0,
bounce.buf.iov_base,
bounce.buf.iov_len);
}
blkio_free_bounce_buffer(s, &bounce);
}
return cod.ret;
}
static int coroutine_fn blkio_co_pwritev(BlockDriverState *bs, int64_t offset,
int64_t bytes, QEMUIOVector *qiov, BdrvRequestFlags flags)
{
uint32_t blkio_flags = (flags & BDRV_REQ_FUA) ? BLKIO_REQ_FUA : 0;
BlkioCoData cod = {
.coroutine = qemu_coroutine_self(),
};
BDRVBlkioState *s = bs->opaque;
bool use_bounce_buffer =
s->needs_mem_regions && !(flags & BDRV_REQ_REGISTERED_BUF);
BlkioBounceBuf bounce;
struct iovec *iov = qiov->iov;
int iovcnt = qiov->niov;
if (use_bounce_buffer) {
int ret = blkio_alloc_bounce_buffer(s, &bounce, bytes);
if (ret < 0) {
return ret;
}
qemu_iovec_to_buf(qiov, 0, bounce.buf.iov_base, bytes);
iov = &bounce.buf;
iovcnt = 1;
}
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
blkioq_writev(s->blkioq, offset, iov, iovcnt, &cod, blkio_flags);
blkio_submit_io(bs);
}
qemu_coroutine_yield();
if (use_bounce_buffer) {
blkio_free_bounce_buffer(s, &bounce);
}
return cod.ret;
}
static int coroutine_fn blkio_co_flush(BlockDriverState *bs)
{
BDRVBlkioState *s = bs->opaque;
BlkioCoData cod = {
.coroutine = qemu_coroutine_self(),
};
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
blkioq_flush(s->blkioq, &cod, 0);
blkio_submit_io(bs);
}
qemu_coroutine_yield();
return cod.ret;
}
static int coroutine_fn blkio_co_pwrite_zeroes(BlockDriverState *bs,
int64_t offset, int64_t bytes, BdrvRequestFlags flags)
{
BDRVBlkioState *s = bs->opaque;
BlkioCoData cod = {
.coroutine = qemu_coroutine_self(),
};
uint32_t blkio_flags = 0;
if (flags & BDRV_REQ_FUA) {
blkio_flags |= BLKIO_REQ_FUA;
}
if (!(flags & BDRV_REQ_MAY_UNMAP)) {
blkio_flags |= BLKIO_REQ_NO_UNMAP;
}
if (flags & BDRV_REQ_NO_FALLBACK) {
blkio_flags |= BLKIO_REQ_NO_FALLBACK;
}
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
blkioq_write_zeroes(s->blkioq, offset, bytes, &cod, blkio_flags);
blkio_submit_io(bs);
}
qemu_coroutine_yield();
return cod.ret;
}
static void blkio_io_unplug(BlockDriverState *bs)
{
BDRVBlkioState *s = bs->opaque;
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
blkio_submit_io(bs);
}
}
typedef enum {
BMRR_OK,
BMRR_SKIP,
BMRR_FAIL,
} BlkioMemRegionResult;
/*
* Produce a struct blkio_mem_region for a given address and size.
*
* This function produces identical results when called multiple times with the
* same arguments. This property is necessary because blkio_unmap_mem_region()
* must receive the same struct blkio_mem_region field values that were passed
* to blkio_map_mem_region().
*/
static BlkioMemRegionResult
blkio_mem_region_from_host(BlockDriverState *bs,
void *host, size_t size,
struct blkio_mem_region *region,
Error **errp)
{
BDRVBlkioState *s = bs->opaque;
int fd = -1;
ram_addr_t fd_offset = 0;
if (((uintptr_t)host | size) % s->mem_region_alignment) {
error_setg(errp, "unaligned buf %p with size %zu", host, size);
return BMRR_FAIL;
}
/* Attempt to find the fd for the underlying memory */
if (s->needs_mem_region_fd) {
RAMBlock *ram_block;
RAMBlock *end_block;
ram_addr_t offset;
/*
* bdrv_register_buf() is called with the BQL held so mr lives at least
* until this function returns.
*/
ram_block = qemu_ram_block_from_host(host, false, &fd_offset);
if (ram_block) {
fd = qemu_ram_get_fd(ram_block);
}
if (fd == -1) {
/*
* Ideally every RAMBlock would have an fd. pc-bios and other
* things don't. Luckily they are usually not I/O buffers and we
* can just ignore them.
*/
return BMRR_SKIP;
}
/* Make sure the fd covers the entire range */
end_block = qemu_ram_block_from_host(host + size - 1, false, &offset);
if (ram_block != end_block) {
error_setg(errp, "registered buffer at %p with size %zu extends "
"beyond RAMBlock", host, size);
return BMRR_FAIL;
}
}
*region = (struct blkio_mem_region){
.addr = host,
.len = size,
.fd = fd,
.fd_offset = fd_offset,
};
return BMRR_OK;
}
static bool blkio_register_buf(BlockDriverState *bs, void *host, size_t size,
Error **errp)
{
BDRVBlkioState *s = bs->opaque;
struct blkio_mem_region region;
BlkioMemRegionResult region_result;
int ret;
/*
* Mapping memory regions conflicts with RAM discard (virtio-mem) when
* there is pinning, so only do it when necessary.
*/
if (!s->needs_mem_regions && s->may_pin_mem_regions) {
return true;
}
region_result = blkio_mem_region_from_host(bs, host, size, &region, errp);
if (region_result == BMRR_SKIP) {
return true;
} else if (region_result != BMRR_OK) {
return false;
}
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
ret = blkio_map_mem_region(s->blkio, &region);
}
if (ret < 0) {
error_setg(errp, "Failed to add blkio mem region %p with size %zu: %s",
host, size, blkio_get_error_msg());
return false;
}
return true;
}
static void blkio_unregister_buf(BlockDriverState *bs, void *host, size_t size)
{
BDRVBlkioState *s = bs->opaque;
struct blkio_mem_region region;
/* See blkio_register_buf() */
if (!s->needs_mem_regions && s->may_pin_mem_regions) {
return;
}
if (blkio_mem_region_from_host(bs, host, size, &region, NULL) != BMRR_OK) {
return;
}
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
blkio_unmap_mem_region(s->blkio, &region);
}
}
static int blkio_io_uring_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
const char *filename = qdict_get_str(options, "filename");
BDRVBlkioState *s = bs->opaque;
int ret;
ret = blkio_set_str(s->blkio, "path", filename);
qdict_del(options, "filename");
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to set path: %s",
blkio_get_error_msg());
return ret;
}
if (flags & BDRV_O_NOCACHE) {
ret = blkio_set_bool(s->blkio, "direct", true);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to set direct: %s",
blkio_get_error_msg());
return ret;
}
}
return 0;
}
static int blkio_nvme_io_uring(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
const char *path = qdict_get_try_str(options, "path");
BDRVBlkioState *s = bs->opaque;
int ret;
if (!path) {
error_setg(errp, "missing 'path' option");
return -EINVAL;
}
ret = blkio_set_str(s->blkio, "path", path);
qdict_del(options, "path");
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to set path: %s",
blkio_get_error_msg());
return ret;
}
if (!(flags & BDRV_O_NOCACHE)) {
error_setg(errp, "cache.direct=off is not supported");
return -EINVAL;
}
return 0;
}
static int blkio_virtio_blk_common_open(BlockDriverState *bs,
QDict *options, int flags, Error **errp)
{
const char *path = qdict_get_try_str(options, "path");
BDRVBlkioState *s = bs->opaque;
int ret;
if (!path) {
error_setg(errp, "missing 'path' option");
return -EINVAL;
}
ret = blkio_set_str(s->blkio, "path", path);
qdict_del(options, "path");
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to set path: %s",
blkio_get_error_msg());
return ret;
}
if (!(flags & BDRV_O_NOCACHE)) {
error_setg(errp, "cache.direct=off is not supported");
return -EINVAL;
}
return 0;
}
static int blkio_file_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
const char *blkio_driver = bs->drv->protocol_name;
BDRVBlkioState *s = bs->opaque;
int ret;
ret = blkio_create(blkio_driver, &s->blkio);
if (ret < 0) {
error_setg_errno(errp, -ret, "blkio_create failed: %s",
blkio_get_error_msg());
return ret;
}
if (strcmp(blkio_driver, DRIVER_IO_URING) == 0) {
ret = blkio_io_uring_open(bs, options, flags, errp);
} else if (strcmp(blkio_driver, DRIVER_NVME_IO_URING) == 0) {
ret = blkio_nvme_io_uring(bs, options, flags, errp);
} else if (strcmp(blkio_driver, DRIVER_VIRTIO_BLK_VFIO_PCI) == 0) {
ret = blkio_virtio_blk_common_open(bs, options, flags, errp);
} else if (strcmp(blkio_driver, DRIVER_VIRTIO_BLK_VHOST_USER) == 0) {
ret = blkio_virtio_blk_common_open(bs, options, flags, errp);
} else if (strcmp(blkio_driver, DRIVER_VIRTIO_BLK_VHOST_VDPA) == 0) {
ret = blkio_virtio_blk_common_open(bs, options, flags, errp);
} else {
g_assert_not_reached();
}
if (ret < 0) {
blkio_destroy(&s->blkio);
return ret;
}
if (!(flags & BDRV_O_RDWR)) {
ret = blkio_set_bool(s->blkio, "read-only", true);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to set read-only: %s",
blkio_get_error_msg());
blkio_destroy(&s->blkio);
return ret;
}
}
ret = blkio_connect(s->blkio);
if (ret < 0) {
error_setg_errno(errp, -ret, "blkio_connect failed: %s",
blkio_get_error_msg());
blkio_destroy(&s->blkio);
return ret;
}
ret = blkio_get_bool(s->blkio,
"needs-mem-regions",
&s->needs_mem_regions);
if (ret < 0) {
error_setg_errno(errp, -ret,
"failed to get needs-mem-regions: %s",
blkio_get_error_msg());
blkio_destroy(&s->blkio);
return ret;
}
ret = blkio_get_bool(s->blkio,
"needs-mem-region-fd",
&s->needs_mem_region_fd);
if (ret < 0) {
error_setg_errno(errp, -ret,
"failed to get needs-mem-region-fd: %s",
blkio_get_error_msg());
blkio_destroy(&s->blkio);
return ret;
}
ret = blkio_get_uint64(s->blkio,
"mem-region-alignment",
&s->mem_region_alignment);
if (ret < 0) {
error_setg_errno(errp, -ret,
"failed to get mem-region-alignment: %s",
blkio_get_error_msg());
blkio_destroy(&s->blkio);
return ret;
}
ret = blkio_get_bool(s->blkio,
"may-pin-mem-regions",
&s->may_pin_mem_regions);
if (ret < 0) {
/* Be conservative (assume pinning) if the property is not supported */
s->may_pin_mem_regions = s->needs_mem_regions;
}
/*
* Notify if libblkio drivers pin memory and prevent features like
* virtio-mem from working.
*/
if (s->may_pin_mem_regions) {
ret = ram_block_discard_disable(true);
if (ret < 0) {
error_setg_errno(errp, -ret, "ram_block_discard_disable() failed");
blkio_destroy(&s->blkio);
return ret;
}
}
ret = blkio_start(s->blkio);
if (ret < 0) {
error_setg_errno(errp, -ret, "blkio_start failed: %s",
blkio_get_error_msg());
blkio_destroy(&s->blkio);
if (s->may_pin_mem_regions) {
ram_block_discard_disable(false);
}
return ret;
}
bs->supported_write_flags = BDRV_REQ_FUA | BDRV_REQ_REGISTERED_BUF;
bs->supported_zero_flags = BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP |
BDRV_REQ_NO_FALLBACK;
qemu_mutex_init(&s->blkio_lock);
qemu_co_mutex_init(&s->bounce_lock);
qemu_co_queue_init(&s->bounce_available);
QLIST_INIT(&s->bounce_bufs);
s->blkioq = blkio_get_queue(s->blkio, 0);
s->completion_fd = blkioq_get_completion_fd(s->blkioq);
blkio_attach_aio_context(bs, bdrv_get_aio_context(bs));
return 0;
}
static void blkio_close(BlockDriverState *bs)
{
BDRVBlkioState *s = bs->opaque;
/* There is no destroy() API for s->bounce_lock */
qemu_mutex_destroy(&s->blkio_lock);
blkio_detach_aio_context(bs);
blkio_destroy(&s->blkio);
if (s->may_pin_mem_regions) {
ram_block_discard_disable(false);
}
}
static int64_t blkio_getlength(BlockDriverState *bs)
{
BDRVBlkioState *s = bs->opaque;
uint64_t capacity;
int ret;
WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
ret = blkio_get_uint64(s->blkio, "capacity", &capacity);
}
if (ret < 0) {
return -ret;
}
return capacity;
}
static int coroutine_fn blkio_truncate(BlockDriverState *bs, int64_t offset,
bool exact, PreallocMode prealloc,
BdrvRequestFlags flags, Error **errp)
{
int64_t current_length;
if (prealloc != PREALLOC_MODE_OFF) {
error_setg(errp, "Unsupported preallocation mode '%s'",
PreallocMode_str(prealloc));
return -ENOTSUP;
}
current_length = blkio_getlength(bs);
if (offset > current_length) {
error_setg(errp, "Cannot grow device");
return -EINVAL;
} else if (exact && offset != current_length) {
error_setg(errp, "Cannot resize device");
return -ENOTSUP;
}
return 0;
}
static int blkio_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
{
return 0;
}
static void blkio_refresh_limits(BlockDriverState *bs, Error **errp)
{
BDRVBlkioState *s = bs->opaque;
QEMU_LOCK_GUARD(&s->blkio_lock);
int value;
int ret;
ret = blkio_get_int(s->blkio, "request-alignment", &value);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to get \"request-alignment\": %s",
blkio_get_error_msg());
return;
}
bs->bl.request_alignment = value;
if (bs->bl.request_alignment < 1 ||
bs->bl.request_alignment >= INT_MAX ||
!is_power_of_2(bs->bl.request_alignment)) {
error_setg(errp, "invalid \"request-alignment\" value %" PRIu32 ", "
"must be a power of 2 less than INT_MAX",
bs->bl.request_alignment);
return;
}
ret = blkio_get_int(s->blkio, "optimal-io-size", &value);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to get \"optimal-io-size\": %s",
blkio_get_error_msg());
return;
}
bs->bl.opt_transfer = value;
if (bs->bl.opt_transfer > INT_MAX ||
(bs->bl.opt_transfer % bs->bl.request_alignment)) {
error_setg(errp, "invalid \"optimal-io-size\" value %" PRIu32 ", must "
"be a multiple of %" PRIu32, bs->bl.opt_transfer,
bs->bl.request_alignment);
return;
}
ret = blkio_get_int(s->blkio, "max-transfer", &value);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to get \"max-transfer\": %s",
blkio_get_error_msg());
return;
}
bs->bl.max_transfer = value;
if ((bs->bl.max_transfer % bs->bl.request_alignment) ||
(bs->bl.opt_transfer && (bs->bl.max_transfer % bs->bl.opt_transfer))) {
error_setg(errp, "invalid \"max-transfer\" value %" PRIu32 ", must be "
"a multiple of %" PRIu32 " and %" PRIu32 " (if non-zero)",
bs->bl.max_transfer, bs->bl.request_alignment,
bs->bl.opt_transfer);
return;
}
ret = blkio_get_int(s->blkio, "buf-alignment", &value);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to get \"buf-alignment\": %s",
blkio_get_error_msg());
return;
}
if (value < 1) {
error_setg(errp, "invalid \"buf-alignment\" value %d, must be "
"positive", value);
return;
}
bs->bl.min_mem_alignment = value;
ret = blkio_get_int(s->blkio, "optimal-buf-alignment", &value);
if (ret < 0) {
error_setg_errno(errp, -ret,
"failed to get \"optimal-buf-alignment\": %s",
blkio_get_error_msg());
return;
}
if (value < 1) {
error_setg(errp, "invalid \"optimal-buf-alignment\" value %d, "
"must be positive", value);
return;
}
bs->bl.opt_mem_alignment = value;
ret = blkio_get_int(s->blkio, "max-segments", &value);
if (ret < 0) {
error_setg_errno(errp, -ret, "failed to get \"max-segments\": %s",
blkio_get_error_msg());
return;
}
if (value < 1) {
error_setg(errp, "invalid \"max-segments\" value %d, must be positive",
value);
return;
}
bs->bl.max_iov = value;
}
/*
* TODO
* Missing libblkio APIs:
* - block_status
* - co_invalidate_cache
*
* Out of scope?
* - create
* - truncate
*/
#define BLKIO_DRIVER(name, ...) \
{ \
.format_name = name, \
.protocol_name = name, \
.instance_size = sizeof(BDRVBlkioState), \
.bdrv_file_open = blkio_file_open, \
.bdrv_close = blkio_close, \
.bdrv_getlength = blkio_getlength, \
.bdrv_co_truncate = blkio_truncate, \
.bdrv_get_info = blkio_get_info, \
.bdrv_attach_aio_context = blkio_attach_aio_context, \
.bdrv_detach_aio_context = blkio_detach_aio_context, \
.bdrv_co_pdiscard = blkio_co_pdiscard, \
.bdrv_co_preadv = blkio_co_preadv, \
.bdrv_co_pwritev = blkio_co_pwritev, \
.bdrv_co_flush_to_disk = blkio_co_flush, \
.bdrv_co_pwrite_zeroes = blkio_co_pwrite_zeroes, \
.bdrv_io_unplug = blkio_io_unplug, \
.bdrv_refresh_limits = blkio_refresh_limits, \
.bdrv_register_buf = blkio_register_buf, \
.bdrv_unregister_buf = blkio_unregister_buf, \
__VA_ARGS__ \
}
static BlockDriver bdrv_io_uring = BLKIO_DRIVER(
DRIVER_IO_URING,
.bdrv_needs_filename = true,
);
static BlockDriver bdrv_nvme_io_uring = BLKIO_DRIVER(
DRIVER_NVME_IO_URING,
);
static BlockDriver bdrv_virtio_blk_vfio_pci = BLKIO_DRIVER(
DRIVER_VIRTIO_BLK_VFIO_PCI
);
static BlockDriver bdrv_virtio_blk_vhost_user = BLKIO_DRIVER(
DRIVER_VIRTIO_BLK_VHOST_USER
);
static BlockDriver bdrv_virtio_blk_vhost_vdpa = BLKIO_DRIVER(
DRIVER_VIRTIO_BLK_VHOST_VDPA
);
static void bdrv_blkio_init(void)
{
bdrv_register(&bdrv_io_uring);
bdrv_register(&bdrv_nvme_io_uring);
bdrv_register(&bdrv_virtio_blk_vfio_pci);
bdrv_register(&bdrv_virtio_blk_vhost_user);
bdrv_register(&bdrv_virtio_blk_vhost_vdpa);
}
block_init(bdrv_blkio_init);