linux/drivers/dma/dmatest.c
Andy Shevchenko bcc567e311 dmatest: do not allow to interrupt ongoing tests
When user interrupts ongoing transfers the dmatest may end up with console
lockup, oops, or data mismatch. This patch prevents user to abort any ongoing
test.

Documentation is updated accordingly.

Signed-off-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
Reported-by: Will Deacon <will.deacon@arm.com>
Tested-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2013-06-08 02:13:44 +05:30

1322 lines
33 KiB
C

/*
* DMA Engine test module
*
* Copyright (C) 2007 Atmel Corporation
* Copyright (C) 2013 Intel Corporation
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/freezer.h>
#include <linux/init.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/random.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/ctype.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/seq_file.h>
static unsigned int test_buf_size = 16384;
module_param(test_buf_size, uint, S_IRUGO);
MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
static char test_channel[20];
module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
static char test_device[20];
module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
static unsigned int threads_per_chan = 1;
module_param(threads_per_chan, uint, S_IRUGO);
MODULE_PARM_DESC(threads_per_chan,
"Number of threads to start per channel (default: 1)");
static unsigned int max_channels;
module_param(max_channels, uint, S_IRUGO);
MODULE_PARM_DESC(max_channels,
"Maximum number of channels to use (default: all)");
static unsigned int iterations;
module_param(iterations, uint, S_IRUGO);
MODULE_PARM_DESC(iterations,
"Iterations before stopping test (default: infinite)");
static unsigned int xor_sources = 3;
module_param(xor_sources, uint, S_IRUGO);
MODULE_PARM_DESC(xor_sources,
"Number of xor source buffers (default: 3)");
static unsigned int pq_sources = 3;
module_param(pq_sources, uint, S_IRUGO);
MODULE_PARM_DESC(pq_sources,
"Number of p+q source buffers (default: 3)");
static int timeout = 3000;
module_param(timeout, uint, S_IRUGO);
MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
"Pass -1 for infinite timeout");
/* Maximum amount of mismatched bytes in buffer to print */
#define MAX_ERROR_COUNT 32
/*
* Initialization patterns. All bytes in the source buffer has bit 7
* set, all bytes in the destination buffer has bit 7 cleared.
*
* Bit 6 is set for all bytes which are to be copied by the DMA
* engine. Bit 5 is set for all bytes which are to be overwritten by
* the DMA engine.
*
* The remaining bits are the inverse of a counter which increments by
* one for each byte address.
*/
#define PATTERN_SRC 0x80
#define PATTERN_DST 0x00
#define PATTERN_COPY 0x40
#define PATTERN_OVERWRITE 0x20
#define PATTERN_COUNT_MASK 0x1f
enum dmatest_error_type {
DMATEST_ET_OK,
DMATEST_ET_MAP_SRC,
DMATEST_ET_MAP_DST,
DMATEST_ET_PREP,
DMATEST_ET_SUBMIT,
DMATEST_ET_TIMEOUT,
DMATEST_ET_DMA_ERROR,
DMATEST_ET_DMA_IN_PROGRESS,
DMATEST_ET_VERIFY,
DMATEST_ET_VERIFY_BUF,
};
struct dmatest_verify_buffer {
unsigned int index;
u8 expected;
u8 actual;
};
struct dmatest_verify_result {
unsigned int error_count;
struct dmatest_verify_buffer data[MAX_ERROR_COUNT];
u8 pattern;
bool is_srcbuf;
};
struct dmatest_thread_result {
struct list_head node;
unsigned int n;
unsigned int src_off;
unsigned int dst_off;
unsigned int len;
enum dmatest_error_type type;
union {
unsigned long data;
dma_cookie_t cookie;
enum dma_status status;
int error;
struct dmatest_verify_result *vr;
};
};
struct dmatest_result {
struct list_head node;
char *name;
struct list_head results;
};
struct dmatest_info;
struct dmatest_thread {
struct list_head node;
struct dmatest_info *info;
struct task_struct *task;
struct dma_chan *chan;
u8 **srcs;
u8 **dsts;
enum dma_transaction_type type;
bool done;
};
struct dmatest_chan {
struct list_head node;
struct dma_chan *chan;
struct list_head threads;
};
/**
* struct dmatest_params - test parameters.
* @buf_size: size of the memcpy test buffer
* @channel: bus ID of the channel to test
* @device: bus ID of the DMA Engine to test
* @threads_per_chan: number of threads to start per channel
* @max_channels: maximum number of channels to use
* @iterations: iterations before stopping test
* @xor_sources: number of xor source buffers
* @pq_sources: number of p+q source buffers
* @timeout: transfer timeout in msec, -1 for infinite timeout
*/
struct dmatest_params {
unsigned int buf_size;
char channel[20];
char device[20];
unsigned int threads_per_chan;
unsigned int max_channels;
unsigned int iterations;
unsigned int xor_sources;
unsigned int pq_sources;
int timeout;
};
/**
* struct dmatest_info - test information.
* @params: test parameters
* @lock: access protection to the fields of this structure
*/
struct dmatest_info {
/* Test parameters */
struct dmatest_params params;
/* Internal state */
struct list_head channels;
unsigned int nr_channels;
struct mutex lock;
/* debugfs related stuff */
struct dentry *root;
struct dmatest_params dbgfs_params;
/* Test results */
struct list_head results;
struct mutex results_lock;
};
static struct dmatest_info test_info;
static bool dmatest_match_channel(struct dmatest_params *params,
struct dma_chan *chan)
{
if (params->channel[0] == '\0')
return true;
return strcmp(dma_chan_name(chan), params->channel) == 0;
}
static bool dmatest_match_device(struct dmatest_params *params,
struct dma_device *device)
{
if (params->device[0] == '\0')
return true;
return strcmp(dev_name(device->dev), params->device) == 0;
}
static unsigned long dmatest_random(void)
{
unsigned long buf;
get_random_bytes(&buf, sizeof(buf));
return buf;
}
static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
unsigned int buf_size)
{
unsigned int i;
u8 *buf;
for (; (buf = *bufs); bufs++) {
for (i = 0; i < start; i++)
buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
for ( ; i < start + len; i++)
buf[i] = PATTERN_SRC | PATTERN_COPY
| (~i & PATTERN_COUNT_MASK);
for ( ; i < buf_size; i++)
buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
buf++;
}
}
static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
unsigned int buf_size)
{
unsigned int i;
u8 *buf;
for (; (buf = *bufs); bufs++) {
for (i = 0; i < start; i++)
buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
for ( ; i < start + len; i++)
buf[i] = PATTERN_DST | PATTERN_OVERWRITE
| (~i & PATTERN_COUNT_MASK);
for ( ; i < buf_size; i++)
buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
}
}
static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
unsigned int start, unsigned int end, unsigned int counter,
u8 pattern, bool is_srcbuf)
{
unsigned int i;
unsigned int error_count = 0;
u8 actual;
u8 expected;
u8 *buf;
unsigned int counter_orig = counter;
struct dmatest_verify_buffer *vb;
for (; (buf = *bufs); bufs++) {
counter = counter_orig;
for (i = start; i < end; i++) {
actual = buf[i];
expected = pattern | (~counter & PATTERN_COUNT_MASK);
if (actual != expected) {
if (error_count < MAX_ERROR_COUNT && vr) {
vb = &vr->data[error_count];
vb->index = i;
vb->expected = expected;
vb->actual = actual;
}
error_count++;
}
counter++;
}
}
if (error_count > MAX_ERROR_COUNT)
pr_warning("%s: %u errors suppressed\n",
current->comm, error_count - MAX_ERROR_COUNT);
return error_count;
}
/* poor man's completion - we want to use wait_event_freezable() on it */
struct dmatest_done {
bool done;
wait_queue_head_t *wait;
};
static void dmatest_callback(void *arg)
{
struct dmatest_done *done = arg;
done->done = true;
wake_up_all(done->wait);
}
static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
unsigned int count)
{
while (count--)
dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
}
static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
unsigned int count)
{
while (count--)
dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
}
static unsigned int min_odd(unsigned int x, unsigned int y)
{
unsigned int val = min(x, y);
return val % 2 ? val : val - 1;
}
static char *verify_result_get_one(struct dmatest_verify_result *vr,
unsigned int i)
{
struct dmatest_verify_buffer *vb = &vr->data[i];
u8 diff = vb->actual ^ vr->pattern;
static char buf[512];
char *msg;
if (vr->is_srcbuf)
msg = "srcbuf overwritten!";
else if ((vr->pattern & PATTERN_COPY)
&& (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
msg = "dstbuf not copied!";
else if (diff & PATTERN_SRC)
msg = "dstbuf was copied!";
else
msg = "dstbuf mismatch!";
snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg,
vb->index, vb->expected, vb->actual);
return buf;
}
static char *thread_result_get(const char *name,
struct dmatest_thread_result *tr)
{
static const char * const messages[] = {
[DMATEST_ET_OK] = "No errors",
[DMATEST_ET_MAP_SRC] = "src mapping error",
[DMATEST_ET_MAP_DST] = "dst mapping error",
[DMATEST_ET_PREP] = "prep error",
[DMATEST_ET_SUBMIT] = "submit error",
[DMATEST_ET_TIMEOUT] = "test timed out",
[DMATEST_ET_DMA_ERROR] =
"got completion callback (DMA_ERROR)",
[DMATEST_ET_DMA_IN_PROGRESS] =
"got completion callback (DMA_IN_PROGRESS)",
[DMATEST_ET_VERIFY] = "errors",
[DMATEST_ET_VERIFY_BUF] = "verify errors",
};
static char buf[512];
snprintf(buf, sizeof(buf) - 1,
"%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
tr->len, tr->data);
return buf;
}
static int thread_result_add(struct dmatest_info *info,
struct dmatest_result *r, enum dmatest_error_type type,
unsigned int n, unsigned int src_off, unsigned int dst_off,
unsigned int len, unsigned long data)
{
struct dmatest_thread_result *tr;
tr = kzalloc(sizeof(*tr), GFP_KERNEL);
if (!tr)
return -ENOMEM;
tr->type = type;
tr->n = n;
tr->src_off = src_off;
tr->dst_off = dst_off;
tr->len = len;
tr->data = data;
mutex_lock(&info->results_lock);
list_add_tail(&tr->node, &r->results);
mutex_unlock(&info->results_lock);
pr_warn("%s\n", thread_result_get(r->name, tr));
return 0;
}
static unsigned int verify_result_add(struct dmatest_info *info,
struct dmatest_result *r, unsigned int n,
unsigned int src_off, unsigned int dst_off, unsigned int len,
u8 **bufs, int whence, unsigned int counter, u8 pattern,
bool is_srcbuf)
{
struct dmatest_verify_result *vr;
unsigned int error_count;
unsigned int buf_off = is_srcbuf ? src_off : dst_off;
unsigned int start, end;
if (whence < 0) {
start = 0;
end = buf_off;
} else if (whence > 0) {
start = buf_off + len;
end = info->params.buf_size;
} else {
start = buf_off;
end = buf_off + len;
}
vr = kmalloc(sizeof(*vr), GFP_KERNEL);
if (!vr) {
pr_warn("dmatest: No memory to store verify result\n");
return dmatest_verify(NULL, bufs, start, end, counter, pattern,
is_srcbuf);
}
vr->pattern = pattern;
vr->is_srcbuf = is_srcbuf;
error_count = dmatest_verify(vr, bufs, start, end, counter, pattern,
is_srcbuf);
if (error_count) {
vr->error_count = error_count;
thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off,
dst_off, len, (unsigned long)vr);
return error_count;
}
kfree(vr);
return 0;
}
static void result_free(struct dmatest_info *info, const char *name)
{
struct dmatest_result *r, *_r;
mutex_lock(&info->results_lock);
list_for_each_entry_safe(r, _r, &info->results, node) {
struct dmatest_thread_result *tr, *_tr;
if (name && strcmp(r->name, name))
continue;
list_for_each_entry_safe(tr, _tr, &r->results, node) {
if (tr->type == DMATEST_ET_VERIFY_BUF)
kfree(tr->vr);
list_del(&tr->node);
kfree(tr);
}
kfree(r->name);
list_del(&r->node);
kfree(r);
}
mutex_unlock(&info->results_lock);
}
static struct dmatest_result *result_init(struct dmatest_info *info,
const char *name)
{
struct dmatest_result *r;
r = kzalloc(sizeof(*r), GFP_KERNEL);
if (r) {
r->name = kstrdup(name, GFP_KERNEL);
INIT_LIST_HEAD(&r->results);
mutex_lock(&info->results_lock);
list_add_tail(&r->node, &info->results);
mutex_unlock(&info->results_lock);
}
return r;
}
/*
* This function repeatedly tests DMA transfers of various lengths and
* offsets for a given operation type until it is told to exit by
* kthread_stop(). There may be multiple threads running this function
* in parallel for a single channel, and there may be multiple channels
* being tested in parallel.
*
* Before each test, the source and destination buffer is initialized
* with a known pattern. This pattern is different depending on
* whether it's in an area which is supposed to be copied or
* overwritten, and different in the source and destination buffers.
* So if the DMA engine doesn't copy exactly what we tell it to copy,
* we'll notice.
*/
static int dmatest_func(void *data)
{
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
struct dmatest_thread *thread = data;
struct dmatest_done done = { .wait = &done_wait };
struct dmatest_info *info;
struct dmatest_params *params;
struct dma_chan *chan;
struct dma_device *dev;
const char *thread_name;
unsigned int src_off, dst_off, len;
unsigned int error_count;
unsigned int failed_tests = 0;
unsigned int total_tests = 0;
dma_cookie_t cookie;
enum dma_status status;
enum dma_ctrl_flags flags;
u8 *pq_coefs = NULL;
int ret;
int src_cnt;
int dst_cnt;
int i;
struct dmatest_result *result;
thread_name = current->comm;
set_freezable();
ret = -ENOMEM;
smp_rmb();
info = thread->info;
params = &info->params;
chan = thread->chan;
dev = chan->device;
if (thread->type == DMA_MEMCPY)
src_cnt = dst_cnt = 1;
else if (thread->type == DMA_XOR) {
/* force odd to ensure dst = src */
src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
dst_cnt = 1;
} else if (thread->type == DMA_PQ) {
/* force odd to ensure dst = src */
src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
dst_cnt = 2;
pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
if (!pq_coefs)
goto err_thread_type;
for (i = 0; i < src_cnt; i++)
pq_coefs[i] = 1;
} else
goto err_thread_type;
result = result_init(info, thread_name);
if (!result)
goto err_srcs;
thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
if (!thread->srcs)
goto err_srcs;
for (i = 0; i < src_cnt; i++) {
thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
if (!thread->srcs[i])
goto err_srcbuf;
}
thread->srcs[i] = NULL;
thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
if (!thread->dsts)
goto err_dsts;
for (i = 0; i < dst_cnt; i++) {
thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
if (!thread->dsts[i])
goto err_dstbuf;
}
thread->dsts[i] = NULL;
set_user_nice(current, 10);
/*
* src buffers are freed by the DMAEngine code with dma_unmap_single()
* dst buffers are freed by ourselves below
*/
flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT
| DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
while (!kthread_should_stop()
&& !(params->iterations && total_tests >= params->iterations)) {
struct dma_async_tx_descriptor *tx = NULL;
dma_addr_t dma_srcs[src_cnt];
dma_addr_t dma_dsts[dst_cnt];
u8 align = 0;
total_tests++;
/* honor alignment restrictions */
if (thread->type == DMA_MEMCPY)
align = dev->copy_align;
else if (thread->type == DMA_XOR)
align = dev->xor_align;
else if (thread->type == DMA_PQ)
align = dev->pq_align;
if (1 << align > params->buf_size) {
pr_err("%u-byte buffer too small for %d-byte alignment\n",
params->buf_size, 1 << align);
break;
}
len = dmatest_random() % params->buf_size + 1;
len = (len >> align) << align;
if (!len)
len = 1 << align;
src_off = dmatest_random() % (params->buf_size - len + 1);
dst_off = dmatest_random() % (params->buf_size - len + 1);
src_off = (src_off >> align) << align;
dst_off = (dst_off >> align) << align;
dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size);
dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size);
for (i = 0; i < src_cnt; i++) {
u8 *buf = thread->srcs[i] + src_off;
dma_srcs[i] = dma_map_single(dev->dev, buf, len,
DMA_TO_DEVICE);
ret = dma_mapping_error(dev->dev, dma_srcs[i]);
if (ret) {
unmap_src(dev->dev, dma_srcs, len, i);
thread_result_add(info, result,
DMATEST_ET_MAP_SRC,
total_tests, src_off, dst_off,
len, ret);
failed_tests++;
continue;
}
}
/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
for (i = 0; i < dst_cnt; i++) {
dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
params->buf_size,
DMA_BIDIRECTIONAL);
ret = dma_mapping_error(dev->dev, dma_dsts[i]);
if (ret) {
unmap_src(dev->dev, dma_srcs, len, src_cnt);
unmap_dst(dev->dev, dma_dsts, params->buf_size,
i);
thread_result_add(info, result,
DMATEST_ET_MAP_DST,
total_tests, src_off, dst_off,
len, ret);
failed_tests++;
continue;
}
}
if (thread->type == DMA_MEMCPY)
tx = dev->device_prep_dma_memcpy(chan,
dma_dsts[0] + dst_off,
dma_srcs[0], len,
flags);
else if (thread->type == DMA_XOR)
tx = dev->device_prep_dma_xor(chan,
dma_dsts[0] + dst_off,
dma_srcs, src_cnt,
len, flags);
else if (thread->type == DMA_PQ) {
dma_addr_t dma_pq[dst_cnt];
for (i = 0; i < dst_cnt; i++)
dma_pq[i] = dma_dsts[i] + dst_off;
tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
src_cnt, pq_coefs,
len, flags);
}
if (!tx) {
unmap_src(dev->dev, dma_srcs, len, src_cnt);
unmap_dst(dev->dev, dma_dsts, params->buf_size,
dst_cnt);
thread_result_add(info, result, DMATEST_ET_PREP,
total_tests, src_off, dst_off,
len, 0);
msleep(100);
failed_tests++;
continue;
}
done.done = false;
tx->callback = dmatest_callback;
tx->callback_param = &done;
cookie = tx->tx_submit(tx);
if (dma_submit_error(cookie)) {
thread_result_add(info, result, DMATEST_ET_SUBMIT,
total_tests, src_off, dst_off,
len, cookie);
msleep(100);
failed_tests++;
continue;
}
dma_async_issue_pending(chan);
wait_event_freezable_timeout(done_wait, done.done,
msecs_to_jiffies(params->timeout));
status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
if (!done.done) {
/*
* We're leaving the timed out dma operation with
* dangling pointer to done_wait. To make this
* correct, we'll need to allocate wait_done for
* each test iteration and perform "who's gonna
* free it this time?" dancing. For now, just
* leave it dangling.
*/
thread_result_add(info, result, DMATEST_ET_TIMEOUT,
total_tests, src_off, dst_off,
len, 0);
failed_tests++;
continue;
} else if (status != DMA_SUCCESS) {
enum dmatest_error_type type = (status == DMA_ERROR) ?
DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS;
thread_result_add(info, result, type,
total_tests, src_off, dst_off,
len, status);
failed_tests++;
continue;
}
/* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */
unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
error_count = 0;
pr_debug("%s: verifying source buffer...\n", thread_name);
error_count += verify_result_add(info, result, total_tests,
src_off, dst_off, len, thread->srcs, -1,
0, PATTERN_SRC, true);
error_count += verify_result_add(info, result, total_tests,
src_off, dst_off, len, thread->srcs, 0,
src_off, PATTERN_SRC | PATTERN_COPY, true);
error_count += verify_result_add(info, result, total_tests,
src_off, dst_off, len, thread->srcs, 1,
src_off + len, PATTERN_SRC, true);
pr_debug("%s: verifying dest buffer...\n", thread_name);
error_count += verify_result_add(info, result, total_tests,
src_off, dst_off, len, thread->dsts, -1,
0, PATTERN_DST, false);
error_count += verify_result_add(info, result, total_tests,
src_off, dst_off, len, thread->dsts, 0,
src_off, PATTERN_SRC | PATTERN_COPY, false);
error_count += verify_result_add(info, result, total_tests,
src_off, dst_off, len, thread->dsts, 1,
dst_off + len, PATTERN_DST, false);
if (error_count) {
thread_result_add(info, result, DMATEST_ET_VERIFY,
total_tests, src_off, dst_off,
len, error_count);
failed_tests++;
} else {
thread_result_add(info, result, DMATEST_ET_OK,
total_tests, src_off, dst_off,
len, 0);
}
}
ret = 0;
for (i = 0; thread->dsts[i]; i++)
kfree(thread->dsts[i]);
err_dstbuf:
kfree(thread->dsts);
err_dsts:
for (i = 0; thread->srcs[i]; i++)
kfree(thread->srcs[i]);
err_srcbuf:
kfree(thread->srcs);
err_srcs:
kfree(pq_coefs);
err_thread_type:
pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
thread_name, total_tests, failed_tests, ret);
/* terminate all transfers on specified channels */
if (ret)
dmaengine_terminate_all(chan);
thread->done = true;
if (params->iterations > 0)
while (!kthread_should_stop()) {
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
interruptible_sleep_on(&wait_dmatest_exit);
}
return ret;
}
static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
{
struct dmatest_thread *thread;
struct dmatest_thread *_thread;
int ret;
list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
ret = kthread_stop(thread->task);
pr_debug("dmatest: thread %s exited with status %d\n",
thread->task->comm, ret);
list_del(&thread->node);
kfree(thread);
}
/* terminate all transfers on specified channels */
dmaengine_terminate_all(dtc->chan);
kfree(dtc);
}
static int dmatest_add_threads(struct dmatest_info *info,
struct dmatest_chan *dtc, enum dma_transaction_type type)
{
struct dmatest_params *params = &info->params;
struct dmatest_thread *thread;
struct dma_chan *chan = dtc->chan;
char *op;
unsigned int i;
if (type == DMA_MEMCPY)
op = "copy";
else if (type == DMA_XOR)
op = "xor";
else if (type == DMA_PQ)
op = "pq";
else
return -EINVAL;
for (i = 0; i < params->threads_per_chan; i++) {
thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
if (!thread) {
pr_warning("dmatest: No memory for %s-%s%u\n",
dma_chan_name(chan), op, i);
break;
}
thread->info = info;
thread->chan = dtc->chan;
thread->type = type;
smp_wmb();
thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
dma_chan_name(chan), op, i);
if (IS_ERR(thread->task)) {
pr_warning("dmatest: Failed to run thread %s-%s%u\n",
dma_chan_name(chan), op, i);
kfree(thread);
break;
}
/* srcbuf and dstbuf are allocated by the thread itself */
list_add_tail(&thread->node, &dtc->threads);
}
return i;
}
static int dmatest_add_channel(struct dmatest_info *info,
struct dma_chan *chan)
{
struct dmatest_chan *dtc;
struct dma_device *dma_dev = chan->device;
unsigned int thread_count = 0;
int cnt;
dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
if (!dtc) {
pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
return -ENOMEM;
}
dtc->chan = chan;
INIT_LIST_HEAD(&dtc->threads);
if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
thread_count += cnt > 0 ? cnt : 0;
}
if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
cnt = dmatest_add_threads(info, dtc, DMA_XOR);
thread_count += cnt > 0 ? cnt : 0;
}
if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
cnt = dmatest_add_threads(info, dtc, DMA_PQ);
thread_count += cnt > 0 ? cnt : 0;
}
pr_info("dmatest: Started %u threads using %s\n",
thread_count, dma_chan_name(chan));
list_add_tail(&dtc->node, &info->channels);
info->nr_channels++;
return 0;
}
static bool filter(struct dma_chan *chan, void *param)
{
struct dmatest_params *params = param;
if (!dmatest_match_channel(params, chan) ||
!dmatest_match_device(params, chan->device))
return false;
else
return true;
}
static int __run_threaded_test(struct dmatest_info *info)
{
dma_cap_mask_t mask;
struct dma_chan *chan;
struct dmatest_params *params = &info->params;
int err = 0;
dma_cap_zero(mask);
dma_cap_set(DMA_MEMCPY, mask);
for (;;) {
chan = dma_request_channel(mask, filter, params);
if (chan) {
err = dmatest_add_channel(info, chan);
if (err) {
dma_release_channel(chan);
break; /* add_channel failed, punt */
}
} else
break; /* no more channels available */
if (params->max_channels &&
info->nr_channels >= params->max_channels)
break; /* we have all we need */
}
return err;
}
#ifndef MODULE
static int run_threaded_test(struct dmatest_info *info)
{
int ret;
mutex_lock(&info->lock);
ret = __run_threaded_test(info);
mutex_unlock(&info->lock);
return ret;
}
#endif
static void __stop_threaded_test(struct dmatest_info *info)
{
struct dmatest_chan *dtc, *_dtc;
struct dma_chan *chan;
list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
list_del(&dtc->node);
chan = dtc->chan;
dmatest_cleanup_channel(dtc);
pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan));
dma_release_channel(chan);
}
info->nr_channels = 0;
}
static void stop_threaded_test(struct dmatest_info *info)
{
mutex_lock(&info->lock);
__stop_threaded_test(info);
mutex_unlock(&info->lock);
}
static int __restart_threaded_test(struct dmatest_info *info, bool run)
{
struct dmatest_params *params = &info->params;
/* Stop any running test first */
__stop_threaded_test(info);
if (run == false)
return 0;
/* Clear results from previous run */
result_free(info, NULL);
/* Copy test parameters */
memcpy(params, &info->dbgfs_params, sizeof(*params));
/* Run test with new parameters */
return __run_threaded_test(info);
}
static bool __is_threaded_test_run(struct dmatest_info *info)
{
struct dmatest_chan *dtc;
list_for_each_entry(dtc, &info->channels, node) {
struct dmatest_thread *thread;
list_for_each_entry(thread, &dtc->threads, node) {
if (!thread->done)
return true;
}
}
return false;
}
static ssize_t dtf_write_string(void *to, size_t available, loff_t *ppos,
const void __user *from, size_t count)
{
char tmp[20];
ssize_t len;
len = simple_write_to_buffer(tmp, sizeof(tmp) - 1, ppos, from, count);
if (len >= 0) {
tmp[len] = '\0';
strlcpy(to, strim(tmp), available);
}
return len;
}
static ssize_t dtf_read_channel(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
struct dmatest_info *info = file->private_data;
return simple_read_from_buffer(buf, count, ppos,
info->dbgfs_params.channel,
strlen(info->dbgfs_params.channel));
}
static ssize_t dtf_write_channel(struct file *file, const char __user *buf,
size_t size, loff_t *ppos)
{
struct dmatest_info *info = file->private_data;
return dtf_write_string(info->dbgfs_params.channel,
sizeof(info->dbgfs_params.channel),
ppos, buf, size);
}
static const struct file_operations dtf_channel_fops = {
.read = dtf_read_channel,
.write = dtf_write_channel,
.open = simple_open,
.llseek = default_llseek,
};
static ssize_t dtf_read_device(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
struct dmatest_info *info = file->private_data;
return simple_read_from_buffer(buf, count, ppos,
info->dbgfs_params.device,
strlen(info->dbgfs_params.device));
}
static ssize_t dtf_write_device(struct file *file, const char __user *buf,
size_t size, loff_t *ppos)
{
struct dmatest_info *info = file->private_data;
return dtf_write_string(info->dbgfs_params.device,
sizeof(info->dbgfs_params.device),
ppos, buf, size);
}
static const struct file_operations dtf_device_fops = {
.read = dtf_read_device,
.write = dtf_write_device,
.open = simple_open,
.llseek = default_llseek,
};
static ssize_t dtf_read_run(struct file *file, char __user *user_buf,
size_t count, loff_t *ppos)
{
struct dmatest_info *info = file->private_data;
char buf[3];
mutex_lock(&info->lock);
if (__is_threaded_test_run(info)) {
buf[0] = 'Y';
} else {
__stop_threaded_test(info);
buf[0] = 'N';
}
mutex_unlock(&info->lock);
buf[1] = '\n';
buf[2] = 0x00;
return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
}
static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
size_t count, loff_t *ppos)
{
struct dmatest_info *info = file->private_data;
char buf[16];
bool bv;
int ret = 0;
if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1))))
return -EFAULT;
if (strtobool(buf, &bv) == 0) {
mutex_lock(&info->lock);
if (__is_threaded_test_run(info))
ret = -EBUSY;
else
ret = __restart_threaded_test(info, bv);
mutex_unlock(&info->lock);
}
return ret ? ret : count;
}
static const struct file_operations dtf_run_fops = {
.read = dtf_read_run,
.write = dtf_write_run,
.open = simple_open,
.llseek = default_llseek,
};
static int dtf_results_show(struct seq_file *sf, void *data)
{
struct dmatest_info *info = sf->private;
struct dmatest_result *result;
struct dmatest_thread_result *tr;
unsigned int i;
mutex_lock(&info->results_lock);
list_for_each_entry(result, &info->results, node) {
list_for_each_entry(tr, &result->results, node) {
seq_printf(sf, "%s\n",
thread_result_get(result->name, tr));
if (tr->type == DMATEST_ET_VERIFY_BUF) {
for (i = 0; i < tr->vr->error_count; i++) {
seq_printf(sf, "\t%s\n",
verify_result_get_one(tr->vr, i));
}
}
}
}
mutex_unlock(&info->results_lock);
return 0;
}
static int dtf_results_open(struct inode *inode, struct file *file)
{
return single_open(file, dtf_results_show, inode->i_private);
}
static const struct file_operations dtf_results_fops = {
.open = dtf_results_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
static int dmatest_register_dbgfs(struct dmatest_info *info)
{
struct dentry *d;
struct dmatest_params *params = &info->dbgfs_params;
int ret = -ENOMEM;
d = debugfs_create_dir("dmatest", NULL);
if (IS_ERR(d))
return PTR_ERR(d);
if (!d)
goto err_root;
info->root = d;
/* Copy initial values */
memcpy(params, &info->params, sizeof(*params));
/* Test parameters */
d = debugfs_create_u32("test_buf_size", S_IWUSR | S_IRUGO, info->root,
(u32 *)&params->buf_size);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_file("channel", S_IRUGO | S_IWUSR, info->root,
info, &dtf_channel_fops);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_file("device", S_IRUGO | S_IWUSR, info->root,
info, &dtf_device_fops);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_u32("threads_per_chan", S_IWUSR | S_IRUGO, info->root,
(u32 *)&params->threads_per_chan);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_u32("max_channels", S_IWUSR | S_IRUGO, info->root,
(u32 *)&params->max_channels);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_u32("iterations", S_IWUSR | S_IRUGO, info->root,
(u32 *)&params->iterations);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_u32("xor_sources", S_IWUSR | S_IRUGO, info->root,
(u32 *)&params->xor_sources);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_u32("pq_sources", S_IWUSR | S_IRUGO, info->root,
(u32 *)&params->pq_sources);
if (IS_ERR_OR_NULL(d))
goto err_node;
d = debugfs_create_u32("timeout", S_IWUSR | S_IRUGO, info->root,
(u32 *)&params->timeout);
if (IS_ERR_OR_NULL(d))
goto err_node;
/* Run or stop threaded test */
d = debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root,
info, &dtf_run_fops);
if (IS_ERR_OR_NULL(d))
goto err_node;
/* Results of test in progress */
d = debugfs_create_file("results", S_IRUGO, info->root, info,
&dtf_results_fops);
if (IS_ERR_OR_NULL(d))
goto err_node;
return 0;
err_node:
debugfs_remove_recursive(info->root);
err_root:
pr_err("dmatest: Failed to initialize debugfs\n");
return ret;
}
static int __init dmatest_init(void)
{
struct dmatest_info *info = &test_info;
struct dmatest_params *params = &info->params;
int ret;
memset(info, 0, sizeof(*info));
mutex_init(&info->lock);
INIT_LIST_HEAD(&info->channels);
mutex_init(&info->results_lock);
INIT_LIST_HEAD(&info->results);
/* Set default parameters */
params->buf_size = test_buf_size;
strlcpy(params->channel, test_channel, sizeof(params->channel));
strlcpy(params->device, test_device, sizeof(params->device));
params->threads_per_chan = threads_per_chan;
params->max_channels = max_channels;
params->iterations = iterations;
params->xor_sources = xor_sources;
params->pq_sources = pq_sources;
params->timeout = timeout;
ret = dmatest_register_dbgfs(info);
if (ret)
return ret;
#ifdef MODULE
return 0;
#else
return run_threaded_test(info);
#endif
}
/* when compiled-in wait for drivers to load first */
late_initcall(dmatest_init);
static void __exit dmatest_exit(void)
{
struct dmatest_info *info = &test_info;
debugfs_remove_recursive(info->root);
stop_threaded_test(info);
result_free(info, NULL);
}
module_exit(dmatest_exit);
MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
MODULE_LICENSE("GPL v2");