linux/drivers/crypto/inside-secure/safexcel_ring.c
Pascal van Leeuwen 098e51e517 crypto: inside-secure - Fix Unable to fit even 1 command desc error w/ EIP97
Due to the additions of support for modes like AES-CCM and AES-GCM, which
require large command tokens, the size of the descriptor has grown such that
it now does not fit into the descriptor cache of a standard EIP97 anymore.
This means that the driver no longer works on the Marvell Armada 3700LP chip
(as used on e.g. Espressobin) that it has always supported.
Additionally, performance on EIP197's like Marvell A8K may also degrade
due to being able to fit less descriptors in the on-chip cache.
Putting these tokens into the descriptor was really a hack and not how the
design was supposed to be used - resource allocation did not account for it.

So what this patch does, is move the command token out of the descriptor.
To avoid having to allocate buffers on the fly for these command tokens,
they are stuffed in a "shadow ring", which is a circular buffer of fixed
size blocks that runs in lock-step with the descriptor ring. i.e. there is
one token block per descriptor. The descriptor ring itself is then pre-
populated with the pointers to these token blocks so these do not need to
be filled in when building the descriptors later.

Signed-off-by: Pascal van Leeuwen <pvanleeuwen@rambus.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2019-12-20 14:58:33 +08:00

254 lines
6.5 KiB
C

// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2017 Marvell
*
* Antoine Tenart <antoine.tenart@free-electrons.com>
*/
#include <linux/dma-mapping.h>
#include <linux/spinlock.h>
#include "safexcel.h"
int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *cdr,
struct safexcel_desc_ring *rdr)
{
int i;
struct safexcel_command_desc *cdesc;
dma_addr_t atok;
/* Actual command descriptor ring */
cdr->offset = priv->config.cd_offset;
cdr->base = dmam_alloc_coherent(priv->dev,
cdr->offset * EIP197_DEFAULT_RING_SIZE,
&cdr->base_dma, GFP_KERNEL);
if (!cdr->base)
return -ENOMEM;
cdr->write = cdr->base;
cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
cdr->read = cdr->base;
/* Command descriptor shadow ring for storing additional token data */
cdr->shoffset = priv->config.cdsh_offset;
cdr->shbase = dmam_alloc_coherent(priv->dev,
cdr->shoffset *
EIP197_DEFAULT_RING_SIZE,
&cdr->shbase_dma, GFP_KERNEL);
if (!cdr->shbase)
return -ENOMEM;
cdr->shwrite = cdr->shbase;
cdr->shbase_end = cdr->shbase + cdr->shoffset *
(EIP197_DEFAULT_RING_SIZE - 1);
/*
* Populate command descriptors with physical pointers to shadow descs.
* Note that we only need to do this once if we don't overwrite them.
*/
cdesc = cdr->base;
atok = cdr->shbase_dma;
for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) {
cdesc->atok_lo = lower_32_bits(atok);
cdesc->atok_hi = upper_32_bits(atok);
cdesc = (void *)cdesc + cdr->offset;
atok += cdr->shoffset;
}
rdr->offset = priv->config.rd_offset;
/* Use shoffset for result token offset here */
rdr->shoffset = priv->config.res_offset;
rdr->base = dmam_alloc_coherent(priv->dev,
rdr->offset * EIP197_DEFAULT_RING_SIZE,
&rdr->base_dma, GFP_KERNEL);
if (!rdr->base)
return -ENOMEM;
rdr->write = rdr->base;
rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
rdr->read = rdr->base;
return 0;
}
inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)
{
return (atomic_inc_return(&priv->ring_used) % priv->config.rings);
}
static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *ring,
bool first,
struct safexcel_token **atoken)
{
void *ptr = ring->write;
if (first)
*atoken = ring->shwrite;
if ((ring->write == ring->read - ring->offset) ||
(ring->read == ring->base && ring->write == ring->base_end))
return ERR_PTR(-ENOMEM);
if (ring->write == ring->base_end) {
ring->write = ring->base;
ring->shwrite = ring->shbase;
} else {
ring->write += ring->offset;
ring->shwrite += ring->shoffset;
}
return ptr;
}
static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *ring,
struct result_data_desc **rtoken)
{
void *ptr = ring->write;
/* Result token at relative offset shoffset */
*rtoken = ring->write + ring->shoffset;
if ((ring->write == ring->read - ring->offset) ||
(ring->read == ring->base && ring->write == ring->base_end))
return ERR_PTR(-ENOMEM);
if (ring->write == ring->base_end)
ring->write = ring->base;
else
ring->write += ring->offset;
return ptr;
}
void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *ring)
{
void *ptr = ring->read;
if (ring->write == ring->read)
return ERR_PTR(-ENOENT);
if (ring->read == ring->base_end)
ring->read = ring->base;
else
ring->read += ring->offset;
return ptr;
}
inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv,
int ring)
{
struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
return rdr->read;
}
inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
int ring)
{
struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
return (rdr->read - rdr->base) / rdr->offset;
}
inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc)
{
struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
return ((void *)rdesc - rdr->base) / rdr->offset;
}
void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *ring)
{
if (ring->write == ring->read)
return;
if (ring->write == ring->base) {
ring->write = ring->base_end;
ring->shwrite = ring->shbase_end;
} else {
ring->write -= ring->offset;
ring->shwrite -= ring->shoffset;
}
}
struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
int ring_id,
bool first, bool last,
dma_addr_t data, u32 data_len,
u32 full_data_len,
dma_addr_t context,
struct safexcel_token **atoken)
{
struct safexcel_command_desc *cdesc;
cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr,
first, atoken);
if (IS_ERR(cdesc))
return cdesc;
cdesc->particle_size = data_len;
cdesc->rsvd0 = 0;
cdesc->last_seg = last;
cdesc->first_seg = first;
cdesc->additional_cdata_size = 0;
cdesc->rsvd1 = 0;
cdesc->data_lo = lower_32_bits(data);
cdesc->data_hi = upper_32_bits(data);
if (first) {
/*
* Note that the length here MUST be >0 or else the EIP(1)97
* may hang. Newer EIP197 firmware actually incorporates this
* fix already, but that doesn't help the EIP97 and we may
* also be running older firmware.
*/
cdesc->control_data.packet_length = full_data_len ?: 1;
cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |
EIP197_OPTION_64BIT_CTX |
EIP197_OPTION_CTX_CTRL_IN_CMD |
EIP197_OPTION_RC_AUTO;
cdesc->control_data.type = EIP197_TYPE_BCLA;
cdesc->control_data.context_lo = lower_32_bits(context) |
EIP197_CONTEXT_SMALL;
cdesc->control_data.context_hi = upper_32_bits(context);
}
return cdesc;
}
struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,
int ring_id,
bool first, bool last,
dma_addr_t data, u32 len)
{
struct safexcel_result_desc *rdesc;
struct result_data_desc *rtoken;
rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr,
&rtoken);
if (IS_ERR(rdesc))
return rdesc;
rdesc->particle_size = len;
rdesc->rsvd0 = 0;
rdesc->descriptor_overflow = 0;
rdesc->buffer_overflow = 0;
rdesc->last_seg = last;
rdesc->first_seg = first;
rdesc->result_size = EIP197_RD64_RESULT_SIZE;
rdesc->rsvd1 = 0;
rdesc->data_lo = lower_32_bits(data);
rdesc->data_hi = upper_32_bits(data);
/* Clear length & error code in result token */
rtoken->packet_length = 0;
rtoken->error_code = 0;
return rdesc;
}