linux/drivers/iommu/ipmmu-vmsa.c
Robin Murphy d1e5f26f14 iommu/io-pgtable-arm: Rationalise TTBRn handling
TTBR1 values have so far been redundant since no users implement any
support for split address spaces. Crucially, though, one of the main
reasons for wanting to do so is to be able to manage each half entirely
independently, e.g. context-switching one set of mappings without
disturbing the other. Thus it seems unlikely that tying two tables
together in a single io_pgtable_cfg would ever be particularly desirable
or useful.

Streamline the configs to just a single conceptual TTBR value
representing the allocated table. This paves the way for future users to
support split address spaces by simply allocating a table and dealing
with the detailed TTBRn logistics themselves.

Tested-by: Jordan Crouse <jcrouse@codeaurora.org>
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
[will: Drop change to ttbr value]
Signed-off-by: Will Deacon <will@kernel.org>
2020-01-10 15:39:23 +00:00

1220 lines
31 KiB
C

// SPDX-License-Identifier: GPL-2.0
/*
* IOMMU API for Renesas VMSA-compatible IPMMU
* Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
*
* Copyright (C) 2014 Renesas Electronics Corporation
*/
#include <linux/bitmap.h>
#include <linux/delay.h>
#include <linux/dma-iommu.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/io-pgtable.h>
#include <linux/iommu.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/of_iommu.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/sys_soc.h>
#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
#include <asm/dma-iommu.h>
#include <asm/pgalloc.h>
#else
#define arm_iommu_create_mapping(...) NULL
#define arm_iommu_attach_device(...) -ENODEV
#define arm_iommu_release_mapping(...) do {} while (0)
#define arm_iommu_detach_device(...) do {} while (0)
#endif
#define IPMMU_CTX_MAX 8U
#define IPMMU_CTX_INVALID -1
#define IPMMU_UTLB_MAX 48U
struct ipmmu_features {
bool use_ns_alias_offset;
bool has_cache_leaf_nodes;
unsigned int number_of_contexts;
unsigned int num_utlbs;
bool setup_imbuscr;
bool twobit_imttbcr_sl0;
bool reserved_context;
bool cache_snoop;
unsigned int ctx_offset_base;
unsigned int ctx_offset_stride;
unsigned int utlb_offset_base;
};
struct ipmmu_vmsa_device {
struct device *dev;
void __iomem *base;
struct iommu_device iommu;
struct ipmmu_vmsa_device *root;
const struct ipmmu_features *features;
unsigned int num_ctx;
spinlock_t lock; /* Protects ctx and domains[] */
DECLARE_BITMAP(ctx, IPMMU_CTX_MAX);
struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX];
s8 utlb_ctx[IPMMU_UTLB_MAX];
struct iommu_group *group;
struct dma_iommu_mapping *mapping;
};
struct ipmmu_vmsa_domain {
struct ipmmu_vmsa_device *mmu;
struct iommu_domain io_domain;
struct io_pgtable_cfg cfg;
struct io_pgtable_ops *iop;
unsigned int context_id;
struct mutex mutex; /* Protects mappings */
};
static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
{
return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
}
static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
return fwspec ? fwspec->iommu_priv : NULL;
}
#define TLB_LOOP_TIMEOUT 100 /* 100us */
/* -----------------------------------------------------------------------------
* Registers Definition
*/
#define IM_NS_ALIAS_OFFSET 0x800
/* MMU "context" registers */
#define IMCTR 0x0000 /* R-Car Gen2/3 */
#define IMCTR_INTEN (1 << 2) /* R-Car Gen2/3 */
#define IMCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */
#define IMCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */
#define IMTTBCR 0x0008 /* R-Car Gen2/3 */
#define IMTTBCR_EAE (1 << 31) /* R-Car Gen2/3 */
#define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */
#define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */
#define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */
#define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */
#define IMTTBCR_SL0_LVL_1 (1 << 4) /* R-Car Gen2 only */
#define IMBUSCR 0x000c /* R-Car Gen2 only */
#define IMBUSCR_DVM (1 << 2) /* R-Car Gen2 only */
#define IMBUSCR_BUSSEL_MASK (3 << 0) /* R-Car Gen2 only */
#define IMTTLBR0 0x0010 /* R-Car Gen2/3 */
#define IMTTUBR0 0x0014 /* R-Car Gen2/3 */
#define IMSTR 0x0020 /* R-Car Gen2/3 */
#define IMSTR_MHIT (1 << 4) /* R-Car Gen2/3 */
#define IMSTR_ABORT (1 << 2) /* R-Car Gen2/3 */
#define IMSTR_PF (1 << 1) /* R-Car Gen2/3 */
#define IMSTR_TF (1 << 0) /* R-Car Gen2/3 */
#define IMMAIR0 0x0028 /* R-Car Gen2/3 */
#define IMELAR 0x0030 /* R-Car Gen2/3, IMEAR on R-Car Gen2 */
#define IMEUAR 0x0034 /* R-Car Gen3 only */
/* uTLB registers */
#define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n))
#define IMUCTR0(n) (0x0300 + ((n) * 16)) /* R-Car Gen2/3 */
#define IMUCTR32(n) (0x0600 + (((n) - 32) * 16)) /* R-Car Gen3 only */
#define IMUCTR_TTSEL_MMU(n) ((n) << 4) /* R-Car Gen2/3 */
#define IMUCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */
#define IMUCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */
#define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n))
#define IMUASID0(n) (0x0308 + ((n) * 16)) /* R-Car Gen2/3 */
#define IMUASID32(n) (0x0608 + (((n) - 32) * 16)) /* R-Car Gen3 only */
/* -----------------------------------------------------------------------------
* Root device handling
*/
static struct platform_driver ipmmu_driver;
static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu)
{
return mmu->root == mmu;
}
static int __ipmmu_check_device(struct device *dev, void *data)
{
struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);
struct ipmmu_vmsa_device **rootp = data;
if (ipmmu_is_root(mmu))
*rootp = mmu;
return 0;
}
static struct ipmmu_vmsa_device *ipmmu_find_root(void)
{
struct ipmmu_vmsa_device *root = NULL;
return driver_for_each_device(&ipmmu_driver.driver, NULL, &root,
__ipmmu_check_device) == 0 ? root : NULL;
}
/* -----------------------------------------------------------------------------
* Read/Write Access
*/
static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)
{
return ioread32(mmu->base + offset);
}
static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,
u32 data)
{
iowrite32(data, mmu->base + offset);
}
static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu,
unsigned int context_id, unsigned int reg)
{
return mmu->features->ctx_offset_base +
context_id * mmu->features->ctx_offset_stride + reg;
}
static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu,
unsigned int context_id, unsigned int reg)
{
return ipmmu_read(mmu, ipmmu_ctx_reg(mmu, context_id, reg));
}
static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu,
unsigned int context_id, unsigned int reg, u32 data)
{
ipmmu_write(mmu, ipmmu_ctx_reg(mmu, context_id, reg), data);
}
static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain,
unsigned int reg)
{
return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg);
}
static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain,
unsigned int reg, u32 data)
{
ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);
}
static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain,
unsigned int reg, u32 data)
{
if (domain->mmu != domain->mmu->root)
ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data);
ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);
}
static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg)
{
return mmu->features->utlb_offset_base + reg;
}
static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu,
unsigned int utlb, u32 data)
{
ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUASID(utlb)), data);
}
static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu,
unsigned int utlb, u32 data)
{
ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data);
}
/* -----------------------------------------------------------------------------
* TLB and microTLB Management
*/
/* Wait for any pending TLB invalidations to complete */
static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain)
{
unsigned int count = 0;
while (ipmmu_ctx_read_root(domain, IMCTR) & IMCTR_FLUSH) {
cpu_relax();
if (++count == TLB_LOOP_TIMEOUT) {
dev_err_ratelimited(domain->mmu->dev,
"TLB sync timed out -- MMU may be deadlocked\n");
return;
}
udelay(1);
}
}
static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain)
{
u32 reg;
reg = ipmmu_ctx_read_root(domain, IMCTR);
reg |= IMCTR_FLUSH;
ipmmu_ctx_write_all(domain, IMCTR, reg);
ipmmu_tlb_sync(domain);
}
/*
* Enable MMU translation for the microTLB.
*/
static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,
unsigned int utlb)
{
struct ipmmu_vmsa_device *mmu = domain->mmu;
/*
* TODO: Reference-count the microTLB as several bus masters can be
* connected to the same microTLB.
*/
/* TODO: What should we set the ASID to ? */
ipmmu_imuasid_write(mmu, utlb, 0);
/* TODO: Do we need to flush the microTLB ? */
ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) |
IMUCTR_FLUSH | IMUCTR_MMUEN);
mmu->utlb_ctx[utlb] = domain->context_id;
}
/*
* Disable MMU translation for the microTLB.
*/
static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,
unsigned int utlb)
{
struct ipmmu_vmsa_device *mmu = domain->mmu;
ipmmu_imuctr_write(mmu, utlb, 0);
mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;
}
static void ipmmu_tlb_flush_all(void *cookie)
{
struct ipmmu_vmsa_domain *domain = cookie;
ipmmu_tlb_invalidate(domain);
}
static void ipmmu_tlb_flush(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
ipmmu_tlb_flush_all(cookie);
}
static const struct iommu_flush_ops ipmmu_flush_ops = {
.tlb_flush_all = ipmmu_tlb_flush_all,
.tlb_flush_walk = ipmmu_tlb_flush,
.tlb_flush_leaf = ipmmu_tlb_flush,
};
/* -----------------------------------------------------------------------------
* Domain/Context Management
*/
static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,
struct ipmmu_vmsa_domain *domain)
{
unsigned long flags;
int ret;
spin_lock_irqsave(&mmu->lock, flags);
ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx);
if (ret != mmu->num_ctx) {
mmu->domains[ret] = domain;
set_bit(ret, mmu->ctx);
} else
ret = -EBUSY;
spin_unlock_irqrestore(&mmu->lock, flags);
return ret;
}
static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
unsigned int context_id)
{
unsigned long flags;
spin_lock_irqsave(&mmu->lock, flags);
clear_bit(context_id, mmu->ctx);
mmu->domains[context_id] = NULL;
spin_unlock_irqrestore(&mmu->lock, flags);
}
static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain)
{
u64 ttbr;
u32 tmp;
/* TTBR0 */
ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr;
ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);
ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);
/*
* TTBCR
* We use long descriptors and allocate the whole 32-bit VA space to
* TTBR0.
*/
if (domain->mmu->features->twobit_imttbcr_sl0)
tmp = IMTTBCR_SL0_TWOBIT_LVL_1;
else
tmp = IMTTBCR_SL0_LVL_1;
if (domain->mmu->features->cache_snoop)
tmp |= IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
IMTTBCR_IRGN0_WB_WA;
ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp);
/* MAIR0 */
ipmmu_ctx_write_root(domain, IMMAIR0,
domain->cfg.arm_lpae_s1_cfg.mair);
/* IMBUSCR */
if (domain->mmu->features->setup_imbuscr)
ipmmu_ctx_write_root(domain, IMBUSCR,
ipmmu_ctx_read_root(domain, IMBUSCR) &
~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK));
/*
* IMSTR
* Clear all interrupt flags.
*/
ipmmu_ctx_write_root(domain, IMSTR, ipmmu_ctx_read_root(domain, IMSTR));
/*
* IMCTR
* Enable the MMU and interrupt generation. The long-descriptor
* translation table format doesn't use TEX remapping. Don't enable AF
* software management as we have no use for it. Flush the TLB as
* required when modifying the context registers.
*/
ipmmu_ctx_write_all(domain, IMCTR,
IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN);
}
static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
{
int ret;
/*
* Allocate the page table operations.
*
* VMSA states in section B3.6.3 "Control of Secure or Non-secure memory
* access, Long-descriptor format" that the NStable bit being set in a
* table descriptor will result in the NStable and NS bits of all child
* entries being ignored and considered as being set. The IPMMU seems
* not to comply with this, as it generates a secure access page fault
* if any of the NStable and NS bits isn't set when running in
* non-secure mode.
*/
domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS;
domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K;
domain->cfg.ias = 32;
domain->cfg.oas = 40;
domain->cfg.tlb = &ipmmu_flush_ops;
domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32);
domain->io_domain.geometry.force_aperture = true;
/*
* TODO: Add support for coherent walk through CCI with DVM and remove
* cache handling. For now, delegate it to the io-pgtable code.
*/
domain->cfg.coherent_walk = false;
domain->cfg.iommu_dev = domain->mmu->root->dev;
/*
* Find an unused context.
*/
ret = ipmmu_domain_allocate_context(domain->mmu->root, domain);
if (ret < 0)
return ret;
domain->context_id = ret;
domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
domain);
if (!domain->iop) {
ipmmu_domain_free_context(domain->mmu->root,
domain->context_id);
return -EINVAL;
}
ipmmu_domain_setup_context(domain);
return 0;
}
static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
{
if (!domain->mmu)
return;
/*
* Disable the context. Flush the TLB as required when modifying the
* context registers.
*
* TODO: Is TLB flush really needed ?
*/
ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH);
ipmmu_tlb_sync(domain);
ipmmu_domain_free_context(domain->mmu->root, domain->context_id);
}
/* -----------------------------------------------------------------------------
* Fault Handling
*/
static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)
{
const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF;
struct ipmmu_vmsa_device *mmu = domain->mmu;
unsigned long iova;
u32 status;
status = ipmmu_ctx_read_root(domain, IMSTR);
if (!(status & err_mask))
return IRQ_NONE;
iova = ipmmu_ctx_read_root(domain, IMELAR);
if (IS_ENABLED(CONFIG_64BIT))
iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32;
/*
* Clear the error status flags. Unlike traditional interrupt flag
* registers that must be cleared by writing 1, this status register
* seems to require 0. The error address register must be read before,
* otherwise its value will be 0.
*/
ipmmu_ctx_write_root(domain, IMSTR, 0);
/* Log fatal errors. */
if (status & IMSTR_MHIT)
dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n",
iova);
if (status & IMSTR_ABORT)
dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n",
iova);
if (!(status & (IMSTR_PF | IMSTR_TF)))
return IRQ_NONE;
/*
* Try to handle page faults and translation faults.
*
* TODO: We need to look up the faulty device based on the I/O VA. Use
* the IOMMU device for now.
*/
if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))
return IRQ_HANDLED;
dev_err_ratelimited(mmu->dev,
"Unhandled fault: status 0x%08x iova 0x%lx\n",
status, iova);
return IRQ_HANDLED;
}
static irqreturn_t ipmmu_irq(int irq, void *dev)
{
struct ipmmu_vmsa_device *mmu = dev;
irqreturn_t status = IRQ_NONE;
unsigned int i;
unsigned long flags;
spin_lock_irqsave(&mmu->lock, flags);
/*
* Check interrupts for all active contexts.
*/
for (i = 0; i < mmu->num_ctx; i++) {
if (!mmu->domains[i])
continue;
if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED)
status = IRQ_HANDLED;
}
spin_unlock_irqrestore(&mmu->lock, flags);
return status;
}
/* -----------------------------------------------------------------------------
* IOMMU Operations
*/
static struct iommu_domain *__ipmmu_domain_alloc(unsigned type)
{
struct ipmmu_vmsa_domain *domain;
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
mutex_init(&domain->mutex);
return &domain->io_domain;
}
static struct iommu_domain *ipmmu_domain_alloc(unsigned type)
{
struct iommu_domain *io_domain = NULL;
switch (type) {
case IOMMU_DOMAIN_UNMANAGED:
io_domain = __ipmmu_domain_alloc(type);
break;
case IOMMU_DOMAIN_DMA:
io_domain = __ipmmu_domain_alloc(type);
if (io_domain && iommu_get_dma_cookie(io_domain)) {
kfree(io_domain);
io_domain = NULL;
}
break;
}
return io_domain;
}
static void ipmmu_domain_free(struct iommu_domain *io_domain)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
/*
* Free the domain resources. We assume that all devices have already
* been detached.
*/
iommu_put_dma_cookie(io_domain);
ipmmu_domain_destroy_context(domain);
free_io_pgtable_ops(domain->iop);
kfree(domain);
}
static int ipmmu_attach_device(struct iommu_domain *io_domain,
struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
unsigned int i;
int ret = 0;
if (!mmu) {
dev_err(dev, "Cannot attach to IPMMU\n");
return -ENXIO;
}
mutex_lock(&domain->mutex);
if (!domain->mmu) {
/* The domain hasn't been used yet, initialize it. */
domain->mmu = mmu;
ret = ipmmu_domain_init_context(domain);
if (ret < 0) {
dev_err(dev, "Unable to initialize IPMMU context\n");
domain->mmu = NULL;
} else {
dev_info(dev, "Using IPMMU context %u\n",
domain->context_id);
}
} else if (domain->mmu != mmu) {
/*
* Something is wrong, we can't attach two devices using
* different IOMMUs to the same domain.
*/
dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
dev_name(mmu->dev), dev_name(domain->mmu->dev));
ret = -EINVAL;
} else
dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);
mutex_unlock(&domain->mutex);
if (ret < 0)
return ret;
for (i = 0; i < fwspec->num_ids; ++i)
ipmmu_utlb_enable(domain, fwspec->ids[i]);
return 0;
}
static void ipmmu_detach_device(struct iommu_domain *io_domain,
struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
unsigned int i;
for (i = 0; i < fwspec->num_ids; ++i)
ipmmu_utlb_disable(domain, fwspec->ids[i]);
/*
* TODO: Optimize by disabling the context when no device is attached.
*/
}
static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
if (!domain)
return -ENODEV;
return domain->iop->map(domain->iop, iova, paddr, size, prot);
}
static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
return domain->iop->unmap(domain->iop, iova, size, gather);
}
static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
if (domain->mmu)
ipmmu_tlb_flush_all(domain);
}
static void ipmmu_iotlb_sync(struct iommu_domain *io_domain,
struct iommu_iotlb_gather *gather)
{
ipmmu_flush_iotlb_all(io_domain);
}
static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
dma_addr_t iova)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
/* TODO: Is locking needed ? */
return domain->iop->iova_to_phys(domain->iop, iova);
}
static int ipmmu_init_platform_device(struct device *dev,
struct of_phandle_args *args)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct platform_device *ipmmu_pdev;
ipmmu_pdev = of_find_device_by_node(args->np);
if (!ipmmu_pdev)
return -ENODEV;
fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev);
return 0;
}
static const struct soc_device_attribute soc_rcar_gen3[] = {
{ .soc_id = "r8a774a1", },
{ .soc_id = "r8a774b1", },
{ .soc_id = "r8a774c0", },
{ .soc_id = "r8a7795", },
{ .soc_id = "r8a7796", },
{ .soc_id = "r8a77965", },
{ .soc_id = "r8a77970", },
{ .soc_id = "r8a77990", },
{ .soc_id = "r8a77995", },
{ /* sentinel */ }
};
static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = {
{ .soc_id = "r8a774b1", },
{ .soc_id = "r8a774c0", },
{ .soc_id = "r8a7795", .revision = "ES3.*" },
{ .soc_id = "r8a77965", },
{ .soc_id = "r8a77990", },
{ .soc_id = "r8a77995", },
{ /* sentinel */ }
};
static const char * const rcar_gen3_slave_whitelist[] = {
};
static bool ipmmu_slave_whitelist(struct device *dev)
{
unsigned int i;
/*
* For R-Car Gen3 use a white list to opt-in slave devices.
* For Other SoCs, this returns true anyway.
*/
if (!soc_device_match(soc_rcar_gen3))
return true;
/* Check whether this R-Car Gen3 can use the IPMMU correctly or not */
if (!soc_device_match(soc_rcar_gen3_whitelist))
return false;
/* Check whether this slave device can work with the IPMMU */
for (i = 0; i < ARRAY_SIZE(rcar_gen3_slave_whitelist); i++) {
if (!strcmp(dev_name(dev), rcar_gen3_slave_whitelist[i]))
return true;
}
/* Otherwise, do not allow use of IPMMU */
return false;
}
static int ipmmu_of_xlate(struct device *dev,
struct of_phandle_args *spec)
{
if (!ipmmu_slave_whitelist(dev))
return -ENODEV;
iommu_fwspec_add_ids(dev, spec->args, 1);
/* Initialize once - xlate() will call multiple times */
if (to_ipmmu(dev))
return 0;
return ipmmu_init_platform_device(dev, spec);
}
static int ipmmu_init_arm_mapping(struct device *dev)
{
struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
struct iommu_group *group;
int ret;
/* Create a device group and add the device to it. */
group = iommu_group_alloc();
if (IS_ERR(group)) {
dev_err(dev, "Failed to allocate IOMMU group\n");
return PTR_ERR(group);
}
ret = iommu_group_add_device(group, dev);
iommu_group_put(group);
if (ret < 0) {
dev_err(dev, "Failed to add device to IPMMU group\n");
return ret;
}
/*
* Create the ARM mapping, used by the ARM DMA mapping core to allocate
* VAs. This will allocate a corresponding IOMMU domain.
*
* TODO:
* - Create one mapping per context (TLB).
* - Make the mapping size configurable ? We currently use a 2GB mapping
* at a 1GB offset to ensure that NULL VAs will fault.
*/
if (!mmu->mapping) {
struct dma_iommu_mapping *mapping;
mapping = arm_iommu_create_mapping(&platform_bus_type,
SZ_1G, SZ_2G);
if (IS_ERR(mapping)) {
dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");
ret = PTR_ERR(mapping);
goto error;
}
mmu->mapping = mapping;
}
/* Attach the ARM VA mapping to the device. */
ret = arm_iommu_attach_device(dev, mmu->mapping);
if (ret < 0) {
dev_err(dev, "Failed to attach device to VA mapping\n");
goto error;
}
return 0;
error:
iommu_group_remove_device(dev);
if (mmu->mapping)
arm_iommu_release_mapping(mmu->mapping);
return ret;
}
static int ipmmu_add_device(struct device *dev)
{
struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
struct iommu_group *group;
int ret;
/*
* Only let through devices that have been verified in xlate()
*/
if (!mmu)
return -ENODEV;
if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) {
ret = ipmmu_init_arm_mapping(dev);
if (ret)
return ret;
} else {
group = iommu_group_get_for_dev(dev);
if (IS_ERR(group))
return PTR_ERR(group);
iommu_group_put(group);
}
iommu_device_link(&mmu->iommu, dev);
return 0;
}
static void ipmmu_remove_device(struct device *dev)
{
struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
iommu_device_unlink(&mmu->iommu, dev);
arm_iommu_detach_device(dev);
iommu_group_remove_device(dev);
}
static struct iommu_group *ipmmu_find_group(struct device *dev)
{
struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
struct iommu_group *group;
if (mmu->group)
return iommu_group_ref_get(mmu->group);
group = iommu_group_alloc();
if (!IS_ERR(group))
mmu->group = group;
return group;
}
static const struct iommu_ops ipmmu_ops = {
.domain_alloc = ipmmu_domain_alloc,
.domain_free = ipmmu_domain_free,
.attach_dev = ipmmu_attach_device,
.detach_dev = ipmmu_detach_device,
.map = ipmmu_map,
.unmap = ipmmu_unmap,
.flush_iotlb_all = ipmmu_flush_iotlb_all,
.iotlb_sync = ipmmu_iotlb_sync,
.iova_to_phys = ipmmu_iova_to_phys,
.add_device = ipmmu_add_device,
.remove_device = ipmmu_remove_device,
.device_group = ipmmu_find_group,
.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
.of_xlate = ipmmu_of_xlate,
};
/* -----------------------------------------------------------------------------
* Probe/remove and init
*/
static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)
{
unsigned int i;
/* Disable all contexts. */
for (i = 0; i < mmu->num_ctx; ++i)
ipmmu_ctx_write(mmu, i, IMCTR, 0);
}
static const struct ipmmu_features ipmmu_features_default = {
.use_ns_alias_offset = true,
.has_cache_leaf_nodes = false,
.number_of_contexts = 1, /* software only tested with one context */
.num_utlbs = 32,
.setup_imbuscr = true,
.twobit_imttbcr_sl0 = false,
.reserved_context = false,
.cache_snoop = true,
.ctx_offset_base = 0,
.ctx_offset_stride = 0x40,
.utlb_offset_base = 0,
};
static const struct ipmmu_features ipmmu_features_rcar_gen3 = {
.use_ns_alias_offset = false,
.has_cache_leaf_nodes = true,
.number_of_contexts = 8,
.num_utlbs = 48,
.setup_imbuscr = false,
.twobit_imttbcr_sl0 = true,
.reserved_context = true,
.cache_snoop = false,
.ctx_offset_base = 0,
.ctx_offset_stride = 0x40,
.utlb_offset_base = 0,
};
static const struct of_device_id ipmmu_of_ids[] = {
{
.compatible = "renesas,ipmmu-vmsa",
.data = &ipmmu_features_default,
}, {
.compatible = "renesas,ipmmu-r8a774a1",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a774b1",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a774c0",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a7795",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a7796",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a77965",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a77970",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a77990",
.data = &ipmmu_features_rcar_gen3,
}, {
.compatible = "renesas,ipmmu-r8a77995",
.data = &ipmmu_features_rcar_gen3,
}, {
/* Terminator */
},
};
static int ipmmu_probe(struct platform_device *pdev)
{
struct ipmmu_vmsa_device *mmu;
struct resource *res;
int irq;
int ret;
mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
if (!mmu) {
dev_err(&pdev->dev, "cannot allocate device data\n");
return -ENOMEM;
}
mmu->dev = &pdev->dev;
spin_lock_init(&mmu->lock);
bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);
mmu->features = of_device_get_match_data(&pdev->dev);
memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);
dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
/* Map I/O memory and request IRQ. */
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
mmu->base = devm_ioremap_resource(&pdev->dev, res);
if (IS_ERR(mmu->base))
return PTR_ERR(mmu->base);
/*
* The IPMMU has two register banks, for secure and non-secure modes.
* The bank mapped at the beginning of the IPMMU address space
* corresponds to the running mode of the CPU. When running in secure
* mode the non-secure register bank is also available at an offset.
*
* Secure mode operation isn't clearly documented and is thus currently
* not implemented in the driver. Furthermore, preliminary tests of
* non-secure operation with the main register bank were not successful.
* Offset the registers base unconditionally to point to the non-secure
* alias space for now.
*/
if (mmu->features->use_ns_alias_offset)
mmu->base += IM_NS_ALIAS_OFFSET;
mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts);
/*
* Determine if this IPMMU instance is a root device by checking for
* the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property.
*/
if (!mmu->features->has_cache_leaf_nodes ||
!of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL))
mmu->root = mmu;
else
mmu->root = ipmmu_find_root();
/*
* Wait until the root device has been registered for sure.
*/
if (!mmu->root)
return -EPROBE_DEFER;
/* Root devices have mandatory IRQs */
if (ipmmu_is_root(mmu)) {
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0,
dev_name(&pdev->dev), mmu);
if (ret < 0) {
dev_err(&pdev->dev, "failed to request IRQ %d\n", irq);
return ret;
}
ipmmu_device_reset(mmu);
if (mmu->features->reserved_context) {
dev_info(&pdev->dev, "IPMMU context 0 is reserved\n");
set_bit(0, mmu->ctx);
}
}
/*
* Register the IPMMU to the IOMMU subsystem in the following cases:
* - R-Car Gen2 IPMMU (all devices registered)
* - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device)
*/
if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) {
ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,
dev_name(&pdev->dev));
if (ret)
return ret;
iommu_device_set_ops(&mmu->iommu, &ipmmu_ops);
iommu_device_set_fwnode(&mmu->iommu,
&pdev->dev.of_node->fwnode);
ret = iommu_device_register(&mmu->iommu);
if (ret)
return ret;
#if defined(CONFIG_IOMMU_DMA)
if (!iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, &ipmmu_ops);
#endif
}
/*
* We can't create the ARM mapping here as it requires the bus to have
* an IOMMU, which only happens when bus_set_iommu() is called in
* ipmmu_init() after the probe function returns.
*/
platform_set_drvdata(pdev, mmu);
return 0;
}
static int ipmmu_remove(struct platform_device *pdev)
{
struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
iommu_device_sysfs_remove(&mmu->iommu);
iommu_device_unregister(&mmu->iommu);
arm_iommu_release_mapping(mmu->mapping);
ipmmu_device_reset(mmu);
return 0;
}
#ifdef CONFIG_PM_SLEEP
static int ipmmu_resume_noirq(struct device *dev)
{
struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);
unsigned int i;
/* Reset root MMU and restore contexts */
if (ipmmu_is_root(mmu)) {
ipmmu_device_reset(mmu);
for (i = 0; i < mmu->num_ctx; i++) {
if (!mmu->domains[i])
continue;
ipmmu_domain_setup_context(mmu->domains[i]);
}
}
/* Re-enable active micro-TLBs */
for (i = 0; i < mmu->features->num_utlbs; i++) {
if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID)
continue;
ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i);
}
return 0;
}
static const struct dev_pm_ops ipmmu_pm = {
SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq)
};
#define DEV_PM_OPS &ipmmu_pm
#else
#define DEV_PM_OPS NULL
#endif /* CONFIG_PM_SLEEP */
static struct platform_driver ipmmu_driver = {
.driver = {
.name = "ipmmu-vmsa",
.of_match_table = of_match_ptr(ipmmu_of_ids),
.pm = DEV_PM_OPS,
},
.probe = ipmmu_probe,
.remove = ipmmu_remove,
};
static int __init ipmmu_init(void)
{
struct device_node *np;
static bool setup_done;
int ret;
if (setup_done)
return 0;
np = of_find_matching_node(NULL, ipmmu_of_ids);
if (!np)
return 0;
of_node_put(np);
ret = platform_driver_register(&ipmmu_driver);
if (ret < 0)
return ret;
#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
if (!iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, &ipmmu_ops);
#endif
setup_done = true;
return 0;
}
subsys_initcall(ipmmu_init);