linux/drivers/iommu/tegra-smmu.c
Linus Torvalds a13de74e47 IOMMU Updates for Linux v6.3:
Including:
 
 	- Consolidate iommu_map/unmap functions. There have been
 	  blocking and atomic variants so far, but that was problematic
 	  as this approach does not scale with required new variants
 	  which just differ in the GFP flags used.
 	  So Jason consolidated this back into single functions that
 	  take a GFP parameter. This has the potential to cause
 	  conflicts with other trees, as they introduce new call-sites
 	  for the changed functions. I offered them to pull in the
 	  branch containing these changes and resolve it, but I am not
 	  sure everyone did that. The conflicts this caused with
 	  upstream up to v6.2-rc8 are resolved in the final merge
 	  commit.
 
 	- Retire the detach_dev() call-back in iommu_ops
 
 	- Arm SMMU updates from Will:
 	  - Device-tree binding updates:
 	    * Cater for three power domains on SM6375
 	    * Document existing compatible strings for Qualcomm SoCs
 	    * Tighten up clocks description for platform-specific compatible strings
 	  - Enable Qualcomm workarounds for some additional platforms that need them
 
 	- Intel VT-d updates from Lu Baolu:
 	  - Add Intel IOMMU performance monitoring support
 	  - Set No Execute Enable bit in PASID table entry
 	  - Two performance optimizations
 	  - Fix PASID directory pointer coherency
 	  - Fix missed rollbacks in error path
 	  - Cleanups
 
 	- Apple t8110 DART support
 
 	- Exynos IOMMU:
 	  - Implement better fault handling
 	  - Error handling fixes
 
 	- Renesas IPMMU:
 	  - Add device tree bindings for r8a779g0
 
 	- AMD IOMMU:
 	  - Various fixes for handling on SNP-enabled systems and
 	    handling of faults with unknown request-ids
 	  - Cleanups and other small fixes
 
 	- Various other smaller fixes and cleanups
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEr9jSbILcajRFYWYyK/BELZcBGuMFAmP0hDwACgkQK/BELZcB
 GuM43RAA0YieShO+X0h6TFGfbK0zVoPd91giZehWBv9rHK7pP4iY8UEtBLBWGx/t
 CId4t98mmKmC212zz8QxrwAEzyTIRY+2t1yrpG2aVkoTYk8inMb07TU37wganh3O
 T0QccXN+9b2BS4k8yro5f3uX0d/C1JQVcMowwr53VMb/e73huqP1VTbz06/CIWMH
 DUhVRCzmNhSvoUOT5n7g6+ZDH+pot8WPZbtHV7FowEsmPCRc7Fj8kXyI9FEwKwrZ
 hIV5Y+6Lej8nQScgbO8MfblJym3VrBoSoM4GY2w0L0rjQw6m+Xtea5rT0W39YVWy
 YpiscLTL8TIMPP9zK1dXVygTaABK4J2iWmheHPkpKXIhK0iuH3Dke0Do5p6DNITj
 7J2YlaNEB480D5hvNBKsbbGHavgGPT8m529Sz0R7mSC7omRzqiG5Vsb46IXL+2bc
 92ojjYNfXb6OCtagIr2LMBLZRL2JCODqF1dUmyZfA8GKOHLP5kZXoMM+sZbQ2aUL
 1LOxRZVx+tlb9V4VaH1ZSs/6eM+HLDzjtHeu3PoWYf6mW4AEt4S/yl9SKAkGdBqt
 jCUErmYB1nU/eefqG1jhWRpQeJabcT3Oe30NZru1pfMoREThhjbAACw1JxWtoe1X
 ipGpV6lAP7tQUGuRk3/9O1lNqElJuNwC5lVTjS4FJ38vYQhQbao=
 =ZaZV
 -----END PGP SIGNATURE-----

Merge tag 'iommu-updates-v6.3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull iommu updates from Joerg Roedel:

 - Consolidate iommu_map/unmap functions.

   There have been blocking and atomic variants so far, but that was
   problematic as this approach does not scale with required new
   variants which just differ in the GFP flags used. So Jason
   consolidated this back into single functions that take a GFP
   parameter.

 - Retire the detach_dev() call-back in iommu_ops

 - Arm SMMU updates from Will:
     - Device-tree binding updates:
         - Cater for three power domains on SM6375
         - Document existing compatible strings for Qualcomm SoCs
         - Tighten up clocks description for platform-specific
           compatible strings
     - Enable Qualcomm workarounds for some additional platforms that
       need them

 - Intel VT-d updates from Lu Baolu:
     - Add Intel IOMMU performance monitoring support
     - Set No Execute Enable bit in PASID table entry
     - Two performance optimizations
     - Fix PASID directory pointer coherency
     - Fix missed rollbacks in error path
     - Cleanups

 - Apple t8110 DART support

 - Exynos IOMMU:
     - Implement better fault handling
     - Error handling fixes

 - Renesas IPMMU:
     - Add device tree bindings for r8a779g0

 - AMD IOMMU:
     - Various fixes for handling on SNP-enabled systems and
       handling of faults with unknown request-ids
     - Cleanups and other small fixes

 - Various other smaller fixes and cleanups

* tag 'iommu-updates-v6.3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (71 commits)
  iommu/amd: Skip attach device domain is same as new domain
  iommu: Attach device group to old domain in error path
  iommu/vt-d: Allow to use flush-queue when first level is default
  iommu/vt-d: Fix PASID directory pointer coherency
  iommu/vt-d: Avoid superfluous IOTLB tracking in lazy mode
  iommu/vt-d: Fix error handling in sva enable/disable paths
  iommu/amd: Improve page fault error reporting
  iommu/amd: Do not identity map v2 capable device when snp is enabled
  iommu: Fix error unwind in iommu_group_alloc()
  iommu/of: mark an unused function as __maybe_unused
  iommu: dart: DART_T8110_ERROR range should be 0 to 5
  iommu/vt-d: Enable IOMMU perfmon support
  iommu/vt-d: Add IOMMU perfmon overflow handler support
  iommu/vt-d: Support cpumask for IOMMU perfmon
  iommu/vt-d: Add IOMMU perfmon support
  iommu/vt-d: Support Enhanced Command Interface
  iommu/vt-d: Retrieve IOMMU perfmon capability information
  iommu/vt-d: Support size of the register set in DRHD
  iommu/vt-d: Set No Execute Enable bit in PASID table entry
  iommu/vt-d: Remove sva from intel_svm_dev
  ...
2023-02-24 13:40:13 -08:00

1161 lines
27 KiB
C

// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved.
*/
#include <linux/bitops.h>
#include <linux/debugfs.h>
#include <linux/err.h>
#include <linux/iommu.h>
#include <linux/kernel.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/dma-mapping.h>
#include <soc/tegra/ahb.h>
#include <soc/tegra/mc.h>
struct tegra_smmu_group {
struct list_head list;
struct tegra_smmu *smmu;
const struct tegra_smmu_group_soc *soc;
struct iommu_group *group;
unsigned int swgroup;
};
struct tegra_smmu {
void __iomem *regs;
struct device *dev;
struct tegra_mc *mc;
const struct tegra_smmu_soc *soc;
struct list_head groups;
unsigned long pfn_mask;
unsigned long tlb_mask;
unsigned long *asids;
struct mutex lock;
struct list_head list;
struct dentry *debugfs;
struct iommu_device iommu; /* IOMMU Core code handle */
};
struct tegra_smmu_as {
struct iommu_domain domain;
struct tegra_smmu *smmu;
unsigned int use_count;
spinlock_t lock;
u32 *count;
struct page **pts;
struct page *pd;
dma_addr_t pd_dma;
unsigned id;
u32 attr;
};
static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
{
return container_of(dom, struct tegra_smmu_as, domain);
}
static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
unsigned long offset)
{
writel(value, smmu->regs + offset);
}
static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
{
return readl(smmu->regs + offset);
}
#define SMMU_CONFIG 0x010
#define SMMU_CONFIG_ENABLE (1 << 0)
#define SMMU_TLB_CONFIG 0x14
#define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
#define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
#define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
#define SMMU_PTC_CONFIG 0x18
#define SMMU_PTC_CONFIG_ENABLE (1 << 29)
#define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)
#define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)
#define SMMU_PTB_ASID 0x01c
#define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
#define SMMU_PTB_DATA 0x020
#define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
#define SMMU_TLB_FLUSH 0x030
#define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
#define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
#define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0)
#define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
SMMU_TLB_FLUSH_VA_MATCH_SECTION)
#define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \
SMMU_TLB_FLUSH_VA_MATCH_GROUP)
#define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31)
#define SMMU_PTC_FLUSH 0x034
#define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)
#define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)
#define SMMU_PTC_FLUSH_HI 0x9b8
#define SMMU_PTC_FLUSH_HI_MASK 0x3
/* per-SWGROUP SMMU_*_ASID register */
#define SMMU_ASID_ENABLE (1 << 31)
#define SMMU_ASID_MASK 0x7f
#define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)
/* page table definitions */
#define SMMU_NUM_PDE 1024
#define SMMU_NUM_PTE 1024
#define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)
#define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)
#define SMMU_PDE_SHIFT 22
#define SMMU_PTE_SHIFT 12
#define SMMU_PAGE_MASK (~(SMMU_SIZE_PT-1))
#define SMMU_OFFSET_IN_PAGE(x) ((unsigned long)(x) & ~SMMU_PAGE_MASK)
#define SMMU_PFN_PHYS(x) ((phys_addr_t)(x) << SMMU_PTE_SHIFT)
#define SMMU_PHYS_PFN(x) ((unsigned long)((x) >> SMMU_PTE_SHIFT))
#define SMMU_PD_READABLE (1 << 31)
#define SMMU_PD_WRITABLE (1 << 30)
#define SMMU_PD_NONSECURE (1 << 29)
#define SMMU_PDE_READABLE (1 << 31)
#define SMMU_PDE_WRITABLE (1 << 30)
#define SMMU_PDE_NONSECURE (1 << 29)
#define SMMU_PDE_NEXT (1 << 28)
#define SMMU_PTE_READABLE (1 << 31)
#define SMMU_PTE_WRITABLE (1 << 30)
#define SMMU_PTE_NONSECURE (1 << 29)
#define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
SMMU_PDE_NONSECURE)
static unsigned int iova_pd_index(unsigned long iova)
{
return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
}
static unsigned int iova_pt_index(unsigned long iova)
{
return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
}
static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
{
addr >>= 12;
return (addr & smmu->pfn_mask) == addr;
}
static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde)
{
return (dma_addr_t)(pde & smmu->pfn_mask) << 12;
}
static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
{
smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
}
static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
unsigned long offset)
{
u32 value;
offset &= ~(smmu->mc->soc->atom_size - 1);
if (smmu->mc->soc->num_address_bits > 32) {
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
#else
value = 0;
#endif
smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
}
value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
smmu_writel(smmu, value, SMMU_PTC_FLUSH);
}
static inline void smmu_flush_tlb(struct tegra_smmu *smmu)
{
smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);
}
static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
unsigned long asid)
{
u32 value;
if (smmu->soc->num_asids == 4)
value = (asid & 0x3) << 29;
else
value = (asid & 0x7f) << 24;
value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL;
smmu_writel(smmu, value, SMMU_TLB_FLUSH);
}
static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
unsigned long asid,
unsigned long iova)
{
u32 value;
if (smmu->soc->num_asids == 4)
value = (asid & 0x3) << 29;
else
value = (asid & 0x7f) << 24;
value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova);
smmu_writel(smmu, value, SMMU_TLB_FLUSH);
}
static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
unsigned long asid,
unsigned long iova)
{
u32 value;
if (smmu->soc->num_asids == 4)
value = (asid & 0x3) << 29;
else
value = (asid & 0x7f) << 24;
value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova);
smmu_writel(smmu, value, SMMU_TLB_FLUSH);
}
static inline void smmu_flush(struct tegra_smmu *smmu)
{
smmu_readl(smmu, SMMU_PTB_ASID);
}
static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)
{
unsigned long id;
id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);
if (id >= smmu->soc->num_asids)
return -ENOSPC;
set_bit(id, smmu->asids);
*idp = id;
return 0;
}
static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
{
clear_bit(id, smmu->asids);
}
static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
{
struct tegra_smmu_as *as;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
as = kzalloc(sizeof(*as), GFP_KERNEL);
if (!as)
return NULL;
as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
if (!as->pd) {
kfree(as);
return NULL;
}
as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
if (!as->count) {
__free_page(as->pd);
kfree(as);
return NULL;
}
as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
if (!as->pts) {
kfree(as->count);
__free_page(as->pd);
kfree(as);
return NULL;
}
spin_lock_init(&as->lock);
/* setup aperture */
as->domain.geometry.aperture_start = 0;
as->domain.geometry.aperture_end = 0xffffffff;
as->domain.geometry.force_aperture = true;
return &as->domain;
}
static void tegra_smmu_domain_free(struct iommu_domain *domain)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
/* TODO: free page directory and page tables */
WARN_ON_ONCE(as->use_count);
kfree(as->count);
kfree(as->pts);
kfree(as);
}
static const struct tegra_smmu_swgroup *
tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)
{
const struct tegra_smmu_swgroup *group = NULL;
unsigned int i;
for (i = 0; i < smmu->soc->num_swgroups; i++) {
if (smmu->soc->swgroups[i].swgroup == swgroup) {
group = &smmu->soc->swgroups[i];
break;
}
}
return group;
}
static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,
unsigned int asid)
{
const struct tegra_smmu_swgroup *group;
unsigned int i;
u32 value;
group = tegra_smmu_find_swgroup(smmu, swgroup);
if (group) {
value = smmu_readl(smmu, group->reg);
value &= ~SMMU_ASID_MASK;
value |= SMMU_ASID_VALUE(asid);
value |= SMMU_ASID_ENABLE;
smmu_writel(smmu, value, group->reg);
} else {
pr_warn("%s group from swgroup %u not found\n", __func__,
swgroup);
/* No point moving ahead if group was not found */
return;
}
for (i = 0; i < smmu->soc->num_clients; i++) {
const struct tegra_mc_client *client = &smmu->soc->clients[i];
if (client->swgroup != swgroup)
continue;
value = smmu_readl(smmu, client->regs.smmu.reg);
value |= BIT(client->regs.smmu.bit);
smmu_writel(smmu, value, client->regs.smmu.reg);
}
}
static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,
unsigned int asid)
{
const struct tegra_smmu_swgroup *group;
unsigned int i;
u32 value;
group = tegra_smmu_find_swgroup(smmu, swgroup);
if (group) {
value = smmu_readl(smmu, group->reg);
value &= ~SMMU_ASID_MASK;
value |= SMMU_ASID_VALUE(asid);
value &= ~SMMU_ASID_ENABLE;
smmu_writel(smmu, value, group->reg);
}
for (i = 0; i < smmu->soc->num_clients; i++) {
const struct tegra_mc_client *client = &smmu->soc->clients[i];
if (client->swgroup != swgroup)
continue;
value = smmu_readl(smmu, client->regs.smmu.reg);
value &= ~BIT(client->regs.smmu.bit);
smmu_writel(smmu, value, client->regs.smmu.reg);
}
}
static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
struct tegra_smmu_as *as)
{
u32 value;
int err = 0;
mutex_lock(&smmu->lock);
if (as->use_count > 0) {
as->use_count++;
goto unlock;
}
as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
DMA_TO_DEVICE);
if (dma_mapping_error(smmu->dev, as->pd_dma)) {
err = -ENOMEM;
goto unlock;
}
/* We can't handle 64-bit DMA addresses */
if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
err = -ENOMEM;
goto err_unmap;
}
err = tegra_smmu_alloc_asid(smmu, &as->id);
if (err < 0)
goto err_unmap;
smmu_flush_ptc(smmu, as->pd_dma, 0);
smmu_flush_tlb_asid(smmu, as->id);
smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
smmu_writel(smmu, value, SMMU_PTB_DATA);
smmu_flush(smmu);
as->smmu = smmu;
as->use_count++;
mutex_unlock(&smmu->lock);
return 0;
err_unmap:
dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
unlock:
mutex_unlock(&smmu->lock);
return err;
}
static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
struct tegra_smmu_as *as)
{
mutex_lock(&smmu->lock);
if (--as->use_count > 0) {
mutex_unlock(&smmu->lock);
return;
}
tegra_smmu_free_asid(smmu, as->id);
dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
as->smmu = NULL;
mutex_unlock(&smmu->lock);
}
static int tegra_smmu_attach_dev(struct iommu_domain *domain,
struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct tegra_smmu *smmu = dev_iommu_priv_get(dev);
struct tegra_smmu_as *as = to_smmu_as(domain);
unsigned int index;
int err;
if (!fwspec)
return -ENOENT;
for (index = 0; index < fwspec->num_ids; index++) {
err = tegra_smmu_as_prepare(smmu, as);
if (err)
goto disable;
tegra_smmu_enable(smmu, fwspec->ids[index], as->id);
}
if (index == 0)
return -ENODEV;
return 0;
disable:
while (index--) {
tegra_smmu_disable(smmu, fwspec->ids[index], as->id);
tegra_smmu_as_unprepare(smmu, as);
}
return err;
}
static void tegra_smmu_set_platform_dma(struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct tegra_smmu_as *as = to_smmu_as(domain);
struct tegra_smmu *smmu = as->smmu;
unsigned int index;
if (!fwspec)
return;
for (index = 0; index < fwspec->num_ids; index++) {
tegra_smmu_disable(smmu, fwspec->ids[index], as->id);
tegra_smmu_as_unprepare(smmu, as);
}
}
static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
u32 value)
{
unsigned int pd_index = iova_pd_index(iova);
struct tegra_smmu *smmu = as->smmu;
u32 *pd = page_address(as->pd);
unsigned long offset = pd_index * sizeof(*pd);
/* Set the page directory entry first */
pd[pd_index] = value;
/* The flush the page directory entry from caches */
dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
sizeof(*pd), DMA_TO_DEVICE);
/* And flush the iommu */
smmu_flush_ptc(smmu, as->pd_dma, offset);
smmu_flush_tlb_section(smmu, as->id, iova);
smmu_flush(smmu);
}
static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
{
u32 *pt = page_address(pt_page);
return pt + iova_pt_index(iova);
}
static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
dma_addr_t *dmap)
{
unsigned int pd_index = iova_pd_index(iova);
struct tegra_smmu *smmu = as->smmu;
struct page *pt_page;
u32 *pd;
pt_page = as->pts[pd_index];
if (!pt_page)
return NULL;
pd = page_address(as->pd);
*dmap = smmu_pde_to_dma(smmu, pd[pd_index]);
return tegra_smmu_pte_offset(pt_page, iova);
}
static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
dma_addr_t *dmap, struct page *page)
{
unsigned int pde = iova_pd_index(iova);
struct tegra_smmu *smmu = as->smmu;
if (!as->pts[pde]) {
dma_addr_t dma;
dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
DMA_TO_DEVICE);
if (dma_mapping_error(smmu->dev, dma)) {
__free_page(page);
return NULL;
}
if (!smmu_dma_addr_valid(smmu, dma)) {
dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
DMA_TO_DEVICE);
__free_page(page);
return NULL;
}
as->pts[pde] = page;
tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
SMMU_PDE_NEXT));
*dmap = dma;
} else {
u32 *pd = page_address(as->pd);
*dmap = smmu_pde_to_dma(smmu, pd[pde]);
}
return tegra_smmu_pte_offset(as->pts[pde], iova);
}
static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
{
unsigned int pd_index = iova_pd_index(iova);
as->count[pd_index]++;
}
static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
{
unsigned int pde = iova_pd_index(iova);
struct page *page = as->pts[pde];
/*
* When no entries in this page table are used anymore, return the
* memory page to the system.
*/
if (--as->count[pde] == 0) {
struct tegra_smmu *smmu = as->smmu;
u32 *pd = page_address(as->pd);
dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]);
tegra_smmu_set_pde(as, iova, 0);
dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
__free_page(page);
as->pts[pde] = NULL;
}
}
static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
u32 *pte, dma_addr_t pte_dma, u32 val)
{
struct tegra_smmu *smmu = as->smmu;
unsigned long offset = SMMU_OFFSET_IN_PAGE(pte);
*pte = val;
dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
4, DMA_TO_DEVICE);
smmu_flush_ptc(smmu, pte_dma, offset);
smmu_flush_tlb_group(smmu, as->id, iova);
smmu_flush(smmu);
}
static struct page *as_get_pde_page(struct tegra_smmu_as *as,
unsigned long iova, gfp_t gfp,
unsigned long *flags)
{
unsigned int pde = iova_pd_index(iova);
struct page *page = as->pts[pde];
/* at first check whether allocation needs to be done at all */
if (page)
return page;
/*
* In order to prevent exhaustion of the atomic memory pool, we
* allocate page in a sleeping context if GFP flags permit. Hence
* spinlock needs to be unlocked and re-locked after allocation.
*/
if (gfpflags_allow_blocking(gfp))
spin_unlock_irqrestore(&as->lock, *flags);
page = alloc_page(gfp | __GFP_DMA | __GFP_ZERO);
if (gfpflags_allow_blocking(gfp))
spin_lock_irqsave(&as->lock, *flags);
/*
* In a case of blocking allocation, a concurrent mapping may win
* the PDE allocation. In this case the allocated page isn't needed
* if allocation succeeded and the allocation failure isn't fatal.
*/
if (as->pts[pde]) {
if (page)
__free_page(page);
page = as->pts[pde];
}
return page;
}
static int
__tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp,
unsigned long *flags)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
dma_addr_t pte_dma;
struct page *page;
u32 pte_attrs;
u32 *pte;
page = as_get_pde_page(as, iova, gfp, flags);
if (!page)
return -ENOMEM;
pte = as_get_pte(as, iova, &pte_dma, page);
if (!pte)
return -ENOMEM;
/* If we aren't overwriting a pre-existing entry, increment use */
if (*pte == 0)
tegra_smmu_pte_get_use(as, iova);
pte_attrs = SMMU_PTE_NONSECURE;
if (prot & IOMMU_READ)
pte_attrs |= SMMU_PTE_READABLE;
if (prot & IOMMU_WRITE)
pte_attrs |= SMMU_PTE_WRITABLE;
tegra_smmu_set_pte(as, iova, pte, pte_dma,
SMMU_PHYS_PFN(paddr) | pte_attrs);
return 0;
}
static size_t
__tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
dma_addr_t pte_dma;
u32 *pte;
pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
if (!pte || !*pte)
return 0;
tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
tegra_smmu_pte_put_use(as, iova);
return size;
}
static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
unsigned long flags;
int ret;
spin_lock_irqsave(&as->lock, flags);
ret = __tegra_smmu_map(domain, iova, paddr, size, prot, gfp, &flags);
spin_unlock_irqrestore(&as->lock, flags);
return ret;
}
static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
unsigned long flags;
spin_lock_irqsave(&as->lock, flags);
size = __tegra_smmu_unmap(domain, iova, size, gather);
spin_unlock_irqrestore(&as->lock, flags);
return size;
}
static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
unsigned long pfn;
dma_addr_t pte_dma;
u32 *pte;
pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
if (!pte || !*pte)
return 0;
pfn = *pte & as->smmu->pfn_mask;
return SMMU_PFN_PHYS(pfn) + SMMU_OFFSET_IN_PAGE(iova);
}
static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
{
struct platform_device *pdev;
struct tegra_mc *mc;
pdev = of_find_device_by_node(np);
if (!pdev)
return NULL;
mc = platform_get_drvdata(pdev);
if (!mc) {
put_device(&pdev->dev);
return NULL;
}
return mc->smmu;
}
static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,
struct of_phandle_args *args)
{
const struct iommu_ops *ops = smmu->iommu.ops;
int err;
err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops);
if (err < 0) {
dev_err(dev, "failed to initialize fwspec: %d\n", err);
return err;
}
err = ops->of_xlate(dev, args);
if (err < 0) {
dev_err(dev, "failed to parse SW group ID: %d\n", err);
iommu_fwspec_free(dev);
return err;
}
return 0;
}
static struct iommu_device *tegra_smmu_probe_device(struct device *dev)
{
struct device_node *np = dev->of_node;
struct tegra_smmu *smmu = NULL;
struct of_phandle_args args;
unsigned int index = 0;
int err;
while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
&args) == 0) {
smmu = tegra_smmu_find(args.np);
if (smmu) {
err = tegra_smmu_configure(smmu, dev, &args);
if (err < 0) {
of_node_put(args.np);
return ERR_PTR(err);
}
}
of_node_put(args.np);
index++;
}
smmu = dev_iommu_priv_get(dev);
if (!smmu)
return ERR_PTR(-ENODEV);
return &smmu->iommu;
}
static const struct tegra_smmu_group_soc *
tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup)
{
unsigned int i, j;
for (i = 0; i < smmu->soc->num_groups; i++)
for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)
if (smmu->soc->groups[i].swgroups[j] == swgroup)
return &smmu->soc->groups[i];
return NULL;
}
static void tegra_smmu_group_release(void *iommu_data)
{
struct tegra_smmu_group *group = iommu_data;
struct tegra_smmu *smmu = group->smmu;
mutex_lock(&smmu->lock);
list_del(&group->list);
mutex_unlock(&smmu->lock);
}
static struct iommu_group *tegra_smmu_device_group(struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct tegra_smmu *smmu = dev_iommu_priv_get(dev);
const struct tegra_smmu_group_soc *soc;
unsigned int swgroup = fwspec->ids[0];
struct tegra_smmu_group *group;
struct iommu_group *grp;
/* Find group_soc associating with swgroup */
soc = tegra_smmu_find_group(smmu, swgroup);
mutex_lock(&smmu->lock);
/* Find existing iommu_group associating with swgroup or group_soc */
list_for_each_entry(group, &smmu->groups, list)
if ((group->swgroup == swgroup) || (soc && group->soc == soc)) {
grp = iommu_group_ref_get(group->group);
mutex_unlock(&smmu->lock);
return grp;
}
group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);
if (!group) {
mutex_unlock(&smmu->lock);
return NULL;
}
INIT_LIST_HEAD(&group->list);
group->swgroup = swgroup;
group->smmu = smmu;
group->soc = soc;
if (dev_is_pci(dev))
group->group = pci_device_group(dev);
else
group->group = generic_device_group(dev);
if (IS_ERR(group->group)) {
devm_kfree(smmu->dev, group);
mutex_unlock(&smmu->lock);
return NULL;
}
iommu_group_set_iommudata(group->group, group, tegra_smmu_group_release);
if (soc)
iommu_group_set_name(group->group, soc->name);
list_add_tail(&group->list, &smmu->groups);
mutex_unlock(&smmu->lock);
return group->group;
}
static int tegra_smmu_of_xlate(struct device *dev,
struct of_phandle_args *args)
{
struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
struct tegra_mc *mc = platform_get_drvdata(iommu_pdev);
u32 id = args->args[0];
/*
* Note: we are here releasing the reference of &iommu_pdev->dev, which
* is mc->dev. Although some functions in tegra_smmu_ops may keep using
* its private data beyond this point, it's still safe to do so because
* the SMMU parent device is the same as the MC, so the reference count
* isn't strictly necessary.
*/
put_device(&iommu_pdev->dev);
dev_iommu_priv_set(dev, mc->smmu);
return iommu_fwspec_add_ids(dev, &id, 1);
}
static const struct iommu_ops tegra_smmu_ops = {
.domain_alloc = tegra_smmu_domain_alloc,
.probe_device = tegra_smmu_probe_device,
.device_group = tegra_smmu_device_group,
.set_platform_dma_ops = tegra_smmu_set_platform_dma,
.of_xlate = tegra_smmu_of_xlate,
.pgsize_bitmap = SZ_4K,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = tegra_smmu_attach_dev,
.map = tegra_smmu_map,
.unmap = tegra_smmu_unmap,
.iova_to_phys = tegra_smmu_iova_to_phys,
.free = tegra_smmu_domain_free,
}
};
static void tegra_smmu_ahb_enable(void)
{
static const struct of_device_id ahb_match[] = {
{ .compatible = "nvidia,tegra30-ahb", },
{ }
};
struct device_node *ahb;
ahb = of_find_matching_node(NULL, ahb_match);
if (ahb) {
tegra_ahb_enable_smmu(ahb);
of_node_put(ahb);
}
}
static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
{
struct tegra_smmu *smmu = s->private;
unsigned int i;
u32 value;
seq_printf(s, "swgroup enabled ASID\n");
seq_printf(s, "------------------------\n");
for (i = 0; i < smmu->soc->num_swgroups; i++) {
const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
const char *status;
unsigned int asid;
value = smmu_readl(smmu, group->reg);
if (value & SMMU_ASID_ENABLE)
status = "yes";
else
status = "no";
asid = value & SMMU_ASID_MASK;
seq_printf(s, "%-9s %-7s %#04x\n", group->name, status,
asid);
}
return 0;
}
DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups);
static int tegra_smmu_clients_show(struct seq_file *s, void *data)
{
struct tegra_smmu *smmu = s->private;
unsigned int i;
u32 value;
seq_printf(s, "client enabled\n");
seq_printf(s, "--------------------\n");
for (i = 0; i < smmu->soc->num_clients; i++) {
const struct tegra_mc_client *client = &smmu->soc->clients[i];
const char *status;
value = smmu_readl(smmu, client->regs.smmu.reg);
if (value & BIT(client->regs.smmu.bit))
status = "yes";
else
status = "no";
seq_printf(s, "%-12s %s\n", client->name, status);
}
return 0;
}
DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients);
static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
{
smmu->debugfs = debugfs_create_dir("smmu", NULL);
if (!smmu->debugfs)
return;
debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
&tegra_smmu_swgroups_fops);
debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
&tegra_smmu_clients_fops);
}
static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
{
debugfs_remove_recursive(smmu->debugfs);
}
struct tegra_smmu *tegra_smmu_probe(struct device *dev,
const struct tegra_smmu_soc *soc,
struct tegra_mc *mc)
{
struct tegra_smmu *smmu;
u32 value;
int err;
smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
if (!smmu)
return ERR_PTR(-ENOMEM);
/*
* This is a bit of a hack. Ideally we'd want to simply return this
* value. However iommu_device_register() will attempt to add
* all devices to the IOMMU before we get that far. In order
* not to rely on global variables to track the IOMMU instance, we
* set it here so that it can be looked up from the .probe_device()
* callback via the IOMMU device's .drvdata field.
*/
mc->smmu = smmu;
smmu->asids = devm_bitmap_zalloc(dev, soc->num_asids, GFP_KERNEL);
if (!smmu->asids)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&smmu->groups);
mutex_init(&smmu->lock);
smmu->regs = mc->regs;
smmu->soc = soc;
smmu->dev = dev;
smmu->mc = mc;
smmu->pfn_mask =
BIT_MASK(mc->soc->num_address_bits - SMMU_PTE_SHIFT) - 1;
dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
mc->soc->num_address_bits, smmu->pfn_mask);
smmu->tlb_mask = (1 << fls(smmu->soc->num_tlb_lines)) - 1;
dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,
smmu->tlb_mask);
value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
if (soc->supports_request_limit)
value |= SMMU_PTC_CONFIG_REQ_LIMIT(8);
smmu_writel(smmu, value, SMMU_PTC_CONFIG);
value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);
if (soc->supports_round_robin_arbitration)
value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;
smmu_writel(smmu, value, SMMU_TLB_CONFIG);
smmu_flush_ptc_all(smmu);
smmu_flush_tlb(smmu);
smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
smmu_flush(smmu);
tegra_smmu_ahb_enable();
err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));
if (err)
return ERR_PTR(err);
err = iommu_device_register(&smmu->iommu, &tegra_smmu_ops, dev);
if (err) {
iommu_device_sysfs_remove(&smmu->iommu);
return ERR_PTR(err);
}
if (IS_ENABLED(CONFIG_DEBUG_FS))
tegra_smmu_debugfs_init(smmu);
return smmu;
}
void tegra_smmu_remove(struct tegra_smmu *smmu)
{
iommu_device_unregister(&smmu->iommu);
iommu_device_sysfs_remove(&smmu->iommu);
if (IS_ENABLED(CONFIG_DEBUG_FS))
tegra_smmu_debugfs_exit(smmu);
}