mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
![Benjamin Herrenschmidt](/assets/img/avatar_default.png)
Instead of punching a hole in the linear mapping, just use normal cachable memory, and apply the flush sequence documented in the CPC625 (aka U3) user manual. This allows us to remove quite a bit of code related to the early allocation of the DART and the hole in the linear mapping. We can also get rid of the copy of the DART for suspend/resume as the original memory can just be saved/restored now, as long as we properly sync the caches. Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> [mpe: Integrate dart_init() fix to return ENODEV when DART disabled] Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
471 lines
12 KiB
C
471 lines
12 KiB
C
/*
|
|
* arch/powerpc/sysdev/dart_iommu.c
|
|
*
|
|
* Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
|
|
* Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
|
|
* IBM Corporation
|
|
*
|
|
* Based on pSeries_iommu.c:
|
|
* Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
|
|
* Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
|
|
*
|
|
* Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
|
|
*
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/types.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/string.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/suspend.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/gfp.h>
|
|
#include <asm/io.h>
|
|
#include <asm/prom.h>
|
|
#include <asm/iommu.h>
|
|
#include <asm/pci-bridge.h>
|
|
#include <asm/machdep.h>
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/ppc-pci.h>
|
|
|
|
#include "dart.h"
|
|
|
|
/* DART table address and size */
|
|
static u32 *dart_tablebase;
|
|
static unsigned long dart_tablesize;
|
|
|
|
/* Mapped base address for the dart */
|
|
static unsigned int __iomem *dart;
|
|
|
|
/* Dummy val that entries are set to when unused */
|
|
static unsigned int dart_emptyval;
|
|
|
|
static struct iommu_table iommu_table_dart;
|
|
static int iommu_table_dart_inited;
|
|
static int dart_dirty;
|
|
static int dart_is_u4;
|
|
|
|
#define DART_U4_BYPASS_BASE 0x8000000000ull
|
|
|
|
#define DBG(...)
|
|
|
|
static DEFINE_SPINLOCK(invalidate_lock);
|
|
|
|
static inline void dart_tlb_invalidate_all(void)
|
|
{
|
|
unsigned long l = 0;
|
|
unsigned int reg, inv_bit;
|
|
unsigned long limit;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&invalidate_lock, flags);
|
|
|
|
DBG("dart: flush\n");
|
|
|
|
/* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
|
|
* control register and wait for it to clear.
|
|
*
|
|
* Gotcha: Sometimes, the DART won't detect that the bit gets
|
|
* set. If so, clear it and set it again.
|
|
*/
|
|
|
|
limit = 0;
|
|
|
|
inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
|
|
retry:
|
|
l = 0;
|
|
reg = DART_IN(DART_CNTL);
|
|
reg |= inv_bit;
|
|
DART_OUT(DART_CNTL, reg);
|
|
|
|
while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
|
|
l++;
|
|
if (l == (1L << limit)) {
|
|
if (limit < 4) {
|
|
limit++;
|
|
reg = DART_IN(DART_CNTL);
|
|
reg &= ~inv_bit;
|
|
DART_OUT(DART_CNTL, reg);
|
|
goto retry;
|
|
} else
|
|
panic("DART: TLB did not flush after waiting a long "
|
|
"time. Buggy U3 ?");
|
|
}
|
|
|
|
spin_unlock_irqrestore(&invalidate_lock, flags);
|
|
}
|
|
|
|
static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
|
|
{
|
|
unsigned int reg;
|
|
unsigned int l, limit;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&invalidate_lock, flags);
|
|
|
|
reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
|
|
(bus_rpn & DART_CNTL_U4_IONE_MASK);
|
|
DART_OUT(DART_CNTL, reg);
|
|
|
|
limit = 0;
|
|
wait_more:
|
|
l = 0;
|
|
while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
|
|
rmb();
|
|
l++;
|
|
}
|
|
|
|
if (l == (1L << limit)) {
|
|
if (limit < 4) {
|
|
limit++;
|
|
goto wait_more;
|
|
} else
|
|
panic("DART: TLB did not flush after waiting a long "
|
|
"time. Buggy U4 ?");
|
|
}
|
|
|
|
spin_unlock_irqrestore(&invalidate_lock, flags);
|
|
}
|
|
|
|
static void dart_cache_sync(unsigned int *base, unsigned int count)
|
|
{
|
|
/*
|
|
* We add 1 to the number of entries to flush, following a
|
|
* comment in Darwin indicating that the memory controller
|
|
* can prefetch unmapped memory under some circumstances.
|
|
*/
|
|
unsigned long start = (unsigned long)base;
|
|
unsigned long end = start + (count + 1) * sizeof(unsigned int);
|
|
unsigned int tmp;
|
|
|
|
/* Perform a standard cache flush */
|
|
flush_inval_dcache_range(start, end);
|
|
|
|
/*
|
|
* Perform the sequence described in the CPC925 manual to
|
|
* ensure all the data gets to a point the cache incoherent
|
|
* DART hardware will see.
|
|
*/
|
|
asm volatile(" sync;"
|
|
" isync;"
|
|
" dcbf 0,%1;"
|
|
" sync;"
|
|
" isync;"
|
|
" lwz %0,0(%1);"
|
|
" isync" : "=r" (tmp) : "r" (end) : "memory");
|
|
}
|
|
|
|
static void dart_flush(struct iommu_table *tbl)
|
|
{
|
|
mb();
|
|
if (dart_dirty) {
|
|
dart_tlb_invalidate_all();
|
|
dart_dirty = 0;
|
|
}
|
|
}
|
|
|
|
static int dart_build(struct iommu_table *tbl, long index,
|
|
long npages, unsigned long uaddr,
|
|
enum dma_data_direction direction,
|
|
struct dma_attrs *attrs)
|
|
{
|
|
unsigned int *dp, *orig_dp;
|
|
unsigned int rpn;
|
|
long l;
|
|
|
|
DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
|
|
|
|
orig_dp = dp = ((unsigned int*)tbl->it_base) + index;
|
|
|
|
/* On U3, all memory is contiguous, so we can move this
|
|
* out of the loop.
|
|
*/
|
|
l = npages;
|
|
while (l--) {
|
|
rpn = __pa(uaddr) >> DART_PAGE_SHIFT;
|
|
|
|
*(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
|
|
|
|
uaddr += DART_PAGE_SIZE;
|
|
}
|
|
dart_cache_sync(orig_dp, npages);
|
|
|
|
if (dart_is_u4) {
|
|
rpn = index;
|
|
while (npages--)
|
|
dart_tlb_invalidate_one(rpn++);
|
|
} else {
|
|
dart_dirty = 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
static void dart_free(struct iommu_table *tbl, long index, long npages)
|
|
{
|
|
unsigned int *dp, *orig_dp;
|
|
long orig_npages = npages;
|
|
|
|
/* We don't worry about flushing the TLB cache. The only drawback of
|
|
* not doing it is that we won't catch buggy device drivers doing
|
|
* bad DMAs, but then no 32-bit architecture ever does either.
|
|
*/
|
|
|
|
DBG("dart: free at: %lx, %lx\n", index, npages);
|
|
|
|
orig_dp = dp = ((unsigned int *)tbl->it_base) + index;
|
|
|
|
while (npages--)
|
|
*(dp++) = dart_emptyval;
|
|
|
|
dart_cache_sync(orig_dp, orig_npages);
|
|
}
|
|
|
|
static void allocate_dart(void)
|
|
{
|
|
unsigned long tmp;
|
|
|
|
/* 512 pages (2MB) is max DART tablesize. */
|
|
dart_tablesize = 1UL << 21;
|
|
|
|
/*
|
|
* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
|
|
* will blow up an entire large page anyway in the kernel mapping.
|
|
*/
|
|
dart_tablebase = __va(memblock_alloc_base(1UL<<24,
|
|
1UL<<24, 0x80000000L));
|
|
|
|
/* There is no point scanning the DART space for leaks*/
|
|
kmemleak_no_scan((void *)dart_tablebase);
|
|
|
|
/* Allocate a spare page to map all invalid DART pages. We need to do
|
|
* that to work around what looks like a problem with the HT bridge
|
|
* prefetching into invalid pages and corrupting data
|
|
*/
|
|
tmp = memblock_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
|
|
dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
|
|
DARTMAP_RPNMASK);
|
|
|
|
printk(KERN_INFO "DART table allocated at: %p\n", dart_tablebase);
|
|
}
|
|
|
|
static int __init dart_init(struct device_node *dart_node)
|
|
{
|
|
unsigned int i;
|
|
unsigned long base, size;
|
|
struct resource r;
|
|
|
|
/* IOMMU disabled by the user ? bail out */
|
|
if (iommu_is_off)
|
|
return -ENODEV;
|
|
|
|
/*
|
|
* Only use the DART if the machine has more than 1GB of RAM
|
|
* or if requested with iommu=on on cmdline.
|
|
*
|
|
* 1GB of RAM is picked as limit because some default devices
|
|
* (i.e. Airport Extreme) have 30 bit address range limits.
|
|
*/
|
|
|
|
if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull)
|
|
return -ENODEV;
|
|
|
|
/* Get DART registers */
|
|
if (of_address_to_resource(dart_node, 0, &r))
|
|
panic("DART: can't get register base ! ");
|
|
|
|
/* Map in DART registers */
|
|
dart = ioremap(r.start, resource_size(&r));
|
|
if (dart == NULL)
|
|
panic("DART: Cannot map registers!");
|
|
|
|
/* Allocate the DART and dummy page */
|
|
allocate_dart();
|
|
|
|
/* Fill initial table */
|
|
for (i = 0; i < dart_tablesize/4; i++)
|
|
dart_tablebase[i] = dart_emptyval;
|
|
|
|
/* Push to memory */
|
|
dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
|
|
|
|
/* Initialize DART with table base and enable it. */
|
|
base = ((unsigned long)dart_tablebase) >> DART_PAGE_SHIFT;
|
|
size = dart_tablesize >> DART_PAGE_SHIFT;
|
|
if (dart_is_u4) {
|
|
size &= DART_SIZE_U4_SIZE_MASK;
|
|
DART_OUT(DART_BASE_U4, base);
|
|
DART_OUT(DART_SIZE_U4, size);
|
|
DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
|
|
} else {
|
|
size &= DART_CNTL_U3_SIZE_MASK;
|
|
DART_OUT(DART_CNTL,
|
|
DART_CNTL_U3_ENABLE |
|
|
(base << DART_CNTL_U3_BASE_SHIFT) |
|
|
(size << DART_CNTL_U3_SIZE_SHIFT));
|
|
}
|
|
|
|
/* Invalidate DART to get rid of possible stale TLBs */
|
|
dart_tlb_invalidate_all();
|
|
|
|
printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
|
|
dart_is_u4 ? "U4" : "U3");
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct iommu_table_ops iommu_dart_ops = {
|
|
.set = dart_build,
|
|
.clear = dart_free,
|
|
.flush = dart_flush,
|
|
};
|
|
|
|
static void iommu_table_dart_setup(void)
|
|
{
|
|
iommu_table_dart.it_busno = 0;
|
|
iommu_table_dart.it_offset = 0;
|
|
/* it_size is in number of entries */
|
|
iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
|
|
iommu_table_dart.it_page_shift = IOMMU_PAGE_SHIFT_4K;
|
|
|
|
/* Initialize the common IOMMU code */
|
|
iommu_table_dart.it_base = (unsigned long)dart_tablebase;
|
|
iommu_table_dart.it_index = 0;
|
|
iommu_table_dart.it_blocksize = 1;
|
|
iommu_table_dart.it_ops = &iommu_dart_ops;
|
|
iommu_init_table(&iommu_table_dart, -1);
|
|
|
|
/* Reserve the last page of the DART to avoid possible prefetch
|
|
* past the DART mapped area
|
|
*/
|
|
set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
|
|
}
|
|
|
|
static void pci_dma_dev_setup_dart(struct pci_dev *dev)
|
|
{
|
|
if (dart_is_u4)
|
|
set_dma_offset(&dev->dev, DART_U4_BYPASS_BASE);
|
|
set_iommu_table_base(&dev->dev, &iommu_table_dart);
|
|
}
|
|
|
|
static void pci_dma_bus_setup_dart(struct pci_bus *bus)
|
|
{
|
|
if (!iommu_table_dart_inited) {
|
|
iommu_table_dart_inited = 1;
|
|
iommu_table_dart_setup();
|
|
}
|
|
}
|
|
|
|
static bool dart_device_on_pcie(struct device *dev)
|
|
{
|
|
struct device_node *np = of_node_get(dev->of_node);
|
|
|
|
while(np) {
|
|
if (of_device_is_compatible(np, "U4-pcie") ||
|
|
of_device_is_compatible(np, "u4-pcie")) {
|
|
of_node_put(np);
|
|
return true;
|
|
}
|
|
np = of_get_next_parent(np);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static int dart_dma_set_mask(struct device *dev, u64 dma_mask)
|
|
{
|
|
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
|
|
return -EIO;
|
|
|
|
/* U4 supports a DART bypass, we use it for 64-bit capable
|
|
* devices to improve performances. However, that only works
|
|
* for devices connected to U4 own PCIe interface, not bridged
|
|
* through hypertransport. We need the device to support at
|
|
* least 40 bits of addresses.
|
|
*/
|
|
if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) {
|
|
dev_info(dev, "Using 64-bit DMA iommu bypass\n");
|
|
set_dma_ops(dev, &dma_direct_ops);
|
|
} else {
|
|
dev_info(dev, "Using 32-bit DMA via iommu\n");
|
|
set_dma_ops(dev, &dma_iommu_ops);
|
|
}
|
|
|
|
*dev->dma_mask = dma_mask;
|
|
return 0;
|
|
}
|
|
|
|
void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops)
|
|
{
|
|
struct device_node *dn;
|
|
|
|
/* Find the DART in the device-tree */
|
|
dn = of_find_compatible_node(NULL, "dart", "u3-dart");
|
|
if (dn == NULL) {
|
|
dn = of_find_compatible_node(NULL, "dart", "u4-dart");
|
|
if (dn == NULL)
|
|
return; /* use default direct_dma_ops */
|
|
dart_is_u4 = 1;
|
|
}
|
|
|
|
/* Initialize the DART HW */
|
|
if (dart_init(dn) != 0)
|
|
goto bail;
|
|
|
|
/* Setup bypass if supported */
|
|
if (dart_is_u4)
|
|
ppc_md.dma_set_mask = dart_dma_set_mask;
|
|
|
|
controller_ops->dma_dev_setup = pci_dma_dev_setup_dart;
|
|
controller_ops->dma_bus_setup = pci_dma_bus_setup_dart;
|
|
|
|
/* Setup pci_dma ops */
|
|
set_pci_dma_ops(&dma_iommu_ops);
|
|
return;
|
|
|
|
bail:
|
|
/* If init failed, use direct iommu and null setup functions */
|
|
controller_ops->dma_dev_setup = NULL;
|
|
controller_ops->dma_bus_setup = NULL;
|
|
|
|
/* Setup pci_dma ops */
|
|
set_pci_dma_ops(&dma_direct_ops);
|
|
}
|
|
|
|
#ifdef CONFIG_PM
|
|
static void iommu_dart_restore(void)
|
|
{
|
|
dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
|
|
dart_tlb_invalidate_all();
|
|
}
|
|
|
|
static int __init iommu_init_late_dart(void)
|
|
{
|
|
if (!dart_tablebase)
|
|
return 0;
|
|
|
|
ppc_md.iommu_restore = iommu_dart_restore;
|
|
|
|
return 0;
|
|
}
|
|
|
|
late_initcall(iommu_init_late_dart);
|
|
#endif /* CONFIG_PM */
|