mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
![Mike Rapoport](/assets/img/avatar_default.png)
Patch series "memblock: simplify several early memory allocation", v4. These patches simplify some of the early memory allocations by replacing usage of older memblock APIs with newer and shinier ones. Quite a few places in the arch/ code allocated memory using a memblock API that returns a physical address of the allocated area, then converted this physical address to a virtual one and then used memset(0) to clear the allocated range. More recent memblock APIs do all the three steps in one call and their usage simplifies the code. It's important to note that regardless of API used, the core allocation is nearly identical for any set of memblock allocators: first it tries to find a free memory with all the constraints specified by the caller and then falls back to the allocation with some or all constraints disabled. The first three patches perform the conversion of call sites that have exact requirements for the node and the possible memory range. The fourth patch is a bit one-off as it simplifies openrisc's implementation of pte_alloc_one_kernel(), and not only the memblock usage. The fifth patch takes care of simpler cases when the allocation can be satisfied with a simple call to memblock_alloc(). The sixth patch removes one-liner wrappers for memblock_alloc on arm and unicore32, as suggested by Christoph. This patch (of 6): There are a several places that allocate memory using memblock APIs that return a physical address, convert the returned address to the virtual address and frequently also memset(0) the allocated range. Update these places to use memblock allocators already returning a virtual address. Use memblock functions that clear the allocated memory instead of calling memset(0) where appropriate. The calls to memblock_alloc_base() that were not followed by memset(0) are replaced with memblock_alloc_try_nid_raw(). Since the latter does not panic() when the allocation fails, the appropriate panic() calls are added to the call sites. Link: http://lkml.kernel.org/r/1546248566-14910-2-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport <rppt@linux.ibm.com> Cc: Arnd Bergmann <arnd@arndb.de> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: "David S. Miller" <davem@davemloft.net> Cc: Guan Xuetao <gxt@pku.edu.cn> Cc: Greentime Hu <green.hu@gmail.com> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Jonas Bonn <jonas@southpole.se> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Michael Ellerman <mpe@ellerman.id.au> Cc: Michal Hocko <mhocko@suse.com> Cc: Michal Simek <monstr@monstr.eu> Cc: Mark Salter <msalter@redhat.com> Cc: Paul Mackerras <paulus@samba.org> Cc: Rich Felker <dalias@libc.org> Cc: Russell King <linux@armlinux.org.uk> Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi> Cc: Stafford Horne <shorne@gmail.com> Cc: Vincent Chen <deanbo422@gmail.com> Cc: Yoshinori Sato <ysato@users.sourceforge.jp> Cc: Christoph Hellwig <hch@infradead.org> Cc: Michal Simek <michal.simek@xilinx.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
475 lines
12 KiB
C
475 lines
12 KiB
C
/*
|
|
* arch/powerpc/sysdev/dart_iommu.c
|
|
*
|
|
* Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
|
|
* Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
|
|
* IBM Corporation
|
|
*
|
|
* Based on pSeries_iommu.c:
|
|
* Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
|
|
* Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
|
|
*
|
|
* Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
|
|
*
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/types.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/string.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/suspend.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/gfp.h>
|
|
#include <linux/kmemleak.h>
|
|
#include <asm/io.h>
|
|
#include <asm/prom.h>
|
|
#include <asm/iommu.h>
|
|
#include <asm/pci-bridge.h>
|
|
#include <asm/machdep.h>
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/ppc-pci.h>
|
|
|
|
#include "dart.h"
|
|
|
|
/* DART table address and size */
|
|
static u32 *dart_tablebase;
|
|
static unsigned long dart_tablesize;
|
|
|
|
/* Mapped base address for the dart */
|
|
static unsigned int __iomem *dart;
|
|
|
|
/* Dummy val that entries are set to when unused */
|
|
static unsigned int dart_emptyval;
|
|
|
|
static struct iommu_table iommu_table_dart;
|
|
static int iommu_table_dart_inited;
|
|
static int dart_dirty;
|
|
static int dart_is_u4;
|
|
|
|
#define DART_U4_BYPASS_BASE 0x8000000000ull
|
|
|
|
#define DBG(...)
|
|
|
|
static DEFINE_SPINLOCK(invalidate_lock);
|
|
|
|
static inline void dart_tlb_invalidate_all(void)
|
|
{
|
|
unsigned long l = 0;
|
|
unsigned int reg, inv_bit;
|
|
unsigned long limit;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&invalidate_lock, flags);
|
|
|
|
DBG("dart: flush\n");
|
|
|
|
/* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
|
|
* control register and wait for it to clear.
|
|
*
|
|
* Gotcha: Sometimes, the DART won't detect that the bit gets
|
|
* set. If so, clear it and set it again.
|
|
*/
|
|
|
|
limit = 0;
|
|
|
|
inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
|
|
retry:
|
|
l = 0;
|
|
reg = DART_IN(DART_CNTL);
|
|
reg |= inv_bit;
|
|
DART_OUT(DART_CNTL, reg);
|
|
|
|
while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
|
|
l++;
|
|
if (l == (1L << limit)) {
|
|
if (limit < 4) {
|
|
limit++;
|
|
reg = DART_IN(DART_CNTL);
|
|
reg &= ~inv_bit;
|
|
DART_OUT(DART_CNTL, reg);
|
|
goto retry;
|
|
} else
|
|
panic("DART: TLB did not flush after waiting a long "
|
|
"time. Buggy U3 ?");
|
|
}
|
|
|
|
spin_unlock_irqrestore(&invalidate_lock, flags);
|
|
}
|
|
|
|
static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
|
|
{
|
|
unsigned int reg;
|
|
unsigned int l, limit;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&invalidate_lock, flags);
|
|
|
|
reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
|
|
(bus_rpn & DART_CNTL_U4_IONE_MASK);
|
|
DART_OUT(DART_CNTL, reg);
|
|
|
|
limit = 0;
|
|
wait_more:
|
|
l = 0;
|
|
while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
|
|
rmb();
|
|
l++;
|
|
}
|
|
|
|
if (l == (1L << limit)) {
|
|
if (limit < 4) {
|
|
limit++;
|
|
goto wait_more;
|
|
} else
|
|
panic("DART: TLB did not flush after waiting a long "
|
|
"time. Buggy U4 ?");
|
|
}
|
|
|
|
spin_unlock_irqrestore(&invalidate_lock, flags);
|
|
}
|
|
|
|
static void dart_cache_sync(unsigned int *base, unsigned int count)
|
|
{
|
|
/*
|
|
* We add 1 to the number of entries to flush, following a
|
|
* comment in Darwin indicating that the memory controller
|
|
* can prefetch unmapped memory under some circumstances.
|
|
*/
|
|
unsigned long start = (unsigned long)base;
|
|
unsigned long end = start + (count + 1) * sizeof(unsigned int);
|
|
unsigned int tmp;
|
|
|
|
/* Perform a standard cache flush */
|
|
flush_inval_dcache_range(start, end);
|
|
|
|
/*
|
|
* Perform the sequence described in the CPC925 manual to
|
|
* ensure all the data gets to a point the cache incoherent
|
|
* DART hardware will see.
|
|
*/
|
|
asm volatile(" sync;"
|
|
" isync;"
|
|
" dcbf 0,%1;"
|
|
" sync;"
|
|
" isync;"
|
|
" lwz %0,0(%1);"
|
|
" isync" : "=r" (tmp) : "r" (end) : "memory");
|
|
}
|
|
|
|
static void dart_flush(struct iommu_table *tbl)
|
|
{
|
|
mb();
|
|
if (dart_dirty) {
|
|
dart_tlb_invalidate_all();
|
|
dart_dirty = 0;
|
|
}
|
|
}
|
|
|
|
static int dart_build(struct iommu_table *tbl, long index,
|
|
long npages, unsigned long uaddr,
|
|
enum dma_data_direction direction,
|
|
unsigned long attrs)
|
|
{
|
|
unsigned int *dp, *orig_dp;
|
|
unsigned int rpn;
|
|
long l;
|
|
|
|
DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
|
|
|
|
orig_dp = dp = ((unsigned int*)tbl->it_base) + index;
|
|
|
|
/* On U3, all memory is contiguous, so we can move this
|
|
* out of the loop.
|
|
*/
|
|
l = npages;
|
|
while (l--) {
|
|
rpn = __pa(uaddr) >> DART_PAGE_SHIFT;
|
|
|
|
*(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
|
|
|
|
uaddr += DART_PAGE_SIZE;
|
|
}
|
|
dart_cache_sync(orig_dp, npages);
|
|
|
|
if (dart_is_u4) {
|
|
rpn = index;
|
|
while (npages--)
|
|
dart_tlb_invalidate_one(rpn++);
|
|
} else {
|
|
dart_dirty = 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
static void dart_free(struct iommu_table *tbl, long index, long npages)
|
|
{
|
|
unsigned int *dp, *orig_dp;
|
|
long orig_npages = npages;
|
|
|
|
/* We don't worry about flushing the TLB cache. The only drawback of
|
|
* not doing it is that we won't catch buggy device drivers doing
|
|
* bad DMAs, but then no 32-bit architecture ever does either.
|
|
*/
|
|
|
|
DBG("dart: free at: %lx, %lx\n", index, npages);
|
|
|
|
orig_dp = dp = ((unsigned int *)tbl->it_base) + index;
|
|
|
|
while (npages--)
|
|
*(dp++) = dart_emptyval;
|
|
|
|
dart_cache_sync(orig_dp, orig_npages);
|
|
}
|
|
|
|
static void allocate_dart(void)
|
|
{
|
|
unsigned long tmp;
|
|
|
|
/* 512 pages (2MB) is max DART tablesize. */
|
|
dart_tablesize = 1UL << 21;
|
|
|
|
/*
|
|
* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
|
|
* will blow up an entire large page anyway in the kernel mapping.
|
|
*/
|
|
dart_tablebase = memblock_alloc_try_nid_raw(SZ_16M, SZ_16M,
|
|
MEMBLOCK_LOW_LIMIT, SZ_2G,
|
|
NUMA_NO_NODE);
|
|
if (!dart_tablebase)
|
|
panic("Failed to allocate 16MB below 2GB for DART table\n");
|
|
|
|
/* There is no point scanning the DART space for leaks*/
|
|
kmemleak_no_scan((void *)dart_tablebase);
|
|
|
|
/* Allocate a spare page to map all invalid DART pages. We need to do
|
|
* that to work around what looks like a problem with the HT bridge
|
|
* prefetching into invalid pages and corrupting data
|
|
*/
|
|
tmp = memblock_phys_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
|
|
dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
|
|
DARTMAP_RPNMASK);
|
|
|
|
printk(KERN_INFO "DART table allocated at: %p\n", dart_tablebase);
|
|
}
|
|
|
|
static int __init dart_init(struct device_node *dart_node)
|
|
{
|
|
unsigned int i;
|
|
unsigned long base, size;
|
|
struct resource r;
|
|
|
|
/* IOMMU disabled by the user ? bail out */
|
|
if (iommu_is_off)
|
|
return -ENODEV;
|
|
|
|
/*
|
|
* Only use the DART if the machine has more than 1GB of RAM
|
|
* or if requested with iommu=on on cmdline.
|
|
*
|
|
* 1GB of RAM is picked as limit because some default devices
|
|
* (i.e. Airport Extreme) have 30 bit address range limits.
|
|
*/
|
|
|
|
if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull)
|
|
return -ENODEV;
|
|
|
|
/* Get DART registers */
|
|
if (of_address_to_resource(dart_node, 0, &r))
|
|
panic("DART: can't get register base ! ");
|
|
|
|
/* Map in DART registers */
|
|
dart = ioremap(r.start, resource_size(&r));
|
|
if (dart == NULL)
|
|
panic("DART: Cannot map registers!");
|
|
|
|
/* Allocate the DART and dummy page */
|
|
allocate_dart();
|
|
|
|
/* Fill initial table */
|
|
for (i = 0; i < dart_tablesize/4; i++)
|
|
dart_tablebase[i] = dart_emptyval;
|
|
|
|
/* Push to memory */
|
|
dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
|
|
|
|
/* Initialize DART with table base and enable it. */
|
|
base = ((unsigned long)dart_tablebase) >> DART_PAGE_SHIFT;
|
|
size = dart_tablesize >> DART_PAGE_SHIFT;
|
|
if (dart_is_u4) {
|
|
size &= DART_SIZE_U4_SIZE_MASK;
|
|
DART_OUT(DART_BASE_U4, base);
|
|
DART_OUT(DART_SIZE_U4, size);
|
|
DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
|
|
} else {
|
|
size &= DART_CNTL_U3_SIZE_MASK;
|
|
DART_OUT(DART_CNTL,
|
|
DART_CNTL_U3_ENABLE |
|
|
(base << DART_CNTL_U3_BASE_SHIFT) |
|
|
(size << DART_CNTL_U3_SIZE_SHIFT));
|
|
}
|
|
|
|
/* Invalidate DART to get rid of possible stale TLBs */
|
|
dart_tlb_invalidate_all();
|
|
|
|
printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
|
|
dart_is_u4 ? "U4" : "U3");
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct iommu_table_ops iommu_dart_ops = {
|
|
.set = dart_build,
|
|
.clear = dart_free,
|
|
.flush = dart_flush,
|
|
};
|
|
|
|
static void iommu_table_dart_setup(void)
|
|
{
|
|
iommu_table_dart.it_busno = 0;
|
|
iommu_table_dart.it_offset = 0;
|
|
/* it_size is in number of entries */
|
|
iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
|
|
iommu_table_dart.it_page_shift = IOMMU_PAGE_SHIFT_4K;
|
|
|
|
/* Initialize the common IOMMU code */
|
|
iommu_table_dart.it_base = (unsigned long)dart_tablebase;
|
|
iommu_table_dart.it_index = 0;
|
|
iommu_table_dart.it_blocksize = 1;
|
|
iommu_table_dart.it_ops = &iommu_dart_ops;
|
|
iommu_init_table(&iommu_table_dart, -1);
|
|
|
|
/* Reserve the last page of the DART to avoid possible prefetch
|
|
* past the DART mapped area
|
|
*/
|
|
set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
|
|
}
|
|
|
|
static void pci_dma_dev_setup_dart(struct pci_dev *dev)
|
|
{
|
|
if (dart_is_u4)
|
|
set_dma_offset(&dev->dev, DART_U4_BYPASS_BASE);
|
|
set_iommu_table_base(&dev->dev, &iommu_table_dart);
|
|
}
|
|
|
|
static void pci_dma_bus_setup_dart(struct pci_bus *bus)
|
|
{
|
|
if (!iommu_table_dart_inited) {
|
|
iommu_table_dart_inited = 1;
|
|
iommu_table_dart_setup();
|
|
}
|
|
}
|
|
|
|
static bool dart_device_on_pcie(struct device *dev)
|
|
{
|
|
struct device_node *np = of_node_get(dev->of_node);
|
|
|
|
while(np) {
|
|
if (of_device_is_compatible(np, "U4-pcie") ||
|
|
of_device_is_compatible(np, "u4-pcie")) {
|
|
of_node_put(np);
|
|
return true;
|
|
}
|
|
np = of_get_next_parent(np);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static int dart_dma_set_mask(struct device *dev, u64 dma_mask)
|
|
{
|
|
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
|
|
return -EIO;
|
|
|
|
/* U4 supports a DART bypass, we use it for 64-bit capable
|
|
* devices to improve performances. However, that only works
|
|
* for devices connected to U4 own PCIe interface, not bridged
|
|
* through hypertransport. We need the device to support at
|
|
* least 40 bits of addresses.
|
|
*/
|
|
if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) {
|
|
dev_info(dev, "Using 64-bit DMA iommu bypass\n");
|
|
set_dma_ops(dev, &dma_nommu_ops);
|
|
} else {
|
|
dev_info(dev, "Using 32-bit DMA via iommu\n");
|
|
set_dma_ops(dev, &dma_iommu_ops);
|
|
}
|
|
|
|
*dev->dma_mask = dma_mask;
|
|
return 0;
|
|
}
|
|
|
|
void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops)
|
|
{
|
|
struct device_node *dn;
|
|
|
|
/* Find the DART in the device-tree */
|
|
dn = of_find_compatible_node(NULL, "dart", "u3-dart");
|
|
if (dn == NULL) {
|
|
dn = of_find_compatible_node(NULL, "dart", "u4-dart");
|
|
if (dn == NULL)
|
|
return; /* use default direct_dma_ops */
|
|
dart_is_u4 = 1;
|
|
}
|
|
|
|
/* Initialize the DART HW */
|
|
if (dart_init(dn) != 0)
|
|
goto bail;
|
|
|
|
/* Setup bypass if supported */
|
|
if (dart_is_u4)
|
|
ppc_md.dma_set_mask = dart_dma_set_mask;
|
|
|
|
controller_ops->dma_dev_setup = pci_dma_dev_setup_dart;
|
|
controller_ops->dma_bus_setup = pci_dma_bus_setup_dart;
|
|
|
|
/* Setup pci_dma ops */
|
|
set_pci_dma_ops(&dma_iommu_ops);
|
|
return;
|
|
|
|
bail:
|
|
/* If init failed, use direct iommu and null setup functions */
|
|
controller_ops->dma_dev_setup = NULL;
|
|
controller_ops->dma_bus_setup = NULL;
|
|
|
|
/* Setup pci_dma ops */
|
|
set_pci_dma_ops(&dma_nommu_ops);
|
|
}
|
|
|
|
#ifdef CONFIG_PM
|
|
static void iommu_dart_restore(void)
|
|
{
|
|
dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
|
|
dart_tlb_invalidate_all();
|
|
}
|
|
|
|
static int __init iommu_init_late_dart(void)
|
|
{
|
|
if (!dart_tablebase)
|
|
return 0;
|
|
|
|
ppc_md.iommu_restore = iommu_dart_restore;
|
|
|
|
return 0;
|
|
}
|
|
|
|
late_initcall(iommu_init_late_dart);
|
|
#endif /* CONFIG_PM */
|