arm64: dma-mapping: add support for IOMMU mapper
On systems with IOMMUs, it's useful to handle IOMMU mappings in the
dma-mapping layer. This is currently supported on arm but not arm64. Add
support in arm64 by gratuitously lifting most of the IOMMU-related stuff
from dma-mapping.c in arm.
The original arm work was done by Marek Szyprowski in [4ce63fcd91:
"ARM: dma-mapping: add support for IOMMU mapper"].
Change-Id: I1c3c8fe15049fe456751074398fd179ebd2ec64e
Signed-off-by: Mitchel Humpherys <mitchelh@codeaurora.org>
Signed-off-by: Patrick Daly <pdaly@codeaurora.org>
This commit is contained in:
Patrick Daly
parent
cc3ff31083
commit
f0dbb6af93
4 changed files with 1044 additions and 0 deletions
|
|
@ -219,6 +219,32 @@ config NEED_SG_DMA_LENGTH
|
|||
config SMP
|
||||
def_bool y
|
||||
|
||||
config ARM64_DMA_USE_IOMMU
|
||||
bool
|
||||
select ARM_HAS_SG_CHAIN
|
||||
select NEED_SG_DMA_LENGTH
|
||||
|
||||
if ARM64_DMA_USE_IOMMU
|
||||
|
||||
config ARM64_DMA_IOMMU_ALIGNMENT
|
||||
int "Maximum PAGE_SIZE order of alignment for DMA IOMMU buffers"
|
||||
range 4 9
|
||||
default 8
|
||||
help
|
||||
DMA mapping framework by default aligns all buffers to the smallest
|
||||
PAGE_SIZE order which is greater than or equal to the requested buffer
|
||||
size. This works well for buffers up to a few hundreds kilobytes, but
|
||||
for larger buffers it just a waste of address space. Drivers which has
|
||||
relatively small addressing window (like 64Mib) might run out of
|
||||
virtual space with just a few allocations.
|
||||
|
||||
With this parameter you can specify the maximum PAGE_SIZE order for
|
||||
DMA IOMMU buffers. Larger buffers will be aligned only to this
|
||||
specified order. The order is expressed as a power of two multiplied
|
||||
by the PAGE_SIZE.
|
||||
|
||||
endif
|
||||
|
||||
config SWIOTLB
|
||||
def_bool y
|
||||
|
||||
|
|
|
|||
|
|
@ -22,10 +22,19 @@ struct dev_archdata {
|
|||
void *iommu; /* private IOMMU data */
|
||||
#endif
|
||||
bool dma_coherent;
|
||||
#ifdef CONFIG_ARM64_DMA_USE_IOMMU
|
||||
struct dma_iommu_mapping *mapping;
|
||||
#endif
|
||||
};
|
||||
|
||||
struct pdev_archdata {
|
||||
u64 dma_mask;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_ARM64_DMA_USE_IOMMU
|
||||
#define to_dma_iommu_mapping(dev) ((dev)->archdata.mapping)
|
||||
#else
|
||||
#define to_dma_iommu_mapping(dev) NULL
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
|
|||
36
arch/arm64/include/asm/dma-iommu.h
Normal file
36
arch/arm64/include/asm/dma-iommu.h
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
#ifndef ASMARM_DMA_IOMMU_H
|
||||
#define ASMARM_DMA_IOMMU_H
|
||||
|
||||
#ifdef __KERNEL__
|
||||
|
||||
#include <linux/mm_types.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/dma-debug.h>
|
||||
#include <linux/kmemcheck.h>
|
||||
#include <linux/kref.h>
|
||||
|
||||
struct dma_iommu_mapping {
|
||||
/* iommu specific data */
|
||||
struct iommu_domain *domain;
|
||||
|
||||
void *bitmap;
|
||||
size_t bits;
|
||||
unsigned int order;
|
||||
dma_addr_t base;
|
||||
|
||||
spinlock_t lock;
|
||||
struct kref kref;
|
||||
};
|
||||
|
||||
struct dma_iommu_mapping *
|
||||
arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
|
||||
int order);
|
||||
|
||||
void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping);
|
||||
|
||||
int arm_iommu_attach_device(struct device *dev,
|
||||
struct dma_iommu_mapping *mapping);
|
||||
void arm_iommu_detach_device(struct device *dev);
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
#endif
|
||||
|
|
@ -25,12 +25,18 @@
|
|||
#include <linux/genalloc.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/dma-contiguous.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/swiotlb.h>
|
||||
#include <linux/io.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/tlbflush.h>
|
||||
#include <asm/dma-iommu.h>
|
||||
|
||||
#include "mm.h"
|
||||
|
||||
|
||||
static int swiotlb __read_mostly;
|
||||
|
||||
|
|
@ -1073,3 +1079,970 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
|
|||
dev->archdata.dma_coherent = coherent;
|
||||
__iommu_setup_dma_ops(dev, dma_base, size, iommu);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM64_DMA_USE_IOMMU
|
||||
|
||||
/*
|
||||
* Make an area consistent for devices.
|
||||
* Note: Drivers should NOT use this function directly, as it will break
|
||||
* platforms with CONFIG_DMABOUNCE.
|
||||
* Use the driver DMA support - see dma-mapping.h (dma_sync_*)
|
||||
*/
|
||||
static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
|
||||
size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
__dma_map_area(page_address(page) + off, size, dir);
|
||||
}
|
||||
|
||||
static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
|
||||
size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
__dma_unmap_area(page_address(page) + off, size, dir);
|
||||
|
||||
/*
|
||||
* Mark the D-cache clean for this page to avoid extra flushing.
|
||||
*/
|
||||
if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
|
||||
set_bit(PG_dcache_clean, &page->flags);
|
||||
}
|
||||
|
||||
static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
|
||||
{
|
||||
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
|
||||
return -EIO;
|
||||
|
||||
*dev->dma_mask = dma_mask;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* IOMMU */
|
||||
|
||||
static void __dma_clear_buffer(struct page *page, size_t size,
|
||||
unsigned long attrs)
|
||||
{
|
||||
/*
|
||||
* Ensure that the allocated pages are zeroed, and that any data
|
||||
* lurking in the kernel direct-mapped region is invalidated.
|
||||
*/
|
||||
void *ptr = page_address(page);
|
||||
|
||||
if (!(attrs & DMA_ATTR_SKIP_ZEROING))
|
||||
memset(ptr, 0, size);
|
||||
__dma_flush_range(ptr, ptr + size);
|
||||
}
|
||||
|
||||
static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
|
||||
size_t size)
|
||||
{
|
||||
unsigned int order = get_order(size);
|
||||
unsigned int align = 0;
|
||||
unsigned int count, start;
|
||||
unsigned long flags;
|
||||
|
||||
if (order > CONFIG_ARM64_DMA_IOMMU_ALIGNMENT)
|
||||
order = CONFIG_ARM64_DMA_IOMMU_ALIGNMENT;
|
||||
|
||||
count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
|
||||
(1 << mapping->order) - 1) >> mapping->order;
|
||||
|
||||
if (order > mapping->order)
|
||||
align = (1 << (order - mapping->order)) - 1;
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
|
||||
count, align);
|
||||
if (start > mapping->bits) {
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
return DMA_ERROR_CODE;
|
||||
}
|
||||
|
||||
bitmap_set(mapping->bitmap, start, count);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
|
||||
return mapping->base + (start << (mapping->order + PAGE_SHIFT));
|
||||
}
|
||||
|
||||
static inline void __free_iova(struct dma_iommu_mapping *mapping,
|
||||
dma_addr_t addr, size_t size)
|
||||
{
|
||||
unsigned int start = (addr - mapping->base) >>
|
||||
(mapping->order + PAGE_SHIFT);
|
||||
unsigned int count = ((size >> PAGE_SHIFT) +
|
||||
(1 << mapping->order) - 1) >> mapping->order;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
bitmap_clear(mapping->bitmap, start, count);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
}
|
||||
|
||||
static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
|
||||
gfp_t gfp, unsigned long attrs)
|
||||
{
|
||||
struct page **pages;
|
||||
int count = size >> PAGE_SHIFT;
|
||||
int array_size = count * sizeof(struct page *);
|
||||
int i = 0;
|
||||
|
||||
if (array_size <= PAGE_SIZE)
|
||||
pages = kzalloc(array_size, gfp);
|
||||
else
|
||||
pages = vzalloc(array_size);
|
||||
if (!pages)
|
||||
return NULL;
|
||||
|
||||
if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
|
||||
unsigned long order = get_order(size);
|
||||
struct page *page;
|
||||
|
||||
page = dma_alloc_from_contiguous(dev, count, order);
|
||||
if (!page)
|
||||
goto error;
|
||||
|
||||
__dma_clear_buffer(page, size, attrs);
|
||||
|
||||
for (i = 0; i < count; i++)
|
||||
pages[i] = page + i;
|
||||
|
||||
return pages;
|
||||
}
|
||||
|
||||
/*
|
||||
* IOMMU can map any pages, so himem can also be used here
|
||||
*/
|
||||
gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
|
||||
|
||||
while (count) {
|
||||
int j, order = __fls(count);
|
||||
|
||||
pages[i] = alloc_pages(gfp, order);
|
||||
while (!pages[i] && order)
|
||||
pages[i] = alloc_pages(gfp, --order);
|
||||
if (!pages[i])
|
||||
goto error;
|
||||
|
||||
if (order) {
|
||||
split_page(pages[i], order);
|
||||
j = 1 << order;
|
||||
while (--j)
|
||||
pages[i + j] = pages[i] + j;
|
||||
}
|
||||
|
||||
__dma_clear_buffer(pages[i], PAGE_SIZE << order, attrs);
|
||||
i += 1 << order;
|
||||
count -= 1 << order;
|
||||
}
|
||||
|
||||
return pages;
|
||||
error:
|
||||
while (i--)
|
||||
if (pages[i])
|
||||
__free_pages(pages[i], 0);
|
||||
if (array_size <= PAGE_SIZE)
|
||||
kfree(pages);
|
||||
else
|
||||
vfree(pages);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int __iommu_free_buffer(struct device *dev, struct page **pages,
|
||||
size_t size, unsigned long attrs)
|
||||
{
|
||||
int count = size >> PAGE_SHIFT;
|
||||
int array_size = count * sizeof(struct page *);
|
||||
int i;
|
||||
|
||||
if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
|
||||
dma_release_from_contiguous(dev, pages[0], count);
|
||||
} else {
|
||||
for (i = 0; i < count; i++)
|
||||
if (pages[i])
|
||||
__free_pages(pages[i], 0);
|
||||
}
|
||||
|
||||
if (array_size <= PAGE_SIZE)
|
||||
kfree(pages);
|
||||
else
|
||||
vfree(pages);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Create a CPU mapping for a specified pages
|
||||
*/
|
||||
static void *
|
||||
__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
|
||||
const void *caller)
|
||||
{
|
||||
return dma_common_pages_remap(pages, size, VM_USERMAP, prot, caller);
|
||||
}
|
||||
|
||||
/*
|
||||
* Create a mapping in device IO address space for specified pages
|
||||
*/
|
||||
static dma_addr_t
|
||||
__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
||||
dma_addr_t dma_addr, iova;
|
||||
int i, ret;
|
||||
|
||||
dma_addr = __alloc_iova(mapping, size);
|
||||
if (dma_addr == DMA_ERROR_CODE)
|
||||
return dma_addr;
|
||||
|
||||
iova = dma_addr;
|
||||
for (i = 0; i < count; ) {
|
||||
unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
|
||||
phys_addr_t phys = page_to_phys(pages[i]);
|
||||
unsigned int len, j;
|
||||
|
||||
for (j = i + 1; j < count; j++, next_pfn++)
|
||||
if (page_to_pfn(pages[j]) != next_pfn)
|
||||
break;
|
||||
|
||||
len = (j - i) << PAGE_SHIFT;
|
||||
ret = iommu_map(mapping->domain, iova, phys, len,
|
||||
IOMMU_READ|IOMMU_WRITE);
|
||||
if (ret < 0)
|
||||
goto fail;
|
||||
iova += len;
|
||||
i = j;
|
||||
}
|
||||
return dma_addr;
|
||||
fail:
|
||||
iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
|
||||
__free_iova(mapping, dma_addr, size);
|
||||
return DMA_ERROR_CODE;
|
||||
}
|
||||
|
||||
static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova,
|
||||
size_t size)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
|
||||
/*
|
||||
* add optional in-page offset from iova to size and align
|
||||
* result to page size
|
||||
*/
|
||||
size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
|
||||
iova &= PAGE_MASK;
|
||||
|
||||
iommu_unmap(mapping->domain, iova, size);
|
||||
__free_iova(mapping, iova, size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct page **__atomic_get_pages(void *addr)
|
||||
{
|
||||
struct page *page;
|
||||
phys_addr_t phys;
|
||||
|
||||
phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
|
||||
page = phys_to_page(phys);
|
||||
|
||||
return (struct page **)page;
|
||||
}
|
||||
|
||||
static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs)
|
||||
{
|
||||
struct vm_struct *area;
|
||||
|
||||
if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
|
||||
return __atomic_get_pages(cpu_addr);
|
||||
|
||||
if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
|
||||
return cpu_addr;
|
||||
|
||||
area = find_vm_area(cpu_addr);
|
||||
if (area)
|
||||
return area->pages;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void *__iommu_alloc_atomic(struct device *dev, size_t size,
|
||||
dma_addr_t *handle, gfp_t gfp)
|
||||
{
|
||||
struct page *page;
|
||||
void *addr;
|
||||
|
||||
addr = __alloc_from_pool(size, &page, gfp);
|
||||
if (!addr)
|
||||
return NULL;
|
||||
|
||||
*handle = __iommu_create_mapping(dev, &page, size);
|
||||
if (*handle == DMA_ERROR_CODE)
|
||||
goto err_mapping;
|
||||
|
||||
return addr;
|
||||
|
||||
err_mapping:
|
||||
__free_from_pool(addr, size);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
|
||||
dma_addr_t handle, size_t size)
|
||||
{
|
||||
__iommu_remove_mapping(dev, handle, size);
|
||||
__free_from_pool(cpu_addr, size);
|
||||
}
|
||||
|
||||
static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
|
||||
dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
|
||||
{
|
||||
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, true);
|
||||
struct page **pages;
|
||||
void *addr = NULL;
|
||||
|
||||
*handle = DMA_ERROR_CODE;
|
||||
size = PAGE_ALIGN(size);
|
||||
|
||||
if (!gfpflags_allow_blocking(gfp))
|
||||
return __iommu_alloc_atomic(dev, size, handle, gfp);
|
||||
|
||||
/*
|
||||
* Following is a work-around (a.k.a. hack) to prevent pages
|
||||
* with __GFP_COMP being passed to split_page() which cannot
|
||||
* handle them. The real problem is that this flag probably
|
||||
* should be 0 on ARM as it is not supported on this
|
||||
* platform; see CONFIG_HUGETLBFS.
|
||||
*/
|
||||
gfp &= ~(__GFP_COMP);
|
||||
|
||||
pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
|
||||
if (!pages)
|
||||
return NULL;
|
||||
|
||||
*handle = __iommu_create_mapping(dev, pages, size);
|
||||
if (*handle == DMA_ERROR_CODE)
|
||||
goto err_buffer;
|
||||
|
||||
if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
|
||||
return pages;
|
||||
|
||||
addr = __iommu_alloc_remap(pages, size, gfp, prot,
|
||||
__builtin_return_address(0));
|
||||
if (!addr)
|
||||
goto err_mapping;
|
||||
|
||||
return addr;
|
||||
|
||||
err_mapping:
|
||||
__iommu_remove_mapping(dev, *handle, size);
|
||||
err_buffer:
|
||||
__iommu_free_buffer(dev, pages, size, attrs);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
|
||||
void *cpu_addr, dma_addr_t dma_addr, size_t size,
|
||||
unsigned long attrs)
|
||||
{
|
||||
unsigned long uaddr = vma->vm_start;
|
||||
unsigned long usize = vma->vm_end - vma->vm_start;
|
||||
struct page **pages = __iommu_get_pages(cpu_addr, attrs);
|
||||
|
||||
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, true);
|
||||
|
||||
if (!pages)
|
||||
return -ENXIO;
|
||||
|
||||
do {
|
||||
int ret = vm_insert_page(vma, uaddr, *pages++);
|
||||
|
||||
if (ret) {
|
||||
pr_err("Remapping memory failed: %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
uaddr += PAGE_SIZE;
|
||||
usize -= PAGE_SIZE;
|
||||
} while (usize > 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* free a page as defined by the above mapping.
|
||||
* Must not be called with IRQs disabled.
|
||||
*/
|
||||
void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
|
||||
dma_addr_t handle, unsigned long attrs)
|
||||
{
|
||||
struct page **pages;
|
||||
|
||||
size = PAGE_ALIGN(size);
|
||||
|
||||
if (__in_atomic_pool(cpu_addr, size)) {
|
||||
__iommu_free_atomic(dev, cpu_addr, handle, size);
|
||||
return;
|
||||
}
|
||||
|
||||
pages = __iommu_get_pages(cpu_addr, attrs);
|
||||
if (!pages) {
|
||||
WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!(attrs & DMA_ATTR_NO_KERNEL_MAPPING))
|
||||
dma_common_free_remap(cpu_addr, size, VM_USERMAP, true);
|
||||
|
||||
__iommu_remove_mapping(dev, handle, size);
|
||||
__iommu_free_buffer(dev, pages, size, attrs);
|
||||
}
|
||||
|
||||
int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
|
||||
void *cpu_addr, dma_addr_t dma_addr,
|
||||
size_t size, unsigned long attrs)
|
||||
{
|
||||
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
||||
struct page **pages = __iommu_get_pages(cpu_addr, attrs);
|
||||
|
||||
if (!pages)
|
||||
return -ENXIO;
|
||||
|
||||
return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
|
||||
GFP_KERNEL);
|
||||
}
|
||||
|
||||
static int __dma_direction_to_prot(enum dma_data_direction dir)
|
||||
{
|
||||
int prot;
|
||||
|
||||
switch (dir) {
|
||||
case DMA_BIDIRECTIONAL:
|
||||
prot = IOMMU_READ | IOMMU_WRITE;
|
||||
break;
|
||||
case DMA_TO_DEVICE:
|
||||
prot = IOMMU_READ;
|
||||
break;
|
||||
case DMA_FROM_DEVICE:
|
||||
prot = IOMMU_WRITE;
|
||||
break;
|
||||
default:
|
||||
prot = 0;
|
||||
}
|
||||
|
||||
return prot;
|
||||
}
|
||||
|
||||
/*
|
||||
* Map a part of the scatter-gather list into contiguous io address space
|
||||
*/
|
||||
static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
|
||||
size_t size, dma_addr_t *handle,
|
||||
enum dma_data_direction dir, unsigned long attrs,
|
||||
bool is_coherent)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
dma_addr_t iova, iova_base;
|
||||
int ret = 0;
|
||||
unsigned int count;
|
||||
struct scatterlist *s;
|
||||
int prot;
|
||||
|
||||
size = PAGE_ALIGN(size);
|
||||
*handle = DMA_ERROR_CODE;
|
||||
|
||||
iova_base = iova = __alloc_iova(mapping, size);
|
||||
if (iova == DMA_ERROR_CODE)
|
||||
return -ENOMEM;
|
||||
|
||||
for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
|
||||
phys_addr_t phys = page_to_phys(sg_page(s));
|
||||
unsigned int len = PAGE_ALIGN(s->offset + s->length);
|
||||
|
||||
if (!is_coherent &&
|
||||
!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length,
|
||||
dir);
|
||||
|
||||
prot = __dma_direction_to_prot(dir);
|
||||
|
||||
ret = iommu_map(mapping->domain, iova, phys, len, prot);
|
||||
if (ret < 0)
|
||||
goto fail;
|
||||
count += len >> PAGE_SHIFT;
|
||||
iova += len;
|
||||
}
|
||||
*handle = iova_base;
|
||||
|
||||
return 0;
|
||||
fail:
|
||||
iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
|
||||
__free_iova(mapping, iova_base, size);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, unsigned long attrs,
|
||||
bool is_coherent)
|
||||
{
|
||||
struct scatterlist *s = sg, *dma = sg, *start = sg;
|
||||
int i, count = 0;
|
||||
unsigned int offset = s->offset;
|
||||
unsigned int size = s->offset + s->length;
|
||||
unsigned int max = dma_get_max_seg_size(dev);
|
||||
|
||||
for (i = 1; i < nents; i++) {
|
||||
s = sg_next(s);
|
||||
|
||||
s->dma_address = DMA_ERROR_CODE;
|
||||
s->dma_length = 0;
|
||||
|
||||
if (s->offset || (size & ~PAGE_MASK)
|
||||
|| size + s->length > max) {
|
||||
if (__map_sg_chunk(dev, start, size, &dma->dma_address,
|
||||
dir, attrs, is_coherent) < 0)
|
||||
goto bad_mapping;
|
||||
|
||||
dma->dma_address += offset;
|
||||
dma->dma_length = size - offset;
|
||||
|
||||
size = offset = s->offset;
|
||||
start = s;
|
||||
dma = sg_next(dma);
|
||||
count += 1;
|
||||
}
|
||||
size += s->length;
|
||||
}
|
||||
if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
|
||||
is_coherent) < 0)
|
||||
goto bad_mapping;
|
||||
|
||||
dma->dma_address += offset;
|
||||
dma->dma_length = size - offset;
|
||||
|
||||
return count+1;
|
||||
|
||||
bad_mapping:
|
||||
for_each_sg(sg, s, count, i)
|
||||
__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
|
||||
* @dev: valid struct device pointer
|
||||
* @sg: list of buffers
|
||||
* @nents: number of buffers to map
|
||||
* @dir: DMA transfer direction
|
||||
*
|
||||
* Map a set of i/o coherent buffers described by scatterlist in streaming
|
||||
* mode for DMA. The scatter gather list elements are merged together (if
|
||||
* possible) and tagged with the appropriate dma address and length. They are
|
||||
* obtained via sg_dma_{address,length}.
|
||||
*/
|
||||
int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir, unsigned long attrs)
|
||||
{
|
||||
return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
|
||||
* @dev: valid struct device pointer
|
||||
* @sg: list of buffers
|
||||
* @nents: number of buffers to map
|
||||
* @dir: DMA transfer direction
|
||||
*
|
||||
* Map a set of buffers described by scatterlist in streaming mode for DMA.
|
||||
* The scatter gather list elements are merged together (if possible) and
|
||||
* tagged with the appropriate dma address and length. They are obtained via
|
||||
* sg_dma_{address,length}.
|
||||
*/
|
||||
int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir, unsigned long attrs)
|
||||
{
|
||||
return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
|
||||
}
|
||||
|
||||
static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir, unsigned long attrs,
|
||||
bool is_coherent)
|
||||
{
|
||||
struct scatterlist *s;
|
||||
int i;
|
||||
|
||||
for_each_sg(sg, s, nents, i) {
|
||||
if (sg_dma_len(s))
|
||||
__iommu_remove_mapping(dev, sg_dma_address(s),
|
||||
sg_dma_len(s));
|
||||
if (!is_coherent &&
|
||||
!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
__dma_page_dev_to_cpu(sg_page(s), s->offset,
|
||||
s->length, dir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
|
||||
* @dev: valid struct device pointer
|
||||
* @sg: list of buffers
|
||||
* @nents: number of buffers to unmap (same as was passed to dma_map_sg)
|
||||
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
||||
*
|
||||
* Unmap a set of streaming mode DMA translations. Again, CPU access
|
||||
* rules concerning calls here are the same as for dma_unmap_single().
|
||||
*/
|
||||
void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir, unsigned long attrs)
|
||||
{
|
||||
__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
|
||||
* @dev: valid struct device pointer
|
||||
* @sg: list of buffers
|
||||
* @nents: number of buffers to unmap (same as was passed to dma_map_sg)
|
||||
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
||||
*
|
||||
* Unmap a set of streaming mode DMA translations. Again, CPU access
|
||||
* rules concerning calls here are the same as for dma_unmap_single().
|
||||
*/
|
||||
void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, unsigned long attrs)
|
||||
{
|
||||
__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_iommu_sync_sg_for_cpu
|
||||
* @dev: valid struct device pointer
|
||||
* @sg: list of buffers
|
||||
* @nents: number of buffers to map (returned from dma_map_sg)
|
||||
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
||||
*/
|
||||
void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir)
|
||||
{
|
||||
struct scatterlist *s;
|
||||
int i;
|
||||
|
||||
for_each_sg(sg, s, nents, i)
|
||||
__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_iommu_sync_sg_for_device
|
||||
* @dev: valid struct device pointer
|
||||
* @sg: list of buffers
|
||||
* @nents: number of buffers to map (returned from dma_map_sg)
|
||||
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
|
||||
*/
|
||||
void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir)
|
||||
{
|
||||
struct scatterlist *s;
|
||||
int i;
|
||||
|
||||
for_each_sg(sg, s, nents, i)
|
||||
__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* arm_coherent_iommu_map_page
|
||||
* @dev: valid struct device pointer
|
||||
* @page: page that buffer resides in
|
||||
* @offset: offset into page for start of buffer
|
||||
* @size: size of buffer to map
|
||||
* @dir: DMA transfer direction
|
||||
*
|
||||
* Coherent IOMMU aware version of arm_dma_map_page()
|
||||
*/
|
||||
static dma_addr_t arm_coherent_iommu_map_page(struct device *dev,
|
||||
struct page *page, unsigned long offset, size_t size,
|
||||
enum dma_data_direction dir, unsigned long attrs)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
dma_addr_t dma_addr;
|
||||
int ret, prot, len = PAGE_ALIGN(size + offset);
|
||||
|
||||
dma_addr = __alloc_iova(mapping, len);
|
||||
if (dma_addr == DMA_ERROR_CODE)
|
||||
return dma_addr;
|
||||
|
||||
prot = __dma_direction_to_prot(dir);
|
||||
|
||||
ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len,
|
||||
prot);
|
||||
if (ret < 0)
|
||||
goto fail;
|
||||
|
||||
return dma_addr + offset;
|
||||
fail:
|
||||
__free_iova(mapping, dma_addr, len);
|
||||
return DMA_ERROR_CODE;
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_iommu_map_page
|
||||
* @dev: valid struct device pointer
|
||||
* @page: page that buffer resides in
|
||||
* @offset: offset into page for start of buffer
|
||||
* @size: size of buffer to map
|
||||
* @dir: DMA transfer direction
|
||||
*
|
||||
* IOMMU aware version of arm_dma_map_page()
|
||||
*/
|
||||
static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
|
||||
unsigned long offset, size_t size, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
__dma_page_cpu_to_dev(page, offset, size, dir);
|
||||
|
||||
return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_coherent_iommu_unmap_page
|
||||
* @dev: valid struct device pointer
|
||||
* @handle: DMA address of buffer
|
||||
* @size: size of buffer (same as passed to dma_map_page)
|
||||
* @dir: DMA transfer direction (same as passed to dma_map_page)
|
||||
*
|
||||
* Coherent IOMMU aware version of arm_dma_unmap_page()
|
||||
*/
|
||||
static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
|
||||
size_t size, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
dma_addr_t iova = handle & PAGE_MASK;
|
||||
int offset = handle & ~PAGE_MASK;
|
||||
int len = PAGE_ALIGN(size + offset);
|
||||
|
||||
if (!iova)
|
||||
return;
|
||||
|
||||
iommu_unmap(mapping->domain, iova, len);
|
||||
__free_iova(mapping, iova, len);
|
||||
}
|
||||
|
||||
/**
|
||||
* arm_iommu_unmap_page
|
||||
* @dev: valid struct device pointer
|
||||
* @handle: DMA address of buffer
|
||||
* @size: size of buffer (same as passed to dma_map_page)
|
||||
* @dir: DMA transfer direction (same as passed to dma_map_page)
|
||||
*
|
||||
* IOMMU aware version of arm_dma_unmap_page()
|
||||
*/
|
||||
static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
|
||||
size_t size, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
dma_addr_t iova = handle & PAGE_MASK;
|
||||
struct page *page = phys_to_page(iommu_iova_to_phys(
|
||||
mapping->domain, iova));
|
||||
int offset = handle & ~PAGE_MASK;
|
||||
int len = PAGE_ALIGN(size + offset);
|
||||
|
||||
if (!iova)
|
||||
return;
|
||||
|
||||
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
__dma_page_dev_to_cpu(page, offset, size, dir);
|
||||
|
||||
iommu_unmap(mapping->domain, iova, len);
|
||||
__free_iova(mapping, iova, len);
|
||||
}
|
||||
|
||||
static void arm_iommu_sync_single_for_cpu(struct device *dev,
|
||||
dma_addr_t handle, size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
dma_addr_t iova = handle & PAGE_MASK;
|
||||
struct page *page = phys_to_page(iommu_iova_to_phys(
|
||||
mapping->domain, iova));
|
||||
unsigned int offset = handle & ~PAGE_MASK;
|
||||
|
||||
if (!iova)
|
||||
return;
|
||||
|
||||
__dma_page_dev_to_cpu(page, offset, size, dir);
|
||||
}
|
||||
|
||||
static void arm_iommu_sync_single_for_device(struct device *dev,
|
||||
dma_addr_t handle, size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
|
||||
dma_addr_t iova = handle & PAGE_MASK;
|
||||
struct page *page = phys_to_page(iommu_iova_to_phys(
|
||||
mapping->domain, iova));
|
||||
unsigned int offset = handle & ~PAGE_MASK;
|
||||
|
||||
if (!iova)
|
||||
return;
|
||||
|
||||
__dma_page_cpu_to_dev(page, offset, size, dir);
|
||||
}
|
||||
|
||||
const struct dma_map_ops iommu_ops = {
|
||||
.alloc = arm_iommu_alloc_attrs,
|
||||
.free = arm_iommu_free_attrs,
|
||||
.mmap = arm_iommu_mmap_attrs,
|
||||
.get_sgtable = arm_iommu_get_sgtable,
|
||||
|
||||
.map_page = arm_iommu_map_page,
|
||||
.unmap_page = arm_iommu_unmap_page,
|
||||
.sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
|
||||
.sync_single_for_device = arm_iommu_sync_single_for_device,
|
||||
|
||||
.map_sg = arm_iommu_map_sg,
|
||||
.unmap_sg = arm_iommu_unmap_sg,
|
||||
.sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
|
||||
.sync_sg_for_device = arm_iommu_sync_sg_for_device,
|
||||
|
||||
.set_dma_mask = arm_dma_set_mask,
|
||||
};
|
||||
|
||||
const struct dma_map_ops iommu_coherent_ops = {
|
||||
.alloc = arm_iommu_alloc_attrs,
|
||||
.free = arm_iommu_free_attrs,
|
||||
.mmap = arm_iommu_mmap_attrs,
|
||||
.get_sgtable = arm_iommu_get_sgtable,
|
||||
|
||||
.map_page = arm_coherent_iommu_map_page,
|
||||
.unmap_page = arm_coherent_iommu_unmap_page,
|
||||
|
||||
.map_sg = arm_coherent_iommu_map_sg,
|
||||
.unmap_sg = arm_coherent_iommu_unmap_sg,
|
||||
|
||||
.set_dma_mask = arm_dma_set_mask,
|
||||
};
|
||||
|
||||
/**
|
||||
* arm_iommu_create_mapping
|
||||
* @bus: pointer to the bus holding the client device (for IOMMU calls)
|
||||
* @base: start address of the valid IO address space
|
||||
* @size: size of the valid IO address space
|
||||
* @order: accuracy of the IO addresses allocations
|
||||
*
|
||||
* Creates a mapping structure which holds information about used/unused
|
||||
* IO address ranges, which is required to perform memory allocation and
|
||||
* mapping with IOMMU aware functions.
|
||||
*
|
||||
* The client device need to be attached to the mapping with
|
||||
* arm_iommu_attach_device function.
|
||||
*/
|
||||
struct dma_iommu_mapping *
|
||||
arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
|
||||
int order)
|
||||
{
|
||||
unsigned int count = size >> (PAGE_SHIFT + order);
|
||||
unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
|
||||
struct dma_iommu_mapping *mapping;
|
||||
int err = -ENOMEM;
|
||||
|
||||
if (!count)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
|
||||
if (!mapping)
|
||||
goto err;
|
||||
|
||||
mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
|
||||
if (!mapping->bitmap)
|
||||
goto err2;
|
||||
|
||||
mapping->base = base;
|
||||
mapping->bits = BITS_PER_BYTE * bitmap_size;
|
||||
mapping->order = order;
|
||||
spin_lock_init(&mapping->lock);
|
||||
|
||||
mapping->domain = iommu_domain_alloc(bus);
|
||||
if (!mapping->domain)
|
||||
goto err3;
|
||||
|
||||
kref_init(&mapping->kref);
|
||||
return mapping;
|
||||
err3:
|
||||
kfree(mapping->bitmap);
|
||||
err2:
|
||||
kfree(mapping);
|
||||
err:
|
||||
return ERR_PTR(err);
|
||||
}
|
||||
EXPORT_SYMBOL(arm_iommu_create_mapping);
|
||||
|
||||
static void release_iommu_mapping(struct kref *kref)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping =
|
||||
container_of(kref, struct dma_iommu_mapping, kref);
|
||||
|
||||
iommu_domain_free(mapping->domain);
|
||||
kfree(mapping->bitmap);
|
||||
kfree(mapping);
|
||||
}
|
||||
|
||||
void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
|
||||
{
|
||||
if (mapping)
|
||||
kref_put(&mapping->kref, release_iommu_mapping);
|
||||
}
|
||||
EXPORT_SYMBOL(arm_iommu_release_mapping);
|
||||
|
||||
/**
|
||||
* arm_iommu_attach_device
|
||||
* @dev: valid struct device pointer
|
||||
* @mapping: io address space mapping structure (returned from
|
||||
* arm_iommu_create_mapping)
|
||||
*
|
||||
* Attaches specified io address space mapping to the provided device,
|
||||
* this replaces the dma operations (dma_map_ops pointer) with the
|
||||
* IOMMU aware version. More than one client might be attached to
|
||||
* the same io address space mapping.
|
||||
*/
|
||||
int arm_iommu_attach_device(struct device *dev,
|
||||
struct dma_iommu_mapping *mapping)
|
||||
{
|
||||
int err;
|
||||
|
||||
err = iommu_attach_device(mapping->domain, dev);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
kref_get(&mapping->kref);
|
||||
dev->archdata.mapping = mapping;
|
||||
set_dma_ops(dev, &iommu_ops);
|
||||
|
||||
pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(arm_iommu_attach_device);
|
||||
|
||||
/**
|
||||
* arm_iommu_detach_device
|
||||
* @dev: valid struct device pointer
|
||||
*
|
||||
* Detaches the provided device from a previously attached map.
|
||||
* This voids the dma operations (dma_map_ops pointer)
|
||||
*/
|
||||
void arm_iommu_detach_device(struct device *dev)
|
||||
{
|
||||
struct dma_iommu_mapping *mapping;
|
||||
|
||||
mapping = to_dma_iommu_mapping(dev);
|
||||
if (!mapping) {
|
||||
dev_warn(dev, "Not attached\n");
|
||||
return;
|
||||
}
|
||||
|
||||
iommu_detach_device(mapping->domain, dev);
|
||||
kref_put(&mapping->kref, release_iommu_mapping);
|
||||
dev->archdata.mapping = NULL;
|
||||
set_dma_ops(dev, NULL);
|
||||
|
||||
pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
|
||||
}
|
||||
EXPORT_SYMBOL(arm_iommu_detach_device);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in a new issue