This patch cleans up x86_64's DMA mapping dispatching code. Right now
we have three possible IOMMU types: AGP GART, swiotlb and nommu, and
in the future we will also have Xen's x86_64 swiotlb and other HW
IOMMUs for x86_64. In order to support all of them cleanly, this
patch:
- introduces a struct dma_mapping_ops with function pointers for each
of the DMA mapping operations of gart (AMD HW IOMMU), swiotlb
(software IOMMU) and nommu (no IOMMU).
- gets rid of:
if (swiotlb)
return swiotlb_xxx();
in various places in favor of:
if (unlikely(dma_mapping && dma_mapping->xxx))
return dma_mapping->xxx();
in dma-mapping.h.
- in order to keep the fast path fast, if no mapping_ops is specified
the default gart ops are used. For this case there's a tiny
additional cost of an unlikely branch in dma_xxx(), which then calls
gart_xxx().
- I modeled the patch after the existing code structure; that means
that in places nommu and swiotlb use the gart functions and in one
place gart calls into swiotlb. The former is on purpose; the latter
is a wart and will be fixed in the next iteration. Further cleanups
to seperate the three are certainly possible.
The patch is against 2.6.14-rc5, and has been tested with each of
gart, swiotlb and noiommu on a dual CPU AMD machine. Your comments
appreciated!
Signed-Off-By: Muli Ben-Yehuda <[email protected]>
---
arch/x86_64/kernel/Makefile | 2
arch/x86_64/kernel/pci-dma.c | 8
arch/x86_64/kernel/pci-gart.c | 69 +++-----
arch/x86_64/kernel/pci-nommu.c | 85 +++++++---
arch/x86_64/kernel/setup.c | 21 ++
arch/x86_64/mm/init.c | 13 +
b/include/asm-x86_64/gart-mapping.h | 33 ++++
include/asm-x86_64/dma-mapping.h | 295 +++++++++++++++++++++++-------------
include/asm-x86_64/swiotlb.h | 17 +-
9 files changed, 361 insertions(+), 182 deletions(-)
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 arch/x86_64/kernel/Makefile
--- a/arch/x86_64/kernel/Makefile Thu Oct 20 15:00:36 2005
+++ b/arch/x86_64/kernel/Makefile Thu Oct 20 17:58:33 2005
@@ -26,7 +26,7 @@
obj-$(CONFIG_CPU_FREQ) += cpufreq/
obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
obj-$(CONFIG_GART_IOMMU) += pci-gart.o aperture.o
-obj-$(CONFIG_DUMMY_IOMMU) += pci-nommu.o pci-dma.o
+obj-$(CONFIG_DUMMY_IOMMU) += pci-nommu.o pci-gart.o pci-dma.o
obj-$(CONFIG_SWIOTLB) += swiotlb.o
obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_X86_PM_TIMER) += pmtimer.o
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 arch/x86_64/kernel/pci-dma.c
--- a/arch/x86_64/kernel/pci-dma.c Thu Oct 20 15:00:36 2005
+++ b/arch/x86_64/kernel/pci-dma.c Thu Oct 20 17:58:33 2005
@@ -24,7 +24,7 @@
* Device ownership issues as mentioned above for pci_map_single are
* the same here.
*/
-int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
+int nommu_map_sg(struct device *hwdev, struct scatterlist *sg,
int nents, int direction)
{
int i;
@@ -39,13 +39,13 @@
return nents;
}
-EXPORT_SYMBOL(dma_map_sg);
+EXPORT_SYMBOL(nommu_map_sg);
/* Unmap a set of streaming mode DMA translations.
* Again, cpu read rules concerning calls here are the same as for
* pci_unmap_single() above.
*/
-void dma_unmap_sg(struct device *dev, struct scatterlist *sg,
+void nommu_unmap_sg(struct device *dev, struct scatterlist *sg,
int nents, int dir)
{
int i;
@@ -57,4 +57,4 @@
}
}
-EXPORT_SYMBOL(dma_unmap_sg);
+EXPORT_SYMBOL(nommu_unmap_sg);
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 arch/x86_64/kernel/pci-gart.c
--- a/arch/x86_64/kernel/pci-gart.c Thu Oct 20 15:00:36 2005
+++ b/arch/x86_64/kernel/pci-gart.c Thu Oct 20 17:58:33 2005
@@ -40,7 +40,6 @@
u32 *iommu_gatt_base; /* Remapping table */
int no_iommu;
-static int no_agp;
#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow = 1;
int force_iommu = 1;
@@ -203,8 +202,8 @@
* Allocate memory for a coherent mapping.
*/
void *
-dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
- unsigned gfp)
+gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
+ gfp_t gfp)
{
void *memory;
unsigned long dma_mask = 0;
@@ -267,7 +266,7 @@
error:
if (panic_on_overflow)
- panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n", size);
+ panic("gart_alloc_coherent: IOMMU overflow by %lu bytes\n", size);
free_pages((unsigned long)memory, get_order(size));
return NULL;
}
@@ -276,15 +275,10 @@
* Unmap coherent memory.
* The caller must ensure that the device has finished accessing the mapping.
*/
-void dma_free_coherent(struct device *dev, size_t size,
+void gart_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t bus)
{
- if (swiotlb) {
- swiotlb_free_coherent(dev, size, vaddr, bus);
- return;
- }
-
- dma_unmap_single(dev, bus, size, 0);
+ gart_unmap_single(dev, bus, size, 0);
free_pages((unsigned long)vaddr, get_order(size));
}
@@ -403,14 +397,12 @@
}
/* Map a single area into the IOMMU */
-dma_addr_t dma_map_single(struct device *dev, void *addr, size_t size, int dir)
+dma_addr_t gart_map_single(struct device *dev, void *addr, size_t size, int dir)
{
unsigned long phys_mem, bus;
BUG_ON(dir == DMA_NONE);
- if (swiotlb)
- return swiotlb_map_single(dev,addr,size,dir);
if (!dev)
dev = &fallback_dev;
@@ -440,7 +432,7 @@
addr = dma_map_area(dev, addr, s->length, dir, 0);
if (addr == bad_dma_address) {
if (i > 0)
- dma_unmap_sg(dev, sg, i, dir);
+ gart_unmap_sg(dev, sg, i, dir);
nents = 0;
sg[0].dma_length = 0;
break;
@@ -509,7 +501,7 @@
* DMA map all entries in a scatterlist.
* Merge chunks that have page aligned sizes into a continuous mapping.
*/
-int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
+int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
{
int i;
int out;
@@ -521,8 +513,6 @@
if (nents == 0)
return 0;
- if (swiotlb)
- return swiotlb_map_sg(dev,sg,nents,dir);
if (!dev)
dev = &fallback_dev;
@@ -565,7 +555,7 @@
error:
flush_gart(NULL);
- dma_unmap_sg(dev, sg, nents, dir);
+ gart_unmap_sg(dev, sg, nents, dir);
/* When it was forced try again unforced */
if (force_iommu)
return dma_map_sg_nonforce(dev, sg, nents, dir);
@@ -580,17 +570,12 @@
/*
* Free a DMA mapping.
*/
-void dma_unmap_single(struct device *dev, dma_addr_t dma_addr,
+void gart_unmap_single(struct device *dev, dma_addr_t dma_addr,
size_t size, int direction)
{
unsigned long iommu_page;
int npages;
int i;
-
- if (swiotlb) {
- swiotlb_unmap_single(dev,dma_addr,size,direction);
- return;
- }
if (dma_addr < iommu_bus_base + EMERGENCY_PAGES*PAGE_SIZE ||
dma_addr >= iommu_bus_base + iommu_size)
@@ -607,13 +592,10 @@
/*
* Wrapper for pci_unmap_single working with scatterlists.
*/
-void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
+void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
{
int i;
- if (swiotlb) {
- swiotlb_unmap_sg(dev,sg,nents,dir);
- return;
- }
+
for (i = 0; i < nents; i++) {
struct scatterlist *s = &sg[i];
if (!s->dma_length || !s->length)
@@ -622,7 +604,7 @@
}
}
-int dma_supported(struct device *dev, u64 mask)
+int gart_dma_supported(struct device *dev, u64 mask)
{
/* Copied from i386. Doesn't make much sense, because it will
only work for pci_alloc_coherent.
@@ -648,24 +630,22 @@
return 1;
}
-int dma_get_cache_alignment(void)
-{
- return boot_cpu_data.x86_clflush_size;
-}
-
-EXPORT_SYMBOL(dma_unmap_sg);
-EXPORT_SYMBOL(dma_map_sg);
-EXPORT_SYMBOL(dma_map_single);
-EXPORT_SYMBOL(dma_unmap_single);
-EXPORT_SYMBOL(dma_supported);
+EXPORT_SYMBOL(gart_unmap_sg);
+EXPORT_SYMBOL(gart_map_sg);
+EXPORT_SYMBOL(gart_map_single);
+EXPORT_SYMBOL(gart_unmap_single);
+EXPORT_SYMBOL(gart_dma_supported);
EXPORT_SYMBOL(no_iommu);
EXPORT_SYMBOL(force_iommu);
EXPORT_SYMBOL(bad_dma_address);
EXPORT_SYMBOL(iommu_bio_merge);
EXPORT_SYMBOL(iommu_sac_force);
-EXPORT_SYMBOL(dma_get_cache_alignment);
-EXPORT_SYMBOL(dma_alloc_coherent);
-EXPORT_SYMBOL(dma_free_coherent);
+EXPORT_SYMBOL(gart_alloc_coherent);
+EXPORT_SYMBOL(gart_free_coherent);
+
+#ifndef CONFIG_DUMMY_IOMMU
+
+static int no_agp;
static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size)
{
@@ -976,3 +956,4 @@
}
return 1;
}
+#endif /* !defined(CONFIG_DUMMY_IOMMU) */
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 arch/x86_64/kernel/pci-nommu.c
--- a/arch/x86_64/kernel/pci-nommu.c Thu Oct 20 15:00:36 2005
+++ b/arch/x86_64/kernel/pci-nommu.c Thu Oct 20 17:58:33 2005
@@ -7,24 +7,14 @@
#include <asm/proto.h>
#include <asm/processor.h>
-int iommu_merge = 0;
-EXPORT_SYMBOL(iommu_merge);
-
-dma_addr_t bad_dma_address;
-EXPORT_SYMBOL(bad_dma_address);
-
-int iommu_bio_merge = 0;
-EXPORT_SYMBOL(iommu_bio_merge);
-
-int iommu_sac_force = 0;
-EXPORT_SYMBOL(iommu_sac_force);
+extern int iommu_merge;
/*
* Dummy IO MMU functions
*/
-void *dma_alloc_coherent(struct device *hwdev, size_t size,
- dma_addr_t *dma_handle, unsigned gfp)
+void *nommu_alloc_coherent(struct device *hwdev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp)
{
void *ret;
u64 mask;
@@ -50,16 +40,16 @@
memset(ret, 0, size);
return ret;
}
-EXPORT_SYMBOL(dma_alloc_coherent);
+EXPORT_SYMBOL(nommu_alloc_coherent);
-void dma_free_coherent(struct device *hwdev, size_t size,
+void nommu_free_coherent(struct device *hwdev, size_t size,
void *vaddr, dma_addr_t dma_handle)
{
free_pages((unsigned long)vaddr, get_order(size));
}
-EXPORT_SYMBOL(dma_free_coherent);
+EXPORT_SYMBOL(nommu_free_coherent);
-int dma_supported(struct device *hwdev, u64 mask)
+int nommu_dma_supported(struct device *hwdev, u64 mask)
{
/*
* we fall back to GFP_DMA when the mask isn't all 1s,
@@ -73,22 +63,69 @@
return 1;
}
-EXPORT_SYMBOL(dma_supported);
+EXPORT_SYMBOL(nommu_dma_supported);
-int dma_get_cache_alignment(void)
+dma_addr_t
+nommu_map_single(struct device *hwdev, void *ptr, size_t size, int direction)
{
- return boot_cpu_data.x86_clflush_size;
+ dma_addr_t addr;
+
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+ addr = virt_to_bus(ptr);
+
+ if ((addr+size) & ~*hwdev->dma_mask)
+ out_of_line_bug();
+ return addr;
}
-EXPORT_SYMBOL(dma_get_cache_alignment);
+EXPORT_SYMBOL(nommu_map_single);
-static int __init check_ram(void)
+void
+nommu_unmap_single(struct device *hwdev, dma_addr_t dma_addr, size_t size,
+ int direction)
+{
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+ /* Nothing to do */
+}
+EXPORT_SYMBOL(nommu_unmap_single);
+
+static void check_ram(void)
{
if (end_pfn >= 0xffffffff>>PAGE_SHIFT) {
printk(
KERN_ERR "WARNING more than 4GB of memory but IOMMU not compiled in.\n"
KERN_ERR "WARNING 32bit PCI may malfunction.\n");
}
+}
+
+struct dma_mapping_ops nommu_mapping_ops = {
+ .mapping_error = NULL, /* default */
+ .alloc_coherent = nommu_alloc_coherent,
+ .free_coherent = nommu_free_coherent,
+ .map_single = nommu_map_single,
+ .unmap_single = nommu_unmap_single,
+ .sync_single_for_cpu = NULL,
+ .sync_single_for_device = NULL,
+ .sync_sg_for_cpu = NULL,
+ .sync_sg_for_device = NULL,
+ .map_sg = NULL,
+ .unmap_sg = NULL,
+ .dma_supported = nommu_dma_supported,
+};
+
+static int __init nommu_init(void)
+{
+ printk("%s: setting mapping_ops to nommu_mapping_ops(%p)\n",
+ __func__, &nommu_mapping_ops);
+
+ mapping_ops = &nommu_mapping_ops;
+
+ iommu_merge = 0;
+
+ check_ram();
+
return 0;
-}
-__initcall(check_ram);
+}
+__initcall(nommu_init);
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 arch/x86_64/kernel/setup.c
--- a/arch/x86_64/kernel/setup.c Thu Oct 20 15:00:36 2005
+++ b/arch/x86_64/kernel/setup.c Thu Oct 20 17:58:33 2005
@@ -42,6 +42,7 @@
#include <linux/edd.h>
#include <linux/mmzone.h>
#include <linux/kexec.h>
+#include <linux/dma-mapping.h>
#include <asm/mtrr.h>
#include <asm/uaccess.h>
@@ -60,6 +61,7 @@
#include <asm/setup.h>
#include <asm/mach_apic.h>
#include <asm/numa.h>
+#include <asm/swiotlb.h>
/*
* Machine setup..
@@ -86,8 +88,27 @@
#ifdef CONFIG_SWIOTLB
int swiotlb;
+struct dma_mapping_ops swiotlb_mapping_ops = {
+ .mapping_error = swiotlb_dma_mapping_error,
+ .alloc_coherent = NULL, /* we are called via gart_alloc_coherent */
+ .free_coherent = swiotlb_free_coherent,
+ .map_single = swiotlb_map_single,
+ .unmap_single = swiotlb_unmap_single,
+ .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
+ .sync_single_for_device = swiotlb_sync_single_for_device,
+ .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
+ .sync_sg_for_device = swiotlb_sync_sg_for_device,
+ .map_sg = swiotlb_map_sg,
+ .unmap_sg = swiotlb_unmap_sg,
+ /* historically we didn't use swiotlb_dma_supported, so keep it the same way */
+ .dma_supported = NULL
+};
+
EXPORT_SYMBOL(swiotlb);
#endif
+
+struct dma_mapping_ops* mapping_ops;
+EXPORT_SYMBOL(mapping_ops);
/*
* Setup options
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 arch/x86_64/mm/init.c
--- a/arch/x86_64/mm/init.c Thu Oct 20 15:00:36 2005
+++ b/arch/x86_64/mm/init.c Thu Oct 20 17:58:33 2005
@@ -22,6 +22,7 @@
#include <linux/pagemap.h>
#include <linux/bootmem.h>
#include <linux/proc_fs.h>
+#include <linux/dma-mapping.h>
#include <asm/processor.h>
#include <asm/system.h>
@@ -36,6 +37,7 @@
#include <asm/mmu_context.h>
#include <asm/proto.h>
#include <asm/smp.h>
+#include <asm/dma-mapping.h>
#ifndef Dprintk
#define Dprintk(x...)
@@ -388,12 +390,19 @@
{
long codesize, reservedpages, datasize, initsize;
+ printk("%s: setting mapping_ops to NULL\n", __func__);
+ mapping_ops = NULL;
+
#ifdef CONFIG_SWIOTLB
if (!iommu_aperture &&
(end_pfn >= 0xffffffff>>PAGE_SHIFT || force_iommu))
swiotlb = 1;
- if (swiotlb)
- swiotlb_init();
+ if (swiotlb) {
+ swiotlb_init();
+ mapping_ops = &swiotlb_mapping_ops;
+ printk("%s: setting mapping_ops to swiotlb_mapping_ops(%p)\n",
+ __func__, &swiotlb_mapping_ops);
+ }
#endif
/* How many end-of-memory variables you have, grandma! */
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 include/asm-x86_64/dma-mapping.h
--- a/include/asm-x86_64/dma-mapping.h Thu Oct 20 15:00:36 2005
+++ b/include/asm-x86_64/dma-mapping.h Thu Oct 20 17:58:33 2005
@@ -11,120 +11,205 @@
#include <asm/scatterlist.h>
#include <asm/io.h>
#include <asm/swiotlb.h>
+#include <asm/gart-mapping.h>
+
+struct dma_mapping_ops {
+ int (*mapping_error)(dma_addr_t dma_addr);
+ void* (*alloc_coherent)(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp);
+ void (*free_coherent)(struct device *dev, size_t size,
+ void *vaddr, dma_addr_t dma_handle);
+ dma_addr_t (*map_single)(struct device *hwdev, void *ptr,
+ size_t size, int direction);
+ void (*unmap_single)(struct device *dev, dma_addr_t addr,
+ size_t size, int direction);
+ void (*sync_single_for_cpu)(struct device *hwdev,
+ dma_addr_t dma_handle,
+ size_t size, int direction);
+ void (*sync_single_for_device)(struct device *hwdev,
+ dma_addr_t dma_handle,
+ size_t size, int direction);
+ void (*sync_sg_for_cpu)(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int direction);
+ void (*sync_sg_for_device)(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int direction);
+ int (*map_sg)(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction);
+ void (*unmap_sg)(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction);
+ int (*dma_supported)(struct device *hwdev, u64 mask);
+};
extern dma_addr_t bad_dma_address;
-#define dma_mapping_error(x) \
- (swiotlb ? swiotlb_dma_mapping_error(x) : ((x) == bad_dma_address))
-
-void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
- unsigned gfp);
-void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
- dma_addr_t dma_handle);
-
-#ifdef CONFIG_GART_IOMMU
-
-extern dma_addr_t dma_map_single(struct device *hwdev, void *ptr, size_t size,
- int direction);
-extern void dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size,
- int direction);
-
-#else
-
-/* No IOMMU */
-
-static inline dma_addr_t dma_map_single(struct device *hwdev, void *ptr,
- size_t size, int direction)
-{
- dma_addr_t addr;
-
- if (direction == DMA_NONE)
- out_of_line_bug();
- addr = virt_to_bus(ptr);
-
- if ((addr+size) & ~*hwdev->dma_mask)
- out_of_line_bug();
- return addr;
-}
-
-static inline void dma_unmap_single(struct device *hwdev, dma_addr_t dma_addr,
- size_t size, int direction)
-{
- if (direction == DMA_NONE)
- out_of_line_bug();
- /* Nothing to do */
-}
-
-#endif
+extern struct dma_mapping_ops* mapping_ops;
+
+static inline int dma_mapping_error(dma_addr_t dma_addr)
+{
+ if (mapping_ops && mapping_ops->mapping_error)
+ return mapping_ops->mapping_error(dma_addr);
+
+ return (dma_addr == bad_dma_address);
+}
+
+static inline void*
+dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
+ gfp_t gfp)
+{
+ if (unlikely(mapping_ops && mapping_ops->alloc_coherent))
+ return mapping_ops->alloc_coherent(dev, size, dma_handle, gfp);
+
+ return gart_alloc_coherent(dev, size, dma_handle, gfp);
+}
+
+static inline void
+dma_free_coherent(struct device *dev, size_t size, void *vaddr,
+ dma_addr_t dma_handle)
+{
+ if (unlikely(mapping_ops && mapping_ops->free_coherent)) {
+ mapping_ops->free_coherent(dev, size, vaddr, dma_handle);
+ return;
+ }
+
+ gart_free_coherent(dev, size, vaddr, dma_handle);
+}
+
+static inline dma_addr_t
+dma_map_single(struct device *hwdev, void *ptr, size_t size,
+ int direction)
+{
+ if (unlikely(mapping_ops && mapping_ops->map_single))
+ return mapping_ops->map_single(hwdev, ptr, size, direction);
+
+ return gart_map_single(hwdev, ptr, size, direction);
+}
+
+static inline void
+dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size,
+ int direction)
+{
+ if (unlikely(mapping_ops && mapping_ops->unmap_single)) {
+ mapping_ops->unmap_single(dev, addr, size, direction);
+ return;
+ }
+
+ gart_unmap_single(dev, addr, size, direction);
+}
#define dma_map_page(dev,page,offset,size,dir) \
dma_map_single((dev), page_address(page)+(offset), (size), (dir))
+
+#define dma_unmap_page dma_unmap_single
static inline void dma_sync_single_for_cpu(struct device *hwdev,
dma_addr_t dma_handle,
size_t size, int direction)
{
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- if (swiotlb)
- return swiotlb_sync_single_for_cpu(hwdev,dma_handle,size,direction);
-
- flush_write_buffers();
-}
-
-static inline void dma_sync_single_for_device(struct device *hwdev,
- dma_addr_t dma_handle,
- size_t size, int direction)
-{
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- if (swiotlb)
- return swiotlb_sync_single_for_device(hwdev,dma_handle,size,direction);
-
- flush_write_buffers();
-}
-
-#define dma_sync_single_range_for_cpu(dev, dma_handle, offset, size, dir) \
- dma_sync_single_for_cpu(dev, dma_handle, size, dir)
-#define dma_sync_single_range_for_device(dev, dma_handle, offset, size, dir) \
- dma_sync_single_for_device(dev, dma_handle, size, dir)
-
-static inline void dma_sync_sg_for_cpu(struct device *hwdev,
- struct scatterlist *sg,
- int nelems, int direction)
-{
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- if (swiotlb)
- return swiotlb_sync_sg_for_cpu(hwdev,sg,nelems,direction);
-
- flush_write_buffers();
-}
-
-static inline void dma_sync_sg_for_device(struct device *hwdev,
- struct scatterlist *sg,
- int nelems, int direction)
-{
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- if (swiotlb)
- return swiotlb_sync_sg_for_device(hwdev,sg,nelems,direction);
-
- flush_write_buffers();
-}
-
-extern int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
- int nents, int direction);
-extern void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
- int nents, int direction);
-
-#define dma_unmap_page dma_unmap_single
-
-extern int dma_supported(struct device *hwdev, u64 mask);
-extern int dma_get_cache_alignment(void);
+ void (*f)(struct device *hwdev, dma_addr_t dma_handle,
+ size_t size, int direction);
+
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+
+ if (unlikely(mapping_ops && mapping_ops->sync_single_for_cpu)) {
+ f = mapping_ops->sync_single_for_cpu;
+ f(hwdev, dma_handle, size, direction);
+ return;
+ }
+
+ flush_write_buffers();
+}
+
+static inline void
+dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle,
+ size_t size, int direction)
+{
+ void (*f)(struct device *hwdev, dma_addr_t dma_handle,
+ size_t size, int direction);
+
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+
+ if (unlikely(mapping_ops && mapping_ops->sync_single_for_device)) {
+ f = mapping_ops->sync_single_for_device;
+ f(hwdev, dma_handle, size, direction);
+ return;
+ }
+
+ flush_write_buffers();
+}
+
+static inline void
+dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int direction)
+{
+ void (*f)(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int direction);
+
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+
+ if (unlikely(mapping_ops && mapping_ops->sync_sg_for_cpu)) {
+ f = mapping_ops->sync_sg_for_cpu;
+ f(hwdev, sg, nelems, direction);
+ return;
+ }
+
+ flush_write_buffers();
+}
+
+static inline void
+dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int direction)
+{
+ void (*f)(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int direction);
+
+ if (direction == DMA_NONE)
+ out_of_line_bug();
+
+ if (unlikely(mapping_ops && mapping_ops->sync_sg_for_device)) {
+ f = mapping_ops->sync_sg_for_device;
+ f(hwdev, sg, nelems, direction);
+ return;
+ }
+
+ flush_write_buffers();
+}
+
+static inline int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction)
+{
+ if (unlikely(mapping_ops && mapping_ops->map_sg))
+ return mapping_ops->map_sg(hwdev, sg, nents, direction);
+
+ return gart_map_sg(hwdev, sg, nents, direction);
+}
+
+static inline void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction)
+{
+ if (unlikely(mapping_ops && mapping_ops->unmap_sg)) {
+ mapping_ops->unmap_sg(hwdev, sg, nents, direction);
+ return;
+ }
+
+ gart_unmap_sg(hwdev, sg, nents, direction);
+}
+
+static inline int dma_supported(struct device *hwdev, u64 mask)
+{
+ if (mapping_ops && mapping_ops->dma_supported)
+ return mapping_ops->dma_supported(hwdev, mask);
+
+ return gart_dma_supported(hwdev, mask);
+}
+
+/* same for gart, swiotlb, and nommu */
+static inline int dma_get_cache_alignment(void)
+{
+ return boot_cpu_data.x86_clflush_size;
+}
+
#define dma_is_consistent(h) 1
static inline int dma_set_mask(struct device *dev, u64 mask)
@@ -140,4 +225,4 @@
flush_write_buffers();
}
-#endif
+#endif /* _X8664_DMA_MAPPING_H */
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 include/asm-x86_64/swiotlb.h
--- a/include/asm-x86_64/swiotlb.h Thu Oct 20 15:00:36 2005
+++ b/include/asm-x86_64/swiotlb.h Thu Oct 20 17:58:33 2005
@@ -2,6 +2,8 @@
#define _ASM_SWTIOLB_H 1
#include <linux/config.h>
+
+#include <asm/dma-mapping.h>
/* SWIOTLB interface */
@@ -15,6 +17,14 @@
extern void swiotlb_sync_single_for_device(struct device *hwdev,
dma_addr_t dev_addr,
size_t size, int dir);
+extern void swiotlb_sync_single_range_for_cpu(struct device *hwdev,
+ dma_addr_t dev_addr,
+ unsigned long offset,
+ size_t size, int dir);
+extern void swiotlb_sync_single_range_for_device(struct device *hwdev,
+ dma_addr_t dev_addr,
+ unsigned long offset,
+ size_t size, int dir);
extern void swiotlb_sync_sg_for_cpu(struct device *hwdev,
struct scatterlist *sg, int nelems,
int dir);
@@ -27,9 +37,12 @@
int nents, int direction);
extern int swiotlb_dma_mapping_error(dma_addr_t dma_addr);
extern void *swiotlb_alloc_coherent (struct device *hwdev, size_t size,
- dma_addr_t *dma_handle, int flags);
+ dma_addr_t *dma_handle, int gfp);
extern void swiotlb_free_coherent (struct device *hwdev, size_t size,
void *vaddr, dma_addr_t dma_handle);
+extern int swiotlb_dma_supported(struct device *hwdev, u64 mask);
+
+extern struct dma_mapping_ops swiotlb_mapping_ops;
#ifdef CONFIG_SWIOTLB
extern int swiotlb;
@@ -37,4 +50,4 @@
#define swiotlb 0
#endif
-#endif
+#endif /* _ASM_SWTIOLB_H */
diff -r 01355122e23bd7163ea41cc430bfb8a90a396add -r cbfca96927d50471f1dad6d5a9eeae4b9f1ca9c2 include/asm-x86_64/gart-mapping.h
--- /dev/null Thu Oct 20 15:00:36 2005
+++ b/include/asm-x86_64/gart-mapping.h Thu Oct 20 17:58:33 2005
@@ -0,0 +1,33 @@
+#ifndef _ASM_GART_MAPPING_H
+#define _ASM_GART_MAPPING_H 1
+
+#include <linux/config.h>
+
+/* GART DMA mapping implemenation */
+extern void*
+gart_alloc_coherent(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp);
+
+extern void
+gart_free_coherent(struct device *dev, size_t size, void *vaddr,
+ dma_addr_t dma_handle);
+
+extern dma_addr_t
+gart_map_single(struct device *hwdev, void *ptr, size_t size,
+ int direction);
+
+extern void
+gart_unmap_single(struct device *dev, dma_addr_t addr,size_t size,
+ int direction);
+
+extern int
+gart_map_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction);
+
+extern void
+gart_unmap_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction);
+
+extern int gart_dma_supported(struct device *hwdev, u64 mask);
+
+#endif /* _ASM_SWTIOLB_H */
--
Muli Ben-Yehuda
http://www.mulix.org | http://mulix.livejournal.com/
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[Index of Archives]
[Kernel Newbies]
[Netfilter]
[Bugtraq]
[Photo]
[Stuff]
[Gimp]
[Yosemite News]
[MIPS Linux]
[ARM Linux]
[Linux Security]
[Linux RAID]
[Video 4 Linux]
[Linux for the blind]
[Linux Resources]
