tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
4 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
5 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
6 */
7#include <linux/dma-direct.h>
8#include <linux/dma-noncoherent.h>
9#include <linux/dma-contiguous.h>
10#include <linux/highmem.h>
11
12#include <asm/cache.h>
13#include <asm/cpu-type.h>
14#include <asm/dma-coherence.h>
15#include <asm/io.h>
16
17/*
18 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively
19 * fill random cachelines with stale data at any time, requiring an extra
20 * flush post-DMA.
21 *
22 * Warning on the terminology - Linux calls an uncached area coherent; MIPS
23 * terminology calls memory areas with hardware maintained coherency coherent.
24 *
25 * Note that the R14000 and R16000 should also be checked for in this condition.
26 * However this function is only called on non-I/O-coherent systems and only the
27 * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp.
28 * SGI IP32 aka O2.
29 */
30static inline bool cpu_needs_post_dma_flush(struct device *dev)
31{
32 switch (boot_cpu_type()) {
33 case CPU_R10000:
34 case CPU_R12000:
35 case CPU_BMIPS5000:
36 return true;
37 default:
38 /*
39 * Presence of MAARs suggests that the CPU supports
40 * speculatively prefetching data, and therefore requires
41 * the post-DMA flush/invalidate.
42 */
43 return cpu_has_maar;
44 }
45}
46
47void *arch_dma_alloc(struct device *dev, size_t size,
48 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
49{
50 void *ret;
51
52 ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
53 if (ret && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
54 dma_cache_wback_inv((unsigned long) ret, size);
55 ret = (void *)UNCAC_ADDR(ret);
56 }
57
58 return ret;
59}
60
61void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
62 dma_addr_t dma_addr, unsigned long attrs)
63{
64 if (!(attrs & DMA_ATTR_NON_CONSISTENT))
65 cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr);
66 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
67}
68
69long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
70 dma_addr_t dma_addr)
71{
72 unsigned long addr = CAC_ADDR((unsigned long)cpu_addr);
73 return page_to_pfn(virt_to_page((void *)addr));
74}
75
76pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot,
77 unsigned long attrs)
78{
79 if (attrs & DMA_ATTR_WRITE_COMBINE)
80 return pgprot_writecombine(prot);
81 return pgprot_noncached(prot);
82}
83
84static inline void dma_sync_virt(void *addr, size_t size,
85 enum dma_data_direction dir)
86{
87 switch (dir) {
88 case DMA_TO_DEVICE:
89 dma_cache_wback((unsigned long)addr, size);
90 break;
91
92 case DMA_FROM_DEVICE:
93 dma_cache_inv((unsigned long)addr, size);
94 break;
95
96 case DMA_BIDIRECTIONAL:
97 dma_cache_wback_inv((unsigned long)addr, size);
98 break;
99
100 default:
101 BUG();
102 }
103}
104
105/*
106 * A single sg entry may refer to multiple physically contiguous pages. But
107 * we still need to process highmem pages individually. If highmem is not
108 * configured then the bulk of this loop gets optimized out.
109 */
110static inline void dma_sync_phys(phys_addr_t paddr, size_t size,
111 enum dma_data_direction dir)
112{
113 struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
114 unsigned long offset = paddr & ~PAGE_MASK;
115 size_t left = size;
116
117 do {
118 size_t len = left;
119
120 if (PageHighMem(page)) {
121 void *addr;
122
123 if (offset + len > PAGE_SIZE)
124 len = PAGE_SIZE - offset;
125
126 addr = kmap_atomic(page);
127 dma_sync_virt(addr + offset, len, dir);
128 kunmap_atomic(addr);
129 } else
130 dma_sync_virt(page_address(page) + offset, size, dir);
131 offset = 0;
132 page++;
133 left -= len;
134 } while (left);
135}
136
137void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
138 size_t size, enum dma_data_direction dir)
139{
140 dma_sync_phys(paddr, size, dir);
141}
142
143#ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU
144void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
145 size_t size, enum dma_data_direction dir)
146{
147 if (cpu_needs_post_dma_flush(dev))
148 dma_sync_phys(paddr, size, dir);
149}
150#endif
151
152void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
153 enum dma_data_direction direction)
154{
155 BUG_ON(direction == DMA_NONE);
156
157 dma_sync_virt(vaddr, size, direction);
158}
159
160#ifdef CONFIG_DMA_PERDEV_COHERENT
161void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
162 const struct iommu_ops *iommu, bool coherent)
163{
164 dev->dma_coherent = coherent;
165}
166#endif