tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
csky: Add flush_icache_mm to defer flush icache all
Some CPUs don't support icache.va instruction to maintain the whole
smp cores' icache. Using icache.all + IPI casue a lot on performace
and using defer mechanism could reduce the number of calling icache
_flush_all functions.
Signed-off-by: Guo Ren <guoren@linux.alibaba.com>
+77
-11
+2
arch/csky/abiv1/inc/abi/cacheflush.h
+2
arch/csky/abiv1/inc/abi/cacheflush.h
···
48
48
49
49
#define flush_icache_page(vma, page) do {} while (0);
50
50
#define flush_icache_range(start, end) cache_wbinv_range(start, end)
51
+
#define flush_icache_mm_range(mm, start, end) cache_wbinv_range(start, end)
52
+
#define flush_icache_deferred(mm) do {} while (0);
51
53
52
54
#define copy_from_user_page(vma, page, vaddr, dst, src, len) \
53
55
do { \
+55
arch/csky/abiv2/cacheflush.c
+55
arch/csky/abiv2/cacheflush.c
···
28
28
29
29
kunmap_atomic((void *) addr);
30
30
}
31
+
32
+
void flush_icache_deferred(struct mm_struct *mm)
33
+
{
34
+
unsigned int cpu = smp_processor_id();
35
+
cpumask_t *mask = &mm->context.icache_stale_mask;
36
+
37
+
if (cpumask_test_cpu(cpu, mask)) {
38
+
cpumask_clear_cpu(cpu, mask);
39
+
/*
40
+
* Ensure the remote hart's writes are visible to this hart.
41
+
* This pairs with a barrier in flush_icache_mm.
42
+
*/
43
+
smp_mb();
44
+
local_icache_inv_all(NULL);
45
+
}
46
+
}
47
+
48
+
void flush_icache_mm_range(struct mm_struct *mm,
49
+
unsigned long start, unsigned long end)
50
+
{
51
+
unsigned int cpu;
52
+
cpumask_t others, *mask;
53
+
54
+
preempt_disable();
55
+
56
+
#ifdef CONFIG_CPU_HAS_ICACHE_INS
57
+
if (mm == current->mm) {
58
+
icache_inv_range(start, end);
59
+
preempt_enable();
60
+
return;
61
+
}
62
+
#endif
63
+
64
+
/* Mark every hart's icache as needing a flush for this MM. */
65
+
mask = &mm->context.icache_stale_mask;
66
+
cpumask_setall(mask);
67
+
68
+
/* Flush this hart's I$ now, and mark it as flushed. */
69
+
cpu = smp_processor_id();
70
+
cpumask_clear_cpu(cpu, mask);
71
+
local_icache_inv_all(NULL);
72
+
73
+
/*
74
+
* Flush the I$ of other harts concurrently executing, and mark them as
75
+
* flushed.
76
+
*/
77
+
cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
78
+
79
+
if (mm != current->active_mm || !cpumask_empty(&others)) {
80
+
on_each_cpu_mask(&others, local_icache_inv_all, NULL, 1);
81
+
cpumask_clear(mask);
82
+
}
83
+
84
+
preempt_enable();
85
+
}
+11
-3
arch/csky/abiv2/inc/abi/cacheflush.h
+11
-3
arch/csky/abiv2/inc/abi/cacheflush.h
···
31
31
32
32
#define flush_icache_range(start, end) cache_wbinv_range(start, end)
33
33
34
+
void flush_icache_mm_range(struct mm_struct *mm,
35
+
unsigned long start, unsigned long end);
36
+
void flush_icache_deferred(struct mm_struct *mm);
37
+
34
38
#define flush_cache_vmap(start, end) do { } while (0)
35
39
#define flush_cache_vunmap(start, end) do { } while (0)
36
40
37
41
#define copy_to_user_page(vma, page, vaddr, dst, src, len) \
38
42
do { \
39
43
memcpy(dst, src, len); \
40
-
if (vma->vm_flags & VM_EXEC) \
41
-
cache_wbinv_range((unsigned long)dst, \
42
-
(unsigned long)dst + len); \
44
+
if (vma->vm_flags & VM_EXEC) { \
45
+
dcache_wb_range((unsigned long)dst, \
46
+
(unsigned long)dst + len); \
47
+
flush_icache_mm_range(current->mm, \
48
+
(unsigned long)dst, \
49
+
(unsigned long)dst + len); \
50
+
} \
43
51
} while (0)
44
52
#define copy_from_user_page(vma, page, vaddr, dst, src, len) \
45
53
memcpy(dst, src, len)
+1
arch/csky/include/asm/cacheflush.h
+1
arch/csky/include/asm/cacheflush.h
+1
arch/csky/include/asm/mmu.h
+1
arch/csky/include/asm/mmu.h
+2
arch/csky/include/asm/mmu_context.h
+2
arch/csky/include/asm/mmu_context.h
+5
-8
arch/csky/mm/syscache.c
+5
-8
arch/csky/mm/syscache.c
···
3
3
4
4
#include <linux/syscalls.h>
5
5
#include <asm/page.h>
6
-
#include <asm/cache.h>
6
+
#include <asm/cacheflush.h>
7
7
#include <asm/cachectl.h>
8
8
9
9
SYSCALL_DEFINE3(cacheflush,
···
13
13
{
14
14
switch (cache) {
15
15
case ICACHE:
16
-
icache_inv_range((unsigned long)addr,
17
-
(unsigned long)addr + bytes);
18
-
break;
16
+
case BCACHE:
17
+
flush_icache_mm_range(current->mm,
18
+
(unsigned long)addr,
19
+
(unsigned long)addr + bytes);
19
20
case DCACHE:
20
21
dcache_wb_range((unsigned long)addr,
21
22
(unsigned long)addr + bytes);
22
-
break;
23
-
case BCACHE:
24
-
cache_wbinv_range((unsigned long)addr,
25
-
(unsigned long)addr + bytes);
26
23
break;
27
24
default:
28
25
return -EINVAL;