tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
s390/preempt: move preempt_count to the lowcore
Convert s390 to use a field in the struct lowcore for the CPU
preemption count. It is a bit cheaper to access a lowcore field
compared to a thread_info variable and it removes the depencency
on a task related structure.
bloat-o-meter on the vmlinux image for the default configuration
(CONFIG_PREEMPT_NONE=y) reports a small reduction in text size:
add/remove: 0/0 grow/shrink: 18/578 up/down: 228/-5448 (-5220)
A larger improvement is achieved with the default configuration
but with CONFIG_PREEMPT=y and CONFIG_DEBUG_PREEMPT=n:
add/remove: 2/6 grow/shrink: 59/4477 up/down: 1618/-228762 (-227144)
Reviewed-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
+144
-5
+2
-1
arch/s390/include/asm/lowcore.h
+2
-1
arch/s390/include/asm/lowcore.h
···
126
126
__u64 percpu_offset; /* 0x0378 */
127
127
__u64 vdso_per_cpu_data; /* 0x0380 */
128
128
__u64 machine_flags; /* 0x0388 */
129
-
__u8 pad_0x0390[0x0398-0x0390]; /* 0x0390 */
129
+
__u32 preempt_count; /* 0x0390 */
130
+
__u8 pad_0x0394[0x0398-0x0394]; /* 0x0394 */
130
131
__u64 gmap; /* 0x0398 */
131
132
__u32 spinlock_lockval; /* 0x03a0 */
132
133
__u32 fpu_flags; /* 0x03a4 */
+137
arch/s390/include/asm/preempt.h
+137
arch/s390/include/asm/preempt.h
···
1
+
#ifndef __ASM_PREEMPT_H
2
+
#define __ASM_PREEMPT_H
3
+
4
+
#include <asm/current.h>
5
+
#include <linux/thread_info.h>
6
+
#include <asm/atomic_ops.h>
7
+
8
+
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
9
+
10
+
#define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED)
11
+
12
+
static inline int preempt_count(void)
13
+
{
14
+
return READ_ONCE(S390_lowcore.preempt_count) & ~PREEMPT_NEED_RESCHED;
15
+
}
16
+
17
+
static inline void preempt_count_set(int pc)
18
+
{
19
+
int old, new;
20
+
21
+
do {
22
+
old = READ_ONCE(S390_lowcore.preempt_count);
23
+
new = (old & PREEMPT_NEED_RESCHED) |
24
+
(pc & ~PREEMPT_NEED_RESCHED);
25
+
} while (__atomic_cmpxchg(&S390_lowcore.preempt_count,
26
+
old, new) != old);
27
+
}
28
+
29
+
#define init_task_preempt_count(p) do { } while (0)
30
+
31
+
#define init_idle_preempt_count(p, cpu) do { \
32
+
S390_lowcore.preempt_count = PREEMPT_ENABLED; \
33
+
} while (0)
34
+
35
+
static inline void set_preempt_need_resched(void)
36
+
{
37
+
__atomic_and(~PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count);
38
+
}
39
+
40
+
static inline void clear_preempt_need_resched(void)
41
+
{
42
+
__atomic_or(PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count);
43
+
}
44
+
45
+
static inline bool test_preempt_need_resched(void)
46
+
{
47
+
return !(READ_ONCE(S390_lowcore.preempt_count) & PREEMPT_NEED_RESCHED);
48
+
}
49
+
50
+
static inline void __preempt_count_add(int val)
51
+
{
52
+
if (__builtin_constant_p(val) && (val >= -128) && (val <= 127))
53
+
__atomic_add_const(val, &S390_lowcore.preempt_count);
54
+
else
55
+
__atomic_add(val, &S390_lowcore.preempt_count);
56
+
}
57
+
58
+
static inline void __preempt_count_sub(int val)
59
+
{
60
+
__preempt_count_add(-val);
61
+
}
62
+
63
+
static inline bool __preempt_count_dec_and_test(void)
64
+
{
65
+
return __atomic_add(-1, &S390_lowcore.preempt_count) == 1;
66
+
}
67
+
68
+
static inline bool should_resched(int preempt_offset)
69
+
{
70
+
return unlikely(READ_ONCE(S390_lowcore.preempt_count) ==
71
+
preempt_offset);
72
+
}
73
+
74
+
#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
75
+
76
+
#define PREEMPT_ENABLED (0)
77
+
78
+
static inline int preempt_count(void)
79
+
{
80
+
return READ_ONCE(S390_lowcore.preempt_count);
81
+
}
82
+
83
+
static inline void preempt_count_set(int pc)
84
+
{
85
+
S390_lowcore.preempt_count = pc;
86
+
}
87
+
88
+
#define init_task_preempt_count(p) do { } while (0)
89
+
90
+
#define init_idle_preempt_count(p, cpu) do { \
91
+
S390_lowcore.preempt_count = PREEMPT_ENABLED; \
92
+
} while (0)
93
+
94
+
static inline void set_preempt_need_resched(void)
95
+
{
96
+
}
97
+
98
+
static inline void clear_preempt_need_resched(void)
99
+
{
100
+
}
101
+
102
+
static inline bool test_preempt_need_resched(void)
103
+
{
104
+
return false;
105
+
}
106
+
107
+
static inline void __preempt_count_add(int val)
108
+
{
109
+
S390_lowcore.preempt_count += val;
110
+
}
111
+
112
+
static inline void __preempt_count_sub(int val)
113
+
{
114
+
S390_lowcore.preempt_count -= val;
115
+
}
116
+
117
+
static inline bool __preempt_count_dec_and_test(void)
118
+
{
119
+
return !--S390_lowcore.preempt_count && tif_need_resched();
120
+
}
121
+
122
+
static inline bool should_resched(int preempt_offset)
123
+
{
124
+
return unlikely(preempt_count() == preempt_offset &&
125
+
tif_need_resched());
126
+
}
127
+
128
+
#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
129
+
130
+
#ifdef CONFIG_PREEMPT
131
+
extern asmlinkage void preempt_schedule(void);
132
+
#define __preempt_schedule() preempt_schedule()
133
+
extern asmlinkage void preempt_schedule_notrace(void);
134
+
#define __preempt_schedule_notrace() preempt_schedule_notrace()
135
+
#endif /* CONFIG_PREEMPT */
136
+
137
+
#endif /* __ASM_PREEMPT_H */
-2
arch/s390/include/asm/thread_info.h
-2
arch/s390/include/asm/thread_info.h
···
34
34
unsigned long flags; /* low level flags */
35
35
unsigned long sys_call_table; /* System call table address */
36
36
unsigned int cpu; /* current CPU */
37
-
int preempt_count; /* 0 => preemptable, <0 => BUG */
38
37
unsigned int system_call;
39
38
__u64 user_timer;
40
39
__u64 system_timer;
···
48
49
.task = &tsk, \
49
50
.flags = 0, \
50
51
.cpu = 0, \
51
-
.preempt_count = INIT_PREEMPT_COUNT, \
52
52
}
53
53
54
54
#define init_thread_info (init_thread_union.thread_info)
+1
-1
arch/s390/kernel/asm-offsets.c
+1
-1
arch/s390/kernel/asm-offsets.c
···
43
43
OFFSET(__TI_flags, thread_info, flags);
44
44
OFFSET(__TI_sysc_table, thread_info, sys_call_table);
45
45
OFFSET(__TI_cpu, thread_info, cpu);
46
-
OFFSET(__TI_precount, thread_info, preempt_count);
47
46
OFFSET(__TI_user_timer, thread_info, user_timer);
48
47
OFFSET(__TI_system_timer, thread_info, system_timer);
49
48
OFFSET(__TI_last_break, thread_info, last_break);
···
174
175
OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset);
175
176
OFFSET(__LC_VDSO_PER_CPU, lowcore, vdso_per_cpu_data);
176
177
OFFSET(__LC_MACHINE_FLAGS, lowcore, machine_flags);
178
+
OFFSET(__LC_PREEMPT_COUNT, lowcore, preempt_count);
177
179
OFFSET(__LC_GMAP, lowcore, gmap);
178
180
OFFSET(__LC_PASTE, lowcore, paste);
179
181
/* software defined ABI-relevant lowcore locations 0xe00 - 0xe20 */
+1
arch/s390/kernel/early.c
+1
arch/s390/kernel/early.c
···
293
293
psw.addr = (unsigned long) s390_base_pgm_handler;
294
294
S390_lowcore.program_new_psw = psw;
295
295
s390_base_pgm_handler_fn = early_pgm_check_handler;
296
+
S390_lowcore.preempt_count = INIT_PREEMPT_COUNT;
296
297
}
297
298
298
299
static noinline __init void setup_facility_list(void)
+1
-1
arch/s390/kernel/entry.S
+1
-1
arch/s390/kernel/entry.S
···
626
626
jo .Lio_work_user # yes -> do resched & signal
627
627
#ifdef CONFIG_PREEMPT
628
628
# check for preemptive scheduling
629
-
icm %r0,15,__TI_precount(%r12)
629
+
icm %r0,15,__LC_PREEMPT_COUNT
630
630
jnz .Lio_restore # preemption is disabled
631
631
TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
632
632
jno .Lio_restore
+1
arch/s390/kernel/setup.c
+1
arch/s390/kernel/setup.c
···
333
333
lc->thread_info = (unsigned long) &init_thread_union;
334
334
lc->lpp = LPP_MAGIC;
335
335
lc->machine_flags = S390_lowcore.machine_flags;
336
+
lc->preempt_count = S390_lowcore.preempt_count;
336
337
lc->stfl_fac_list = S390_lowcore.stfl_fac_list;
337
338
memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
338
339
MAX_FACILITY_BIT/8);