lib/min_heap: introduce non-inline versions of min heap API functions

+1

drivers/md/bcache/Kconfig

··· 5 5 select BLOCK_HOLDER_DEPRECATED if SYSFS 6 6 select CRC64 7 7 select CLOSURES 8 + select MIN_HEAP 8 9 help 9 10 Allows a block device to be used as cache for other devices; uses 10 11 a btree for indexing and the layout is optimized for SSDs.

+1

drivers/md/dm-vdo/Kconfig

··· 7 7 select DM_BUFIO 8 8 select LZ4_COMPRESS 9 9 select LZ4_DECOMPRESS 10 + select MIN_HEAP 10 11 help 11 12 This device mapper target presents a block device with 12 13 deduplication, compression and thin-provisioning.

+1

fs/bcachefs/Kconfig

··· 24 24 select XXHASH 25 25 select SRCU 26 26 select SYMBOLIC_ERRNAME 27 + select MIN_HEAP 27 28 help 28 29 The bcachefs filesystem - a modern, copy on write filesystem, with 29 30 support for multiple devices, compression, checksumming, etc.

+86 -43

include/linux/min_heap.h

··· 40 40 41 41 /* Initialize a min-heap. */ 42 42 static __always_inline 43 - void __min_heap_init(min_heap_char *heap, void *data, int size) 43 + void __min_heap_init_inline(min_heap_char *heap, void *data, int size) 44 44 { 45 45 heap->nr = 0; 46 46 heap->size = size; ··· 50 50 heap->data = heap->preallocated; 51 51 } 52 52 53 - #define min_heap_init(_heap, _data, _size) \ 54 - __min_heap_init((min_heap_char *)_heap, _data, _size) 53 + #define min_heap_init_inline(_heap, _data, _size) \ 54 + __min_heap_init_inline((min_heap_char *)_heap, _data, _size) 55 55 56 56 /* Get the minimum element from the heap. */ 57 57 static __always_inline 58 - void *__min_heap_peek(struct min_heap_char *heap) 58 + void *__min_heap_peek_inline(struct min_heap_char *heap) 59 59 { 60 60 return heap->nr ? heap->data : NULL; 61 61 } 62 62 63 - #define min_heap_peek(_heap) \ 64 - (__minheap_cast(_heap) __min_heap_peek((min_heap_char *)_heap)) 63 + #define min_heap_peek_inline(_heap) \ 64 + (__minheap_cast(_heap) __min_heap_peek_inline((min_heap_char *)_heap)) 65 65 66 66 /* Check if the heap is full. */ 67 67 static __always_inline 68 - bool __min_heap_full(min_heap_char *heap) 68 + bool __min_heap_full_inline(min_heap_char *heap) 69 69 { 70 70 return heap->nr == heap->size; 71 71 } 72 72 73 - #define min_heap_full(_heap) \ 74 - __min_heap_full((min_heap_char *)_heap) 73 + #define min_heap_full_inline(_heap) \ 74 + __min_heap_full_inline((min_heap_char *)_heap) 75 75 76 76 /* Sift the element at pos down the heap. */ 77 77 static __always_inline 78 - void __min_heap_sift_down(min_heap_char *heap, int pos, size_t elem_size, 79 - const struct min_heap_callbacks *func, void *args) 78 + void __min_heap_sift_down_inline(min_heap_char *heap, int pos, size_t elem_size, 79 + const struct min_heap_callbacks *func, void *args) 80 80 { 81 81 void *left, *right; 82 82 void *data = heap->data; ··· 108 108 } 109 109 } 110 110 111 - #define min_heap_sift_down(_heap, _pos, _func, _args) \ 112 - __min_heap_sift_down((min_heap_char *)_heap, _pos, __minheap_obj_size(_heap), _func, _args) 111 + #define min_heap_sift_down_inline(_heap, _pos, _func, _args) \ 112 + __min_heap_sift_down_inline((min_heap_char *)_heap, _pos, __minheap_obj_size(_heap), \ 113 + _func, _args) 113 114 114 115 /* Sift up ith element from the heap, O(log2(nr)). */ 115 116 static __always_inline 116 - void __min_heap_sift_up(min_heap_char *heap, size_t elem_size, size_t idx, 117 - const struct min_heap_callbacks *func, void *args) 117 + void __min_heap_sift_up_inline(min_heap_char *heap, size_t elem_size, size_t idx, 118 + const struct min_heap_callbacks *func, void *args) 118 119 { 119 120 void *data = heap->data; 120 121 size_t parent; ··· 129 128 } 130 129 } 131 130 132 - #define min_heap_sift_up(_heap, _idx, _func, _args) \ 133 - __min_heap_sift_up((min_heap_char *)_heap, __minheap_obj_size(_heap), _idx, _func, _args) 131 + #define min_heap_sift_up_inline(_heap, _idx, _func, _args) \ 132 + __min_heap_sift_up_inline((min_heap_char *)_heap, __minheap_obj_size(_heap), _idx, \ 133 + _func, _args) 134 134 135 135 /* Floyd's approach to heapification that is O(nr). */ 136 136 static __always_inline 137 - void __min_heapify_all(min_heap_char *heap, size_t elem_size, 138 - const struct min_heap_callbacks *func, void *args) 137 + void __min_heapify_all_inline(min_heap_char *heap, size_t elem_size, 138 + const struct min_heap_callbacks *func, void *args) 139 139 { 140 140 int i; 141 141 142 142 for (i = heap->nr / 2 - 1; i >= 0; i--) 143 - __min_heap_sift_down(heap, i, elem_size, func, args); 143 + __min_heap_sift_down_inline(heap, i, elem_size, func, args); 144 144 } 145 145 146 - #define min_heapify_all(_heap, _func, _args) \ 147 - __min_heapify_all((min_heap_char *)_heap, __minheap_obj_size(_heap), _func, _args) 146 + #define min_heapify_all_inline(_heap, _func, _args) \ 147 + __min_heapify_all_inline((min_heap_char *)_heap, __minheap_obj_size(_heap), _func, _args) 148 148 149 149 /* Remove minimum element from the heap, O(log2(nr)). */ 150 150 static __always_inline 151 - bool __min_heap_pop(min_heap_char *heap, size_t elem_size, 152 - const struct min_heap_callbacks *func, void *args) 151 + bool __min_heap_pop_inline(min_heap_char *heap, size_t elem_size, 152 + const struct min_heap_callbacks *func, void *args) 153 153 { 154 154 void *data = heap->data; 155 155 ··· 160 158 /* Place last element at the root (position 0) and then sift down. */ 161 159 heap->nr--; 162 160 memcpy(data, data + (heap->nr * elem_size), elem_size); 163 - __min_heap_sift_down(heap, 0, elem_size, func, args); 161 + __min_heap_sift_down_inline(heap, 0, elem_size, func, args); 164 162 165 163 return true; 166 164 } 167 165 168 - #define min_heap_pop(_heap, _func, _args) \ 169 - __min_heap_pop((min_heap_char *)_heap, __minheap_obj_size(_heap), _func, _args) 166 + #define min_heap_pop_inline(_heap, _func, _args) \ 167 + __min_heap_pop_inline((min_heap_char *)_heap, __minheap_obj_size(_heap), _func, _args) 170 168 171 169 /* 172 170 * Remove the minimum element and then push the given element. The ··· 174 172 * efficient than a pop followed by a push that does 2. 175 173 */ 176 174 static __always_inline 177 - void __min_heap_pop_push(min_heap_char *heap, 178 - const void *element, size_t elem_size, 179 - const struct min_heap_callbacks *func, 180 - void *args) 175 + void __min_heap_pop_push_inline(min_heap_char *heap, const void *element, size_t elem_size, 176 + const struct min_heap_callbacks *func, void *args) 181 177 { 182 178 memcpy(heap->data, element, elem_size); 183 - __min_heap_sift_down(heap, 0, elem_size, func, args); 179 + __min_heap_sift_down_inline(heap, 0, elem_size, func, args); 184 180 } 185 181 186 - #define min_heap_pop_push(_heap, _element, _func, _args) \ 187 - __min_heap_pop_push((min_heap_char *)_heap, _element, __minheap_obj_size(_heap), _func, _args) 182 + #define min_heap_pop_push_inline(_heap, _element, _func, _args) \ 183 + __min_heap_pop_push_inline((min_heap_char *)_heap, _element, __minheap_obj_size(_heap), \ 184 + _func, _args) 188 185 189 186 /* Push an element on to the heap, O(log2(nr)). */ 190 187 static __always_inline 191 - bool __min_heap_push(min_heap_char *heap, const void *element, size_t elem_size, 192 - const struct min_heap_callbacks *func, void *args) 188 + bool __min_heap_push_inline(min_heap_char *heap, const void *element, size_t elem_size, 189 + const struct min_heap_callbacks *func, void *args) 193 190 { 194 191 void *data = heap->data; 195 192 int pos; ··· 202 201 heap->nr++; 203 202 204 203 /* Sift child at pos up. */ 205 - __min_heap_sift_up(heap, elem_size, pos, func, args); 204 + __min_heap_sift_up_inline(heap, elem_size, pos, func, args); 206 205 207 206 return true; 208 207 } 209 208 210 - #define min_heap_push(_heap, _element, _func, _args) \ 211 - __min_heap_push((min_heap_char *)_heap, _element, __minheap_obj_size(_heap), _func, _args) 209 + #define min_heap_push_inline(_heap, _element, _func, _args) \ 210 + __min_heap_push_inline((min_heap_char *)_heap, _element, __minheap_obj_size(_heap), \ 211 + _func, _args) 212 212 213 213 /* Remove ith element from the heap, O(log2(nr)). */ 214 214 static __always_inline 215 - bool __min_heap_del(min_heap_char *heap, size_t elem_size, size_t idx, 216 - const struct min_heap_callbacks *func, void *args) 215 + bool __min_heap_del_inline(min_heap_char *heap, size_t elem_size, size_t idx, 216 + const struct min_heap_callbacks *func, void *args) 217 217 { 218 218 void *data = heap->data; 219 219 ··· 226 224 if (idx == heap->nr) 227 225 return true; 228 226 func->swp(data + (idx * elem_size), data + (heap->nr * elem_size), args); 229 - __min_heap_sift_up(heap, elem_size, idx, func, args); 230 - __min_heap_sift_down(heap, idx, elem_size, func, args); 227 + __min_heap_sift_up_inline(heap, elem_size, idx, func, args); 228 + __min_heap_sift_down_inline(heap, idx, elem_size, func, args); 231 229 232 230 return true; 233 231 } 234 232 233 + #define min_heap_del_inline(_heap, _idx, _func, _args) \ 234 + __min_heap_del_inline((min_heap_char *)_heap, __minheap_obj_size(_heap), _idx, \ 235 + _func, _args) 236 + 237 + void __min_heap_init(min_heap_char *heap, void *data, int size); 238 + void *__min_heap_peek(struct min_heap_char *heap); 239 + bool __min_heap_full(min_heap_char *heap); 240 + void __min_heap_sift_down(min_heap_char *heap, int pos, size_t elem_size, 241 + const struct min_heap_callbacks *func, void *args); 242 + void __min_heap_sift_up(min_heap_char *heap, size_t elem_size, size_t idx, 243 + const struct min_heap_callbacks *func, void *args); 244 + void __min_heapify_all(min_heap_char *heap, size_t elem_size, 245 + const struct min_heap_callbacks *func, void *args); 246 + bool __min_heap_pop(min_heap_char *heap, size_t elem_size, 247 + const struct min_heap_callbacks *func, void *args); 248 + void __min_heap_pop_push(min_heap_char *heap, const void *element, size_t elem_size, 249 + const struct min_heap_callbacks *func, void *args); 250 + bool __min_heap_push(min_heap_char *heap, const void *element, size_t elem_size, 251 + const struct min_heap_callbacks *func, void *args); 252 + bool __min_heap_del(min_heap_char *heap, size_t elem_size, size_t idx, 253 + const struct min_heap_callbacks *func, void *args); 254 + 255 + #define min_heap_init(_heap, _data, _size) \ 256 + __min_heap_init((min_heap_char *)_heap, _data, _size) 257 + #define min_heap_peek(_heap) \ 258 + (__minheap_cast(_heap) __min_heap_peek((min_heap_char *)_heap)) 259 + #define min_heap_full(_heap) \ 260 + __min_heap_full((min_heap_char *)_heap) 261 + #define min_heap_sift_down(_heap, _pos, _func, _args) \ 262 + __min_heap_sift_down((min_heap_char *)_heap, _pos, __minheap_obj_size(_heap), _func, _args) 263 + #define min_heap_sift_up(_heap, _idx, _func, _args) \ 264 + __min_heap_sift_up((min_heap_char *)_heap, __minheap_obj_size(_heap), _idx, _func, _args) 265 + #define min_heapify_all(_heap, _func, _args) \ 266 + __min_heapify_all((min_heap_char *)_heap, __minheap_obj_size(_heap), _func, _args) 267 + #define min_heap_pop(_heap, _func, _args) \ 268 + __min_heap_pop((min_heap_char *)_heap, __minheap_obj_size(_heap), _func, _args) 269 + #define min_heap_pop_push(_heap, _element, _func, _args) \ 270 + __min_heap_pop_push((min_heap_char *)_heap, _element, __minheap_obj_size(_heap), \ 271 + _func, _args) 272 + #define min_heap_push(_heap, _element, _func, _args) \ 273 + __min_heap_push((min_heap_char *)_heap, _element, __minheap_obj_size(_heap), _func, _args) 235 274 #define min_heap_del(_heap, _idx, _func, _args) \ 236 275 __min_heap_del((min_heap_char *)_heap, __minheap_obj_size(_heap), _idx, _func, _args) 237 276

+3 -3

kernel/events/core.c

··· 3870 3870 perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu); 3871 3871 } 3872 3872 3873 - min_heapify_all(&event_heap, &perf_min_heap, NULL); 3873 + min_heapify_all_inline(&event_heap, &perf_min_heap, NULL); 3874 3874 3875 3875 while (event_heap.nr) { 3876 3876 ret = func(*evt, data); ··· 3879 3879 3880 3880 *evt = perf_event_groups_next(*evt, pmu); 3881 3881 if (*evt) 3882 - min_heap_sift_down(&event_heap, 0, &perf_min_heap, NULL); 3882 + min_heap_sift_down_inline(&event_heap, 0, &perf_min_heap, NULL); 3883 3883 else 3884 - min_heap_pop(&event_heap, &perf_min_heap, NULL); 3884 + min_heap_pop_inline(&event_heap, &perf_min_heap, NULL); 3885 3885 } 3886 3886 3887 3887 return 0;

+3

lib/Kconfig

··· 780 780 781 781 config UNION_FIND 782 782 bool 783 + 784 + config MIN_HEAP 785 + bool

+1

lib/Kconfig.debug

··· 2279 2279 config TEST_MIN_HEAP 2280 2280 tristate "Min heap test" 2281 2281 depends on DEBUG_KERNEL || m 2282 + select MIN_HEAP 2282 2283 help 2283 2284 Enable this to turn on min heap function tests. This test is 2284 2285 executed only once during system boot (so affects only boot time),

+1

lib/Makefile

··· 40 40 lib-$(CONFIG_UNION_FIND) += union_find.o 41 41 lib-$(CONFIG_PRINTK) += dump_stack.o 42 42 lib-$(CONFIG_SMP) += cpumask.o 43 + lib-$(CONFIG_MIN_HEAP) += min_heap.o 43 44 44 45 lib-y += kobject.o klist.o 45 46 obj-y += lockref.o

+70

lib/min_heap.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #include <linux/export.h> 3 + #include <linux/min_heap.h> 4 + 5 + void __min_heap_init(min_heap_char *heap, void *data, int size) 6 + { 7 + __min_heap_init_inline(heap, data, size); 8 + } 9 + EXPORT_SYMBOL(__min_heap_init); 10 + 11 + void *__min_heap_peek(struct min_heap_char *heap) 12 + { 13 + return __min_heap_peek_inline(heap); 14 + } 15 + EXPORT_SYMBOL(__min_heap_peek); 16 + 17 + bool __min_heap_full(min_heap_char *heap) 18 + { 19 + return __min_heap_full_inline(heap); 20 + } 21 + EXPORT_SYMBOL(__min_heap_full); 22 + 23 + void __min_heap_sift_down(min_heap_char *heap, int pos, size_t elem_size, 24 + const struct min_heap_callbacks *func, void *args) 25 + { 26 + __min_heap_sift_down_inline(heap, pos, elem_size, func, args); 27 + } 28 + EXPORT_SYMBOL(__min_heap_sift_down); 29 + 30 + void __min_heap_sift_up(min_heap_char *heap, size_t elem_size, size_t idx, 31 + const struct min_heap_callbacks *func, void *args) 32 + { 33 + __min_heap_sift_up_inline(heap, elem_size, idx, func, args); 34 + } 35 + EXPORT_SYMBOL(__min_heap_sift_up); 36 + 37 + void __min_heapify_all(min_heap_char *heap, size_t elem_size, 38 + const struct min_heap_callbacks *func, void *args) 39 + { 40 + __min_heapify_all_inline(heap, elem_size, func, args); 41 + } 42 + EXPORT_SYMBOL(__min_heapify_all); 43 + 44 + bool __min_heap_pop(min_heap_char *heap, size_t elem_size, 45 + const struct min_heap_callbacks *func, void *args) 46 + { 47 + return __min_heap_pop_inline(heap, elem_size, func, args); 48 + } 49 + EXPORT_SYMBOL(__min_heap_pop); 50 + 51 + void __min_heap_pop_push(min_heap_char *heap, const void *element, size_t elem_size, 52 + const struct min_heap_callbacks *func, void *args) 53 + { 54 + __min_heap_pop_push_inline(heap, element, elem_size, func, args); 55 + } 56 + EXPORT_SYMBOL(__min_heap_pop_push); 57 + 58 + bool __min_heap_push(min_heap_char *heap, const void *element, size_t elem_size, 59 + const struct min_heap_callbacks *func, void *args) 60 + { 61 + return __min_heap_push_inline(heap, element, elem_size, func, args); 62 + } 63 + EXPORT_SYMBOL(__min_heap_push); 64 + 65 + bool __min_heap_del(min_heap_char *heap, size_t elem_size, size_t idx, 66 + const struct min_heap_callbacks *func, void *args) 67 + { 68 + return __min_heap_del_inline(heap, elem_size, idx, func, args); 69 + } 70 + EXPORT_SYMBOL(__min_heap_del);