Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

Merge tag 'slab-for-6.13-v2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab

Pull slab updates from Vlastimil Babka:

- Add new slab_strict_numa boot parameter to enforce per-object memory
policies on top of slab folio policies, for systems where saving cost
of remote accesses is more important than minimizing slab allocation
overhead (Christoph Lameter)

- Fix for freeptr_offset alignment check being too strict for m68k
(Geert Uytterhoeven)

- krealloc() fixes for not violating __GFP_ZERO guarantees on
krealloc() when slub_debug (redzone and object tracking) is enabled
(Feng Tang)

- Fix a memory leak in case sysfs registration fails for a slab cache,
and also no longer fail to create the cache in that case (Hyeonggon
Yoo)

- Fix handling of detected consistency problems (due to buggy slab
user) with slub_debug enabled, so that it does not cause further list
corruption bugs (yuan.gao)

- Code cleanup and kerneldocs polishing (Zhen Lei, Vlastimil Babka)

* tag 'slab-for-6.13-v2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
slab: Fix too strict alignment check in create_cache()
mm/slab: Allow cache creation to proceed even if sysfs registration fails
mm/slub: Avoid list corruption when removing a slab from the full list
mm/slub, kunit: Add testcase for krealloc redzone and zeroing
mm/slub: Improve redzone check and zeroing for krealloc()
mm/slub: Consider kfence case for get_orig_size()
SLUB: Add support for per object memory policies
mm, slab: add kerneldocs for common SLAB_ flags
mm/slab: remove duplicate check in create_cache()
mm/slub: Move krealloc() and related code to slub.c
mm/kasan: Don't store metadata inside kmalloc object when slub_debug_orig_size is on

+325 -137
+10
Documentation/admin-guide/kernel-parameters.txt
··· 6158 6158 For more information see Documentation/mm/slub.rst. 6159 6159 (slub_nomerge legacy name also accepted for now) 6160 6160 6161 + slab_strict_numa [MM] 6162 + Support memory policies on a per object level 6163 + in the slab allocator. The default is for memory 6164 + policies to be applied at the folio level when 6165 + a new folio is needed or a partial folio is 6166 + retrieved from the lists. Increases overhead 6167 + in the slab fastpaths but gains more accurate 6168 + NUMA kernel object placement which helps with slow 6169 + interconnects in NUMA systems. 6170 + 6161 6171 slram= [HW,MTD] 6162 6172 6163 6173 smart2= [HW]
+9
Documentation/mm/slub.rst
··· 175 175 ``slab_max_order`` to 0, what cause minimum possible order of 176 176 slabs allocation. 177 177 178 + ``slab_strict_numa`` 179 + Enables the application of memory policies on each 180 + allocation. This results in more accurate placement of 181 + objects which may result in the reduction of accesses 182 + to remote nodes. The default is to only apply memory 183 + policies at the folio level when a new folio is acquired 184 + or a folio is retrieved from the lists. Enabling this 185 + option reduces the fastpath performance of the slab allocator. 186 + 178 187 SLUB Debug output 179 188 ================= 180 189
+41 -19
include/linux/slab.h
··· 77 77 #define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON) 78 78 /* Indicate a kmalloc slab */ 79 79 #define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC) 80 - /* Align objs on cache lines */ 80 + /** 81 + * define SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries. 82 + * 83 + * Sufficiently large objects are aligned on cache line boundary. For object 84 + * size smaller than a half of cache line size, the alignment is on the half of 85 + * cache line size. In general, if object size is smaller than 1/2^n of cache 86 + * line size, the alignment is adjusted to 1/2^n. 87 + * 88 + * If explicit alignment is also requested by the respective 89 + * &struct kmem_cache_args field, the greater of both is alignments is applied. 90 + */ 81 91 #define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN) 82 92 /* Use GFP_DMA memory */ 83 93 #define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA) ··· 97 87 #define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER) 98 88 /* Panic if kmem_cache_create() fails */ 99 89 #define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC) 100 - /* 101 - * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! 90 + /** 91 + * define SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! 102 92 * 103 93 * This delays freeing the SLAB page by a grace period, it does _NOT_ 104 94 * delay object freeing. This means that if you do kmem_cache_free() ··· 109 99 * stays valid, the trick to using this is relying on an independent 110 100 * object validation pass. Something like: 111 101 * 112 - * begin: 113 - * rcu_read_lock(); 114 - * obj = lockless_lookup(key); 115 - * if (obj) { 116 - * if (!try_get_ref(obj)) // might fail for free objects 117 - * rcu_read_unlock(); 118 - * goto begin; 102 + * :: 119 103 * 120 - * if (obj->key != key) { // not the object we expected 121 - * put_ref(obj); 122 - * rcu_read_unlock(); 123 - * goto begin; 124 - * } 125 - * } 104 + * begin: 105 + * rcu_read_lock(); 106 + * obj = lockless_lookup(key); 107 + * if (obj) { 108 + * if (!try_get_ref(obj)) // might fail for free objects 109 + * rcu_read_unlock(); 110 + * goto begin; 111 + * 112 + * if (obj->key != key) { // not the object we expected 113 + * put_ref(obj); 114 + * rcu_read_unlock(); 115 + * goto begin; 116 + * } 117 + * } 126 118 * rcu_read_unlock(); 127 119 * 128 120 * This is useful if we need to approach a kernel structure obliquely, ··· 149 137 * 150 138 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. 151 139 */ 152 - /* Defer freeing slabs to RCU */ 153 140 #define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU) 154 141 /* Trace allocations and frees */ 155 142 #define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE) ··· 181 170 #else 182 171 # define SLAB_FAILSLAB __SLAB_FLAG_UNUSED 183 172 #endif 184 - /* Account to memcg */ 173 + /** 174 + * define SLAB_ACCOUNT - Account allocations to memcg. 175 + * 176 + * All object allocations from this cache will be memcg accounted, regardless of 177 + * __GFP_ACCOUNT being or not being passed to individual allocations. 178 + */ 185 179 #ifdef CONFIG_MEMCG 186 180 # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) 187 181 #else ··· 213 197 #endif 214 198 215 199 /* The following flags affect the page allocator grouping pages by mobility */ 216 - /* Objects are reclaimable */ 200 + /** 201 + * define SLAB_RECLAIM_ACCOUNT - Objects are reclaimable. 202 + * 203 + * Use this flag for caches that have an associated shrinker. As a result, slab 204 + * pages are allocated with __GFP_RECLAIMABLE, which affects grouping pages by 205 + * mobility, and are accounted in SReclaimable counter in /proc/meminfo 206 + */ 217 207 #ifndef CONFIG_SLUB_TINY 218 208 #define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT) 219 209 #else
+42
lib/slub_kunit.c
··· 192 192 KUNIT_EXPECT_EQ(test, 2, slab_errors); 193 193 } 194 194 195 + static void test_krealloc_redzone_zeroing(struct kunit *test) 196 + { 197 + u8 *p; 198 + int i; 199 + struct kmem_cache *s = test_kmem_cache_create("TestSlub_krealloc", 64, 200 + SLAB_KMALLOC|SLAB_STORE_USER|SLAB_RED_ZONE); 201 + 202 + p = alloc_hooks(__kmalloc_cache_noprof(s, GFP_KERNEL, 48)); 203 + memset(p, 0xff, 48); 204 + 205 + kasan_disable_current(); 206 + OPTIMIZER_HIDE_VAR(p); 207 + 208 + /* Test shrink */ 209 + p = krealloc(p, 40, GFP_KERNEL | __GFP_ZERO); 210 + for (i = 40; i < 64; i++) 211 + KUNIT_EXPECT_EQ(test, p[i], SLUB_RED_ACTIVE); 212 + 213 + /* Test grow within the same 64B kmalloc object */ 214 + p = krealloc(p, 56, GFP_KERNEL | __GFP_ZERO); 215 + for (i = 40; i < 56; i++) 216 + KUNIT_EXPECT_EQ(test, p[i], 0); 217 + for (i = 56; i < 64; i++) 218 + KUNIT_EXPECT_EQ(test, p[i], SLUB_RED_ACTIVE); 219 + 220 + validate_slab_cache(s); 221 + KUNIT_EXPECT_EQ(test, 0, slab_errors); 222 + 223 + memset(p, 0xff, 56); 224 + /* Test grow with allocating a bigger 128B object */ 225 + p = krealloc(p, 112, GFP_KERNEL | __GFP_ZERO); 226 + for (i = 0; i < 56; i++) 227 + KUNIT_EXPECT_EQ(test, p[i], 0xff); 228 + for (i = 56; i < 112; i++) 229 + KUNIT_EXPECT_EQ(test, p[i], 0); 230 + 231 + kfree(p); 232 + kasan_enable_current(); 233 + kmem_cache_destroy(s); 234 + } 235 + 195 236 static int test_init(struct kunit *test) 196 237 { 197 238 slab_errors = 0; ··· 255 214 KUNIT_CASE(test_kmalloc_redzone_access), 256 215 KUNIT_CASE(test_kfree_rcu), 257 216 KUNIT_CASE(test_leak_destroy), 217 + KUNIT_CASE(test_krealloc_redzone_zeroing), 258 218 {} 259 219 }; 260 220
+5 -2
mm/kasan/generic.c
··· 392 392 * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can 393 393 * be touched after it was freed, or 394 394 * 2. Object has a constructor, which means it's expected to 395 - * retain its content until the next allocation. 395 + * retain its content until the next allocation, or 396 + * 3. It is from a kmalloc cache which enables the debug option 397 + * to store original size. 396 398 */ 397 - if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor) { 399 + if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor || 400 + slub_debug_orig_size(cache)) { 398 401 cache->kasan_info.free_meta_offset = *size; 399 402 *size += sizeof(struct kasan_free_meta); 400 403 goto free_meta_added;
+11
mm/slab.h
··· 73 73 struct { 74 74 unsigned inuse:16; 75 75 unsigned objects:15; 76 + /* 77 + * If slab debugging is enabled then the 78 + * frozen bit can be reused to indicate 79 + * that the slab was corrupted 80 + */ 76 81 unsigned frozen:1; 77 82 }; 78 83 }; ··· 699 694 700 695 void __check_heap_object(const void *ptr, unsigned long n, 701 696 const struct slab *slab, bool to_user); 697 + 698 + static inline bool slub_debug_orig_size(struct kmem_cache *s) 699 + { 700 + return (kmem_cache_debug_flags(s, SLAB_STORE_USER) && 701 + (s->flags & SLAB_KMALLOC)); 702 + } 702 703 703 704 #ifdef CONFIG_SLUB_DEBUG 704 705 void skip_orig_size_check(struct kmem_cache *s, const void *object);
+14 -89
mm/slab_common.c
··· 222 222 struct kmem_cache *s; 223 223 int err; 224 224 225 - if (WARN_ON(args->useroffset + args->usersize > object_size)) 226 - args->useroffset = args->usersize = 0; 227 - 228 225 /* If a custom freelist pointer is requested make sure it's sane. */ 229 226 err = -EINVAL; 230 227 if (args->use_freeptr_offset && 231 228 (args->freeptr_offset >= object_size || 232 229 !(flags & SLAB_TYPESAFE_BY_RCU) || 233 - !IS_ALIGNED(args->freeptr_offset, sizeof(freeptr_t)))) 230 + !IS_ALIGNED(args->freeptr_offset, __alignof__(freeptr_t)))) 234 231 goto out; 235 232 236 233 err = -ENOMEM; ··· 254 257 * @object_size: The size of objects to be created in this cache. 255 258 * @args: Additional arguments for the cache creation (see 256 259 * &struct kmem_cache_args). 257 - * @flags: See %SLAB_* flags for an explanation of individual @flags. 260 + * @flags: See the desriptions of individual flags. The common ones are listed 261 + * in the description below. 258 262 * 259 263 * Not to be called directly, use the kmem_cache_create() wrapper with the same 260 264 * parameters. 265 + * 266 + * Commonly used @flags: 267 + * 268 + * &SLAB_ACCOUNT - Account allocations to memcg. 269 + * 270 + * &SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries. 271 + * 272 + * &SLAB_RECLAIM_ACCOUNT - Objects are reclaimable. 273 + * 274 + * &SLAB_TYPESAFE_BY_RCU - Slab page (not individual objects) freeing delayed 275 + * by a grace period - see the full description before using. 261 276 * 262 277 * Context: Cannot be called within a interrupt, but can be interrupted. 263 278 * ··· 1207 1198 module_init(slab_proc_init); 1208 1199 1209 1200 #endif /* CONFIG_SLUB_DEBUG */ 1210 - 1211 - static __always_inline __realloc_size(2) void * 1212 - __do_krealloc(const void *p, size_t new_size, gfp_t flags) 1213 - { 1214 - void *ret; 1215 - size_t ks; 1216 - 1217 - /* Check for double-free before calling ksize. */ 1218 - if (likely(!ZERO_OR_NULL_PTR(p))) { 1219 - if (!kasan_check_byte(p)) 1220 - return NULL; 1221 - ks = ksize(p); 1222 - } else 1223 - ks = 0; 1224 - 1225 - /* If the object still fits, repoison it precisely. */ 1226 - if (ks >= new_size) { 1227 - /* Zero out spare memory. */ 1228 - if (want_init_on_alloc(flags)) { 1229 - kasan_disable_current(); 1230 - memset(kasan_reset_tag(p) + new_size, 0, ks - new_size); 1231 - kasan_enable_current(); 1232 - } 1233 - 1234 - p = kasan_krealloc((void *)p, new_size, flags); 1235 - return (void *)p; 1236 - } 1237 - 1238 - ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_); 1239 - if (ret && p) { 1240 - /* Disable KASAN checks as the object's redzone is accessed. */ 1241 - kasan_disable_current(); 1242 - memcpy(ret, kasan_reset_tag(p), ks); 1243 - kasan_enable_current(); 1244 - } 1245 - 1246 - return ret; 1247 - } 1248 - 1249 - /** 1250 - * krealloc - reallocate memory. The contents will remain unchanged. 1251 - * @p: object to reallocate memory for. 1252 - * @new_size: how many bytes of memory are required. 1253 - * @flags: the type of memory to allocate. 1254 - * 1255 - * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 1256 - * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 1257 - * 1258 - * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 1259 - * initial memory allocation, every subsequent call to this API for the same 1260 - * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 1261 - * __GFP_ZERO is not fully honored by this API. 1262 - * 1263 - * This is the case, since krealloc() only knows about the bucket size of an 1264 - * allocation (but not the exact size it was allocated with) and hence 1265 - * implements the following semantics for shrinking and growing buffers with 1266 - * __GFP_ZERO. 1267 - * 1268 - * new bucket 1269 - * 0 size size 1270 - * |--------|----------------| 1271 - * | keep | zero | 1272 - * 1273 - * In any case, the contents of the object pointed to are preserved up to the 1274 - * lesser of the new and old sizes. 1275 - * 1276 - * Return: pointer to the allocated memory or %NULL in case of error 1277 - */ 1278 - void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags) 1279 - { 1280 - void *ret; 1281 - 1282 - if (unlikely(!new_size)) { 1283 - kfree(p); 1284 - return ZERO_SIZE_PTR; 1285 - } 1286 - 1287 - ret = __do_krealloc(p, new_size, flags); 1288 - if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret)) 1289 - kfree(p); 1290 - 1291 - return ret; 1292 - } 1293 - EXPORT_SYMBOL(krealloc_noprof); 1294 1201 1295 1202 /** 1296 1203 * kfree_sensitive - Clear sensitive information in memory before freeing
+193 -27
mm/slub.c
··· 218 218 #endif 219 219 #endif /* CONFIG_SLUB_DEBUG */ 220 220 221 + #ifdef CONFIG_NUMA 222 + static DEFINE_STATIC_KEY_FALSE(strict_numa); 223 + #endif 224 + 221 225 /* Structure holding parameters for get_partial() call chain */ 222 226 struct partial_context { 223 227 gfp_t flags; ··· 232 228 static inline bool kmem_cache_debug(struct kmem_cache *s) 233 229 { 234 230 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS); 235 - } 236 - 237 - static inline bool slub_debug_orig_size(struct kmem_cache *s) 238 - { 239 - return (kmem_cache_debug_flags(s, SLAB_STORE_USER) && 240 - (s->flags & SLAB_KMALLOC)); 241 231 } 242 232 243 233 void *fixup_red_left(struct kmem_cache *s, void *p) ··· 758 760 void *object, unsigned int orig_size) 759 761 { 760 762 void *p = kasan_reset_tag(object); 761 - unsigned int kasan_meta_size; 762 763 763 764 if (!slub_debug_orig_size(s)) 764 765 return; 765 - 766 - /* 767 - * KASAN can save its free meta data inside of the object at offset 0. 768 - * If this meta data size is larger than 'orig_size', it will overlap 769 - * the data redzone in [orig_size+1, object_size]. Thus, we adjust 770 - * 'orig_size' to be as at least as big as KASAN's meta data. 771 - */ 772 - kasan_meta_size = kasan_metadata_size(s, true); 773 - if (kasan_meta_size > orig_size) 774 - orig_size = kasan_meta_size; 775 766 776 767 p += get_info_end(s); 777 768 p += sizeof(struct track) * 2; ··· 771 784 static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) 772 785 { 773 786 void *p = kasan_reset_tag(object); 787 + 788 + if (is_kfence_address(object)) 789 + return kfence_ksize(object); 774 790 775 791 if (!slub_debug_orig_size(s)) 776 792 return s->object_size; ··· 1413 1423 slab->inuse, slab->objects); 1414 1424 return 0; 1415 1425 } 1426 + if (slab->frozen) { 1427 + slab_err(s, slab, "Slab disabled since SLUB metadata consistency check failed"); 1428 + return 0; 1429 + } 1430 + 1416 1431 /* Slab_pad_check fixes things up after itself */ 1417 1432 slab_pad_check(s, slab); 1418 1433 return 1; ··· 1598 1603 slab_fix(s, "Marking all objects used"); 1599 1604 slab->inuse = slab->objects; 1600 1605 slab->freelist = NULL; 1606 + slab->frozen = 1; /* mark consistency-failed slab as frozen */ 1601 1607 } 1602 1608 return false; 1603 1609 } ··· 2740 2744 slab->inuse++; 2741 2745 2742 2746 if (!alloc_debug_processing(s, slab, object, orig_size)) { 2743 - remove_partial(n, slab); 2747 + if (folio_test_slab(slab_folio(slab))) 2748 + remove_partial(n, slab); 2744 2749 return NULL; 2745 2750 } 2746 2751 ··· 3953 3956 object = c->freelist; 3954 3957 slab = c->slab; 3955 3958 3959 + #ifdef CONFIG_NUMA 3960 + if (static_branch_unlikely(&strict_numa) && 3961 + node == NUMA_NO_NODE) { 3962 + 3963 + struct mempolicy *mpol = current->mempolicy; 3964 + 3965 + if (mpol) { 3966 + /* 3967 + * Special BIND rule support. If existing slab 3968 + * is in permitted set then do not redirect 3969 + * to a particular node. 3970 + * Otherwise we apply the memory policy to get 3971 + * the node we need to allocate on. 3972 + */ 3973 + if (mpol->mode != MPOL_BIND || !slab || 3974 + !node_isset(slab_nid(slab), mpol->nodes)) 3975 + 3976 + node = mempolicy_slab_node(); 3977 + } 3978 + } 3979 + #endif 3980 + 3956 3981 if (!USE_LOCKLESS_FAST_PATH() || 3957 3982 unlikely(!object || !slab || !node_match(slab, node))) { 3958 3983 object = __slab_alloc(s, gfpflags, node, addr, c, orig_size); ··· 4746 4727 slab_free(s, slab, x, _RET_IP_); 4747 4728 } 4748 4729 EXPORT_SYMBOL(kfree); 4730 + 4731 + static __always_inline __realloc_size(2) void * 4732 + __do_krealloc(const void *p, size_t new_size, gfp_t flags) 4733 + { 4734 + void *ret; 4735 + size_t ks = 0; 4736 + int orig_size = 0; 4737 + struct kmem_cache *s = NULL; 4738 + 4739 + if (unlikely(ZERO_OR_NULL_PTR(p))) 4740 + goto alloc_new; 4741 + 4742 + /* Check for double-free. */ 4743 + if (!kasan_check_byte(p)) 4744 + return NULL; 4745 + 4746 + if (is_kfence_address(p)) { 4747 + ks = orig_size = kfence_ksize(p); 4748 + } else { 4749 + struct folio *folio; 4750 + 4751 + folio = virt_to_folio(p); 4752 + if (unlikely(!folio_test_slab(folio))) { 4753 + /* Big kmalloc object */ 4754 + WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE); 4755 + WARN_ON(p != folio_address(folio)); 4756 + ks = folio_size(folio); 4757 + } else { 4758 + s = folio_slab(folio)->slab_cache; 4759 + orig_size = get_orig_size(s, (void *)p); 4760 + ks = s->object_size; 4761 + } 4762 + } 4763 + 4764 + /* If the old object doesn't fit, allocate a bigger one */ 4765 + if (new_size > ks) 4766 + goto alloc_new; 4767 + 4768 + /* Zero out spare memory. */ 4769 + if (want_init_on_alloc(flags)) { 4770 + kasan_disable_current(); 4771 + if (orig_size && orig_size < new_size) 4772 + memset(kasan_reset_tag(p) + orig_size, 0, new_size - orig_size); 4773 + else 4774 + memset(kasan_reset_tag(p) + new_size, 0, ks - new_size); 4775 + kasan_enable_current(); 4776 + } 4777 + 4778 + /* Setup kmalloc redzone when needed */ 4779 + if (s && slub_debug_orig_size(s)) { 4780 + set_orig_size(s, (void *)p, new_size); 4781 + if (s->flags & SLAB_RED_ZONE && new_size < ks) 4782 + memset_no_sanitize_memory(kasan_reset_tag(p) + new_size, 4783 + SLUB_RED_ACTIVE, ks - new_size); 4784 + } 4785 + 4786 + p = kasan_krealloc(p, new_size, flags); 4787 + return (void *)p; 4788 + 4789 + alloc_new: 4790 + ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_); 4791 + if (ret && p) { 4792 + /* Disable KASAN checks as the object's redzone is accessed. */ 4793 + kasan_disable_current(); 4794 + memcpy(ret, kasan_reset_tag(p), orig_size ?: ks); 4795 + kasan_enable_current(); 4796 + } 4797 + 4798 + return ret; 4799 + } 4800 + 4801 + /** 4802 + * krealloc - reallocate memory. The contents will remain unchanged. 4803 + * @p: object to reallocate memory for. 4804 + * @new_size: how many bytes of memory are required. 4805 + * @flags: the type of memory to allocate. 4806 + * 4807 + * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 4808 + * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 4809 + * 4810 + * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 4811 + * initial memory allocation, every subsequent call to this API for the same 4812 + * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 4813 + * __GFP_ZERO is not fully honored by this API. 4814 + * 4815 + * When slub_debug_orig_size() is off, krealloc() only knows about the bucket 4816 + * size of an allocation (but not the exact size it was allocated with) and 4817 + * hence implements the following semantics for shrinking and growing buffers 4818 + * with __GFP_ZERO. 4819 + * 4820 + * new bucket 4821 + * 0 size size 4822 + * |--------|----------------| 4823 + * | keep | zero | 4824 + * 4825 + * Otherwise, the original allocation size 'orig_size' could be used to 4826 + * precisely clear the requested size, and the new size will also be stored 4827 + * as the new 'orig_size'. 4828 + * 4829 + * In any case, the contents of the object pointed to are preserved up to the 4830 + * lesser of the new and old sizes. 4831 + * 4832 + * Return: pointer to the allocated memory or %NULL in case of error 4833 + */ 4834 + void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags) 4835 + { 4836 + void *ret; 4837 + 4838 + if (unlikely(!new_size)) { 4839 + kfree(p); 4840 + return ZERO_SIZE_PTR; 4841 + } 4842 + 4843 + ret = __do_krealloc(p, new_size, flags); 4844 + if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret)) 4845 + kfree(p); 4846 + 4847 + return ret; 4848 + } 4849 + EXPORT_SYMBOL(krealloc_noprof); 4749 4850 4750 4851 struct detached_freelist { 4751 4852 struct slab *slab; ··· 5741 5602 __setup("slab_min_objects=", setup_slub_min_objects); 5742 5603 __setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0); 5743 5604 5605 + #ifdef CONFIG_NUMA 5606 + static int __init setup_slab_strict_numa(char *str) 5607 + { 5608 + if (nr_node_ids > 1) { 5609 + static_branch_enable(&strict_numa); 5610 + pr_info("SLUB: Strict NUMA enabled.\n"); 5611 + } else { 5612 + pr_warn("slab_strict_numa parameter set on non NUMA system.\n"); 5613 + } 5614 + 5615 + return 1; 5616 + } 5617 + 5618 + __setup("slab_strict_numa", setup_slab_strict_numa); 5619 + #endif 5620 + 5621 + 5744 5622 #ifdef CONFIG_HARDENED_USERCOPY 5745 5623 /* 5746 5624 * Rejects incorrectly sized objects and objects that are to be copied ··· 6116 5960 s = find_mergeable(size, align, flags, name, ctor); 6117 5961 if (s) { 6118 5962 if (sysfs_slab_alias(s, name)) 6119 - return NULL; 5963 + pr_err("SLUB: Unable to add cache alias %s to sysfs\n", 5964 + name); 6120 5965 6121 5966 s->refcount++; 6122 5967 ··· 6199 6042 if (!alloc_kmem_cache_cpus(s)) 6200 6043 goto out; 6201 6044 6202 - /* Mutex is not taken during early boot */ 6203 - if (slab_state <= UP) { 6204 - err = 0; 6205 - goto out; 6206 - } 6045 + err = 0; 6207 6046 6208 - err = sysfs_slab_add(s); 6209 - if (err) 6047 + /* Mutex is not taken during early boot */ 6048 + if (slab_state <= UP) 6210 6049 goto out; 6050 + 6051 + /* 6052 + * Failing to create sysfs files is not critical to SLUB functionality. 6053 + * If it fails, proceed with cache creation without these files. 6054 + */ 6055 + if (sysfs_slab_add(s)) 6056 + pr_err("SLUB: Unable to add cache %s to sysfs\n", s->name); 6211 6057 6212 6058 if (s->flags & SLAB_STORE_USER) 6213 6059 debugfs_slab_add(s); ··· 7280 7120 7281 7121 void sysfs_slab_unlink(struct kmem_cache *s) 7282 7122 { 7283 - kobject_del(&s->kobj); 7123 + if (s->kobj.state_in_sysfs) 7124 + kobject_del(&s->kobj); 7284 7125 } 7285 7126 7286 7127 void sysfs_slab_release(struct kmem_cache *s) ··· 7310 7149 * If we have a leftover link then remove it. 7311 7150 */ 7312 7151 sysfs_remove_link(&slab_kset->kobj, name); 7152 + /* 7153 + * The original cache may have failed to generate sysfs file. 7154 + * In that case, sysfs_create_link() returns -ENOENT and 7155 + * symbolic link creation is skipped. 7156 + */ 7313 7157 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); 7314 7158 } 7315 7159