tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include "rxe.h"
35#include "rxe_loc.h"
36
37/* info about object pools
38 * note that mr and mw share a single index space
39 * so that one can map an lkey to the correct type of object
40 */
41struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
42 [RXE_TYPE_UC] = {
43 .name = "rxe-uc",
44 .size = sizeof(struct rxe_ucontext),
45 .flags = RXE_POOL_NO_ALLOC,
46 },
47 [RXE_TYPE_PD] = {
48 .name = "rxe-pd",
49 .size = sizeof(struct rxe_pd),
50 .flags = RXE_POOL_NO_ALLOC,
51 },
52 [RXE_TYPE_AH] = {
53 .name = "rxe-ah",
54 .size = sizeof(struct rxe_ah),
55 .flags = RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC,
56 },
57 [RXE_TYPE_SRQ] = {
58 .name = "rxe-srq",
59 .size = sizeof(struct rxe_srq),
60 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
61 .min_index = RXE_MIN_SRQ_INDEX,
62 .max_index = RXE_MAX_SRQ_INDEX,
63 },
64 [RXE_TYPE_QP] = {
65 .name = "rxe-qp",
66 .size = sizeof(struct rxe_qp),
67 .cleanup = rxe_qp_cleanup,
68 .flags = RXE_POOL_INDEX,
69 .min_index = RXE_MIN_QP_INDEX,
70 .max_index = RXE_MAX_QP_INDEX,
71 },
72 [RXE_TYPE_CQ] = {
73 .name = "rxe-cq",
74 .size = sizeof(struct rxe_cq),
75 .cleanup = rxe_cq_cleanup,
76 },
77 [RXE_TYPE_MR] = {
78 .name = "rxe-mr",
79 .size = sizeof(struct rxe_mem),
80 .cleanup = rxe_mem_cleanup,
81 .flags = RXE_POOL_INDEX,
82 .max_index = RXE_MAX_MR_INDEX,
83 .min_index = RXE_MIN_MR_INDEX,
84 },
85 [RXE_TYPE_MW] = {
86 .name = "rxe-mw",
87 .size = sizeof(struct rxe_mem),
88 .flags = RXE_POOL_INDEX,
89 .max_index = RXE_MAX_MW_INDEX,
90 .min_index = RXE_MIN_MW_INDEX,
91 },
92 [RXE_TYPE_MC_GRP] = {
93 .name = "rxe-mc_grp",
94 .size = sizeof(struct rxe_mc_grp),
95 .cleanup = rxe_mc_cleanup,
96 .flags = RXE_POOL_KEY,
97 .key_offset = offsetof(struct rxe_mc_grp, mgid),
98 .key_size = sizeof(union ib_gid),
99 },
100 [RXE_TYPE_MC_ELEM] = {
101 .name = "rxe-mc_elem",
102 .size = sizeof(struct rxe_mc_elem),
103 .flags = RXE_POOL_ATOMIC,
104 },
105};
106
107static inline const char *pool_name(struct rxe_pool *pool)
108{
109 return rxe_type_info[pool->type].name;
110}
111
112static inline struct kmem_cache *pool_cache(struct rxe_pool *pool)
113{
114 return rxe_type_info[pool->type].cache;
115}
116
117static void rxe_cache_clean(size_t cnt)
118{
119 int i;
120 struct rxe_type_info *type;
121
122 for (i = 0; i < cnt; i++) {
123 type = &rxe_type_info[i];
124 if (!(type->flags & RXE_POOL_NO_ALLOC)) {
125 kmem_cache_destroy(type->cache);
126 type->cache = NULL;
127 }
128 }
129}
130
131int rxe_cache_init(void)
132{
133 int err;
134 int i;
135 size_t size;
136 struct rxe_type_info *type;
137
138 for (i = 0; i < RXE_NUM_TYPES; i++) {
139 type = &rxe_type_info[i];
140 size = ALIGN(type->size, RXE_POOL_ALIGN);
141 if (!(type->flags & RXE_POOL_NO_ALLOC)) {
142 type->cache =
143 kmem_cache_create(type->name, size,
144 RXE_POOL_ALIGN,
145 RXE_POOL_CACHE_FLAGS, NULL);
146 if (!type->cache) {
147 pr_err("Unable to init kmem cache for %s\n",
148 type->name);
149 err = -ENOMEM;
150 goto err1;
151 }
152 }
153 }
154
155 return 0;
156
157err1:
158 rxe_cache_clean(i);
159
160 return err;
161}
162
163void rxe_cache_exit(void)
164{
165 rxe_cache_clean(RXE_NUM_TYPES);
166}
167
168static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
169{
170 int err = 0;
171 size_t size;
172
173 if ((max - min + 1) < pool->max_elem) {
174 pr_warn("not enough indices for max_elem\n");
175 err = -EINVAL;
176 goto out;
177 }
178
179 pool->max_index = max;
180 pool->min_index = min;
181
182 size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
183 pool->table = kmalloc(size, GFP_KERNEL);
184 if (!pool->table) {
185 err = -ENOMEM;
186 goto out;
187 }
188
189 pool->table_size = size;
190 bitmap_zero(pool->table, max - min + 1);
191
192out:
193 return err;
194}
195
196int rxe_pool_init(
197 struct rxe_dev *rxe,
198 struct rxe_pool *pool,
199 enum rxe_elem_type type,
200 unsigned int max_elem)
201{
202 int err = 0;
203 size_t size = rxe_type_info[type].size;
204
205 memset(pool, 0, sizeof(*pool));
206
207 pool->rxe = rxe;
208 pool->type = type;
209 pool->max_elem = max_elem;
210 pool->elem_size = ALIGN(size, RXE_POOL_ALIGN);
211 pool->flags = rxe_type_info[type].flags;
212 pool->tree = RB_ROOT;
213 pool->cleanup = rxe_type_info[type].cleanup;
214
215 atomic_set(&pool->num_elem, 0);
216
217 kref_init(&pool->ref_cnt);
218
219 rwlock_init(&pool->pool_lock);
220
221 if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
222 err = rxe_pool_init_index(pool,
223 rxe_type_info[type].max_index,
224 rxe_type_info[type].min_index);
225 if (err)
226 goto out;
227 }
228
229 if (rxe_type_info[type].flags & RXE_POOL_KEY) {
230 pool->key_offset = rxe_type_info[type].key_offset;
231 pool->key_size = rxe_type_info[type].key_size;
232 }
233
234 pool->state = RXE_POOL_STATE_VALID;
235
236out:
237 return err;
238}
239
240static void rxe_pool_release(struct kref *kref)
241{
242 struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
243
244 pool->state = RXE_POOL_STATE_INVALID;
245 kfree(pool->table);
246}
247
248static void rxe_pool_put(struct rxe_pool *pool)
249{
250 kref_put(&pool->ref_cnt, rxe_pool_release);
251}
252
253void rxe_pool_cleanup(struct rxe_pool *pool)
254{
255 unsigned long flags;
256
257 write_lock_irqsave(&pool->pool_lock, flags);
258 pool->state = RXE_POOL_STATE_INVALID;
259 if (atomic_read(&pool->num_elem) > 0)
260 pr_warn("%s pool destroyed with unfree'd elem\n",
261 pool_name(pool));
262 write_unlock_irqrestore(&pool->pool_lock, flags);
263
264 rxe_pool_put(pool);
265}
266
267static u32 alloc_index(struct rxe_pool *pool)
268{
269 u32 index;
270 u32 range = pool->max_index - pool->min_index + 1;
271
272 index = find_next_zero_bit(pool->table, range, pool->last);
273 if (index >= range)
274 index = find_first_zero_bit(pool->table, range);
275
276 WARN_ON_ONCE(index >= range);
277 set_bit(index, pool->table);
278 pool->last = index;
279 return index + pool->min_index;
280}
281
282static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
283{
284 struct rb_node **link = &pool->tree.rb_node;
285 struct rb_node *parent = NULL;
286 struct rxe_pool_entry *elem;
287
288 while (*link) {
289 parent = *link;
290 elem = rb_entry(parent, struct rxe_pool_entry, node);
291
292 if (elem->index == new->index) {
293 pr_warn("element already exists!\n");
294 goto out;
295 }
296
297 if (elem->index > new->index)
298 link = &(*link)->rb_left;
299 else
300 link = &(*link)->rb_right;
301 }
302
303 rb_link_node(&new->node, parent, link);
304 rb_insert_color(&new->node, &pool->tree);
305out:
306 return;
307}
308
309static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
310{
311 struct rb_node **link = &pool->tree.rb_node;
312 struct rb_node *parent = NULL;
313 struct rxe_pool_entry *elem;
314 int cmp;
315
316 while (*link) {
317 parent = *link;
318 elem = rb_entry(parent, struct rxe_pool_entry, node);
319
320 cmp = memcmp((u8 *)elem + pool->key_offset,
321 (u8 *)new + pool->key_offset, pool->key_size);
322
323 if (cmp == 0) {
324 pr_warn("key already exists!\n");
325 goto out;
326 }
327
328 if (cmp > 0)
329 link = &(*link)->rb_left;
330 else
331 link = &(*link)->rb_right;
332 }
333
334 rb_link_node(&new->node, parent, link);
335 rb_insert_color(&new->node, &pool->tree);
336out:
337 return;
338}
339
340void rxe_add_key(void *arg, void *key)
341{
342 struct rxe_pool_entry *elem = arg;
343 struct rxe_pool *pool = elem->pool;
344 unsigned long flags;
345
346 write_lock_irqsave(&pool->pool_lock, flags);
347 memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
348 insert_key(pool, elem);
349 write_unlock_irqrestore(&pool->pool_lock, flags);
350}
351
352void rxe_drop_key(void *arg)
353{
354 struct rxe_pool_entry *elem = arg;
355 struct rxe_pool *pool = elem->pool;
356 unsigned long flags;
357
358 write_lock_irqsave(&pool->pool_lock, flags);
359 rb_erase(&elem->node, &pool->tree);
360 write_unlock_irqrestore(&pool->pool_lock, flags);
361}
362
363void rxe_add_index(void *arg)
364{
365 struct rxe_pool_entry *elem = arg;
366 struct rxe_pool *pool = elem->pool;
367 unsigned long flags;
368
369 write_lock_irqsave(&pool->pool_lock, flags);
370 elem->index = alloc_index(pool);
371 insert_index(pool, elem);
372 write_unlock_irqrestore(&pool->pool_lock, flags);
373}
374
375void rxe_drop_index(void *arg)
376{
377 struct rxe_pool_entry *elem = arg;
378 struct rxe_pool *pool = elem->pool;
379 unsigned long flags;
380
381 write_lock_irqsave(&pool->pool_lock, flags);
382 clear_bit(elem->index - pool->min_index, pool->table);
383 rb_erase(&elem->node, &pool->tree);
384 write_unlock_irqrestore(&pool->pool_lock, flags);
385}
386
387void *rxe_alloc(struct rxe_pool *pool)
388{
389 struct rxe_pool_entry *elem;
390 unsigned long flags;
391
392 might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
393
394 read_lock_irqsave(&pool->pool_lock, flags);
395 if (pool->state != RXE_POOL_STATE_VALID) {
396 read_unlock_irqrestore(&pool->pool_lock, flags);
397 return NULL;
398 }
399 kref_get(&pool->ref_cnt);
400 read_unlock_irqrestore(&pool->pool_lock, flags);
401
402 if (!ib_device_try_get(&pool->rxe->ib_dev))
403 goto out_put_pool;
404
405 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
406 goto out_cnt;
407
408 elem = kmem_cache_zalloc(pool_cache(pool),
409 (pool->flags & RXE_POOL_ATOMIC) ?
410 GFP_ATOMIC : GFP_KERNEL);
411 if (!elem)
412 goto out_cnt;
413
414 elem->pool = pool;
415 kref_init(&elem->ref_cnt);
416
417 return elem;
418
419out_cnt:
420 atomic_dec(&pool->num_elem);
421 ib_device_put(&pool->rxe->ib_dev);
422out_put_pool:
423 rxe_pool_put(pool);
424 return NULL;
425}
426
427int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
428{
429 unsigned long flags;
430
431 might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
432
433 read_lock_irqsave(&pool->pool_lock, flags);
434 if (pool->state != RXE_POOL_STATE_VALID) {
435 read_unlock_irqrestore(&pool->pool_lock, flags);
436 return -EINVAL;
437 }
438 kref_get(&pool->ref_cnt);
439 read_unlock_irqrestore(&pool->pool_lock, flags);
440
441 if (!ib_device_try_get(&pool->rxe->ib_dev))
442 goto out_put_pool;
443
444 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
445 goto out_cnt;
446
447 elem->pool = pool;
448 kref_init(&elem->ref_cnt);
449
450 return 0;
451
452out_cnt:
453 atomic_dec(&pool->num_elem);
454 ib_device_put(&pool->rxe->ib_dev);
455out_put_pool:
456 rxe_pool_put(pool);
457 return -EINVAL;
458}
459
460void rxe_elem_release(struct kref *kref)
461{
462 struct rxe_pool_entry *elem =
463 container_of(kref, struct rxe_pool_entry, ref_cnt);
464 struct rxe_pool *pool = elem->pool;
465
466 if (pool->cleanup)
467 pool->cleanup(elem);
468
469 if (!(pool->flags & RXE_POOL_NO_ALLOC))
470 kmem_cache_free(pool_cache(pool), elem);
471 atomic_dec(&pool->num_elem);
472 ib_device_put(&pool->rxe->ib_dev);
473 rxe_pool_put(pool);
474}
475
476void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
477{
478 struct rb_node *node = NULL;
479 struct rxe_pool_entry *elem = NULL;
480 unsigned long flags;
481
482 read_lock_irqsave(&pool->pool_lock, flags);
483
484 if (pool->state != RXE_POOL_STATE_VALID)
485 goto out;
486
487 node = pool->tree.rb_node;
488
489 while (node) {
490 elem = rb_entry(node, struct rxe_pool_entry, node);
491
492 if (elem->index > index)
493 node = node->rb_left;
494 else if (elem->index < index)
495 node = node->rb_right;
496 else {
497 kref_get(&elem->ref_cnt);
498 break;
499 }
500 }
501
502out:
503 read_unlock_irqrestore(&pool->pool_lock, flags);
504 return node ? elem : NULL;
505}
506
507void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
508{
509 struct rb_node *node = NULL;
510 struct rxe_pool_entry *elem = NULL;
511 int cmp;
512 unsigned long flags;
513
514 read_lock_irqsave(&pool->pool_lock, flags);
515
516 if (pool->state != RXE_POOL_STATE_VALID)
517 goto out;
518
519 node = pool->tree.rb_node;
520
521 while (node) {
522 elem = rb_entry(node, struct rxe_pool_entry, node);
523
524 cmp = memcmp((u8 *)elem + pool->key_offset,
525 key, pool->key_size);
526
527 if (cmp > 0)
528 node = node->rb_left;
529 else if (cmp < 0)
530 node = node->rb_right;
531 else
532 break;
533 }
534
535 if (node)
536 kref_get(&elem->ref_cnt);
537
538out:
539 read_unlock_irqrestore(&pool->pool_lock, flags);
540 return node ? elem : NULL;
541}