Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881// SPDX-License-Identifier: GPL-2.0-or-later/* * Copyright (C) 2017, Microsoft Corporation. * Copyright (c) 2025, Stefan Metzmacher */
#include "internal.h"
bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs){ /* * Test if FRWR (Fast Registration Work Requests) is supported on the * device This implementation requires FRWR on RDMA read/write return * value: true if it is supported */
if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) return false; if (attrs->max_fast_reg_page_list_len == 0) return false; return true;}EXPORT_SYMBOL_GPL(smbdirect_frwr_is_supported);
static void smbdirect_socket_cleanup_work(struct work_struct *work);
static int smbdirect_socket_rdma_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event){ struct smbdirect_socket *sc = id->context; int ret = -ESTALE;
/* * This should be replaced before any real work * starts! So it should never be called! */
if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) ret = -ENETDOWN; if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) ret = event->status; pr_err("%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), rdma_event_msg(sc->rdma.expected_event), rdma_event_msg(event->event), event->status, SMBDIRECT_DEBUG_ERR_PTR(ret)); WARN_ONCE(1, "%s should not be called!\n", __func__); sc->rdma.cm_id = NULL; return -ESTALE;}
int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc){ struct rdma_cm_id *id; int ret;
smbdirect_socket_init(sc);
id = rdma_create_id(net, smbdirect_socket_rdma_event_handler, sc, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(id)) { pr_err("%s: rdma_create_id() failed %1pe\n", __func__, id); return PTR_ERR(id); }
ret = rdma_set_afonly(id, 1); if (ret) { rdma_destroy_id(id); pr_err("%s: rdma_set_afonly() failed %1pe\n", __func__, SMBDIRECT_DEBUG_ERR_PTR(ret)); return ret; }
sc->rdma.cm_id = id;
INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work);
return 0;}
int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc){ struct smbdirect_socket *sc; int ret;
ret = -ENOMEM; sc = kzalloc_obj(*sc); if (!sc) goto alloc_failed;
ret = smbdirect_socket_init_new(net, sc); if (ret) goto init_failed;
kref_init(&sc->refs.destroy);
*_sc = sc; return 0;
init_failed: kfree(sc);alloc_failed: return ret;}EXPORT_SYMBOL_GPL(smbdirect_socket_create_kern);
int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc){ smbdirect_socket_init(sc);
sc->rdma.cm_id = id; sc->rdma.cm_id->context = sc; sc->rdma.cm_id->event_handler = smbdirect_socket_rdma_event_handler;
sc->ib.dev = sc->rdma.cm_id->device;
INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work);
return 0;}
int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc){ struct smbdirect_socket *sc; int ret;
ret = -ENOMEM; sc = kzalloc_obj(*sc); if (!sc) goto alloc_failed;
ret = smbdirect_socket_init_accepting(id, sc); if (ret) goto init_failed;
kref_init(&sc->refs.destroy);
*_sc = sc; return 0;
init_failed: kfree(sc);alloc_failed: return ret;}EXPORT_SYMBOL_GPL(smbdirect_socket_create_accepting);
int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, const struct smbdirect_socket_parameters *sp){ /* * This is only allowed before connect or accept */ WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, "status=%s first_error=%1pe", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); if (sc->status != SMBDIRECT_SOCKET_CREATED) return -EINVAL;
if (sp->flags & ~SMBDIRECT_FLAG_PORT_RANGE_MASK) return -EINVAL;
if (sp->initiator_depth > U8_MAX) return -EINVAL; if (sp->responder_resources > U8_MAX) return -EINVAL;
if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) return -EINVAL; else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB) rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_IB_CA); else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_RNIC);
/* * Make a copy of the callers parameters * from here we only work on the copy * * TODO: do we want consistency checking? */ sc->parameters = *sp;
return 0;}EXPORT_SYMBOL_GPL(smbdirect_socket_set_initial_parameters);
const struct smbdirect_socket_parameters *smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc){ return &sc->parameters;}EXPORT_SYMBOL_GPL(smbdirect_socket_get_current_parameters);
int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, enum ib_poll_context poll_ctx, gfp_t gfp_mask){ /* * This is only allowed before connect or accept */ WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, "status=%s first_error=%1pe", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); if (sc->status != SMBDIRECT_SOCKET_CREATED) return -EINVAL;
sc->ib.poll_ctx = poll_ctx;
sc->send_io.mem.gfp_mask = gfp_mask; sc->recv_io.mem.gfp_mask = gfp_mask; sc->rw_io.mem.gfp_mask = gfp_mask;
return 0;}EXPORT_SYMBOL_GPL(smbdirect_socket_set_kernel_settings);
void smbdirect_socket_set_logging(struct smbdirect_socket *sc, void *private_ptr, bool (*needed)(struct smbdirect_socket *sc, void *private_ptr, unsigned int lvl, unsigned int cls), void (*vaprintf)(struct smbdirect_socket *sc, const char *func, unsigned int line, void *private_ptr, unsigned int lvl, unsigned int cls, struct va_format *vaf)){ sc->logging.private_ptr = private_ptr; sc->logging.needed = needed; sc->logging.vaprintf = vaprintf;}EXPORT_SYMBOL_GPL(smbdirect_socket_set_logging);
static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc){ /* * Wake up all waiters in all wait queues * in order to notice the broken connection. */ wake_up_all(&sc->status_wait); wake_up_all(&sc->listen.wait_queue); wake_up_all(&sc->send_io.bcredits.wait_queue); wake_up_all(&sc->send_io.lcredits.wait_queue); wake_up_all(&sc->send_io.credits.wait_queue); wake_up_all(&sc->send_io.pending.zero_wait_queue); wake_up_all(&sc->recv_io.reassembly.wait_queue); wake_up_all(&sc->rw_io.credits.wait_queue); wake_up_all(&sc->mr_io.ready.wait_queue);}
void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, const char *macro_name, unsigned int lvl, const char *func, unsigned int line, int error, enum smbdirect_socket_status *force_status){ struct smbdirect_socket *psc, *tsc; unsigned long flags; bool was_first = false;
if (!sc->first_error) { ___smbdirect_log_generic(sc, func, line, lvl, SMBDIRECT_LOG_RDMA_EVENT, "%s(%1pe%s%s) called from %s in line=%u status=%s\n", macro_name, SMBDIRECT_DEBUG_ERR_PTR(error), force_status ? ", " : "", force_status ? smbdirect_socket_status_string(*force_status) : "", func, line, smbdirect_socket_status_string(sc->status)); if (error) sc->first_error = error; else sc->first_error = -ECONNABORTED; was_first = true; }
/* * make sure other work (than disconnect_work) * is not queued again but here we don't block and avoid * disable[_delayed]_work_sync() */ disable_work(&sc->connect.work); disable_work(&sc->recv_io.posted.refill_work); disable_work(&sc->idle.immediate_work); sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; disable_delayed_work(&sc->idle.timer_work);
/* * In case we were a listener we need to * disconnect all pending and ready sockets * * First we move ready sockets to pending again. * * Only a socket that was a listener (listen.backlog != -1) owns a * populated listen.ready/pending list. Guarding on that also keeps * lockdep quiet: without it, the listener holds sc->listen.lock while * the loop recurses into each child psc, which takes psc->listen.lock. * Those are always different instances of the same lock class -- a * child never listens, so the nesting is strictly listener -> child * and cannot really deadlock -- but lockdep only sees the class and * reports "possible recursive locking". A child has empty listen * lists and nothing to do here, so skipping it loses nothing, and a * pending child stays on its listener's list for the free path * (smbdirect_socket_destroy) to reap. */ if (sc->listen.backlog != -1) { /* was a listener */ /* * We don't move them to the orphaned list here, * each child does that itself at the end of its * smbdirect_socket_cleanup_work(), see * smbdirect_listen_orphan_socket(). Only that checks * accept.listener, which is still NULL for a child * that smbdirect_listen_connect_request() is still * setting up and will release itself on failure. */ spin_lock_irqsave(&sc->listen.lock, flags); list_splice_init(&sc->listen.ready, &sc->listen.pending); list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) smbdirect_socket_schedule_cleanup(psc, sc->first_error); spin_unlock_irqrestore(&sc->listen.lock, flags); }
switch (sc->status) { case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: case SMBDIRECT_SOCKET_ERROR: case SMBDIRECT_SOCKET_DISCONNECTING: case SMBDIRECT_SOCKET_DISCONNECTED: case SMBDIRECT_SOCKET_DESTROYED: /* * Keep the current error status */ break;
case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; break;
case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; break;
case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; break;
case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; break;
case SMBDIRECT_SOCKET_CREATED: case SMBDIRECT_SOCKET_LISTENING: sc->status = SMBDIRECT_SOCKET_DISCONNECTED; break;
case SMBDIRECT_SOCKET_CONNECTED: sc->status = SMBDIRECT_SOCKET_ERROR; break; }
if (force_status && (was_first || *force_status > sc->status)) sc->status = *force_status;
/* * Wake up all waiters in all wait queues * in order to notice the broken connection. */ smbdirect_socket_wake_up_all(sc);
queue_work(sc->workqueues.cleanup, &sc->disconnect_work);}
static void smbdirect_socket_cleanup_work(struct work_struct *work){ struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, disconnect_work); struct smbdirect_socket *psc, *tsc; unsigned long flags;
/* * This should not never be called in an interrupt! */ WARN_ON_ONCE(in_interrupt());
if (!sc->first_error) { smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, "%s called with first_error==0\n", smbdirect_socket_status_string(sc->status));
sc->first_error = -ECONNABORTED; }
/* * make sure this and other work is not queued again * but here we don't block and avoid * disable[_delayed]_work_sync() */ disable_work(&sc->disconnect_work); disable_work(&sc->connect.work); disable_work(&sc->recv_io.posted.refill_work); disable_work(&sc->idle.immediate_work); sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; disable_delayed_work(&sc->idle.timer_work);
/* * In case we were a listener we need to * disconnect all pending and ready sockets * * First we move ready sockets to pending again. * * Guarded on listen.backlog != -1 for the same reason as in * __smbdirect_socket_schedule_cleanup(): only a listener owns a * populated listen list, and skipping the block for a child avoids * nesting psc->listen.lock under a listener's listen.lock (different * instances of one class -- harmless, but lockdep cannot tell). */ if (sc->listen.backlog != -1) { /* was a listener */ /* * We don't move them to the orphaned list here, * each child does that itself at the end of its * smbdirect_socket_cleanup_work(), see * smbdirect_listen_orphan_socket(). Only that checks * accept.listener, which is still NULL for a child * that smbdirect_listen_connect_request() is still * setting up and will release itself on failure. */ spin_lock_irqsave(&sc->listen.lock, flags); list_splice_init(&sc->listen.ready, &sc->listen.pending); list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) smbdirect_socket_schedule_cleanup(psc, sc->first_error); spin_unlock_irqrestore(&sc->listen.lock, flags); }
switch (sc->status) { case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: case SMBDIRECT_SOCKET_CONNECTED: case SMBDIRECT_SOCKET_ERROR: sc->status = SMBDIRECT_SOCKET_DISCONNECTING; /* * Make sure we hold the callback lock * im order to coordinate with the * rdma_event handlers, typically * smbdirect_connection_rdma_event_handler(), * and smbdirect_socket_destroy(). * * So that the order of ib_drain_qp() * and rdma_disconnect() is controlled * by the mutex. */ rdma_lock_handler(sc->rdma.cm_id); rdma_disconnect(sc->rdma.cm_id); rdma_unlock_handler(sc->rdma.cm_id); break;
case SMBDIRECT_SOCKET_CREATED: case SMBDIRECT_SOCKET_LISTENING: case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: /* * rdma_{accept,connect}() never reached * RDMA_CM_EVENT_ESTABLISHED */ sc->status = SMBDIRECT_SOCKET_DISCONNECTED; break;
case SMBDIRECT_SOCKET_DISCONNECTING: case SMBDIRECT_SOCKET_DISCONNECTED: case SMBDIRECT_SOCKET_DESTROYED: break; }
/* * Wake up all waiters in all wait queues * in order to notice the broken connection. */ smbdirect_socket_wake_up_all(sc);
/* * If we're still on the pending or ready list * of a listener, we started the disconnect * as far as possible above, so we move ourself * to the orphaned list of the listener, * which will release us. * * This is a no-op internally if * sc->accept.listener is NULL. */ smbdirect_listen_orphan_socket(sc);}
static void smbdirect_socket_destroy(struct smbdirect_socket *sc){ struct smbdirect_socket *psc, *tsc; size_t psockets; struct smbdirect_recv_io *recv_io; struct smbdirect_recv_io *recv_tmp; LIST_HEAD(all_list); LIST_HEAD(pending_list); unsigned long flags;
smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "status=%s first_error=%1pe", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
/* * This should not never be called in an interrupt! */ WARN_ON_ONCE(in_interrupt());
if (sc->status == SMBDIRECT_SOCKET_DESTROYED) return;
/* * smbdirect_socket_destroy_sync() doesn't wait * for RDMA_CM_EVENT_DISCONNECTED, so we may still * be in SMBDIRECT_SOCKET_DISCONNECTING * (or already reached SMBDIRECT_SOCKET_DISCONNECTED) */ WARN_ONCE(sc->status < SMBDIRECT_SOCKET_DISCONNECTING, "status=%s first_error=%1pe", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
/* * The listener should clear this before we reach this */ WARN_ONCE(sc->accept.listener, "status=%s first_error=%1pe", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
/* * Wake up all waiters in all wait queues * in order to notice the broken connection. * * Most likely this was already called via * smbdirect_socket_cleanup_work(), but call it again... */ smbdirect_socket_wake_up_all(sc);
disable_work_sync(&sc->disconnect_work); disable_work_sync(&sc->connect.work); disable_work_sync(&sc->recv_io.posted.refill_work); disable_work_sync(&sc->idle.immediate_work); disable_delayed_work_sync(&sc->idle.timer_work); disable_work_sync(&sc->listen.purge_orphaned_work);
if (sc->rdma.cm_id) rdma_lock_handler(sc->rdma.cm_id);
/* * We hold the handler lock, so the rdma event * handlers can't change the status anymore. * * If RDMA_CM_EVENT_DISCONNECTED didn't arrive yet, * we just stop waiting for it here. * * We already disabled disconnect_work above * and before we call rdma_unlock_handler below * we call smbdirect_connection_destroy_qp which * sets sc->ib.qp = NULL. * * Between rdma_unlock_handler() and * rdma_destroy_id() below there's a small * windows where RDMA_CM_EVENT_DISCONNECTED * could still arrive. * * But smbdirect_connection_rdma_event_handler * will be a noop when calling smbdirect_socket_schedule_cleanup* * ib_drain_qp() also won't be called. */ if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) { sc->status = SMBDIRECT_SOCKET_DISCONNECTED; smbdirect_socket_wake_up_all(sc); }
if (sc->ib.qp) { smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "drain qp\n"); ib_drain_qp(sc->ib.qp); }
/* * In case we were a listener we need to * disconnect all pending and ready sockets * * We move ready sockets to pending again. * * Capture them here -- rdma_lock_handler(sc->rdma.cm_id) is held above, * so a concurrent CM CONNECT_REQUEST cannot add more; sc->listen.lock * below only protects the list splice itself -- but DEFER releasing * them until the listener's cm_id is destroyed: * * - smbdirect_socket_release() -> smbdirect_socket_destroy() takes the * child's own rdma_lock_handler() lock (&id_priv->handler_mutex). * The listener's and the child's cm_id are always different * instances, so the nesting cannot really deadlock, but lockdep only * sees one lock class and reports "possible recursive locking". * * - rdma_destroy_id() of a child before the listener's own * rdma_destroy_id() below lets _cma_cancel_listens() walk the freed * child id_priv (KASAN slab-use-after-free in __mutex_lock()). * * The children are independent sockets whose teardown does not need * the listener's handler lock. */ spin_lock_irqsave(&sc->listen.lock, flags); list_splice_tail_init(&sc->listen.ready, &pending_list); list_splice_tail_init(&sc->listen.pending, &pending_list); /* * purge_orphaned_work is already disabled above, * so we also need to release the orphaned sockets. * * Clearing accept.listener under listen.lock * makes us responsible for releasing them and * prevents them from moving themselves to * the orphaned list via * smbdirect_listen_orphan_socket(). */ list_splice_tail_init(&sc->listen.orphaned, &pending_list); list_for_each_entry(psc, &pending_list, accept.list) WRITE_ONCE(psc->accept.listener, NULL); spin_unlock_irqrestore(&sc->listen.lock, flags);
/* It's not possible for upper layer to get to reassembly */ if (sc->listen.backlog == -1) /* was not a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "drain the reassembly queue\n"); spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); list_splice_tail_init(&sc->recv_io.reassembly.list, &all_list); spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); list_for_each_entry_safe(recv_io, recv_tmp, &all_list, list) smbdirect_connection_put_recv_io(recv_io); sc->recv_io.reassembly.data_length = 0;
if (sc->listen.backlog == -1) /* was not a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "freeing mr list\n"); smbdirect_connection_destroy_mr_list(sc);
if (sc->listen.backlog == -1) /* was not a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "destroying qp\n"); smbdirect_connection_destroy_qp(sc); if (sc->rdma.cm_id) { rdma_unlock_handler(sc->rdma.cm_id); smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "destroying cm_id\n"); rdma_destroy_id(sc->rdma.cm_id); sc->rdma.cm_id = NULL; }
/* * The listener's rdma_lock_handler() lock is dropped and its cm_id is * destroyed, so it is safe to release the child sockets captured * above: each release recurses into smbdirect_socket_destroy() and * takes that child's own handler_mutex without nesting it under the * listener's, and _cma_cancel_listens() can no longer reach them. */ psockets = list_count_nodes(&pending_list); if (sc->listen.backlog != -1) /* was a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "release %zu pending sockets\n", psockets); list_for_each_entry_safe(psc, tsc, &pending_list, accept.list) { list_del_init(&psc->accept.list); smbdirect_socket_release(psc); } if (sc->listen.backlog != -1) /* was a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "released %zu pending sockets\n", psockets);
if (sc->listen.backlog == -1) /* was not a listener */ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "destroying mem pools\n"); smbdirect_connection_destroy_mem_pools(sc);
sc->status = SMBDIRECT_SOCKET_DESTROYED;
smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "rdma session destroyed\n");}
void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc){ smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "status=%s first_error=%1pe", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));
/* * This should not never be called in an interrupt! */ WARN_ON_ONCE(in_interrupt());
/* * First we try to disable the work * without disable_work_sync() in a * non blocking way, if it's already * running it will be handles by * disable_work_sync() below. * * Here we just want to make sure queue_work() in * smbdirect_socket_schedule_cleanup_lvl() * is a no-op. */ disable_work(&sc->disconnect_work);
if (!sc->first_error) /* * SMBDIRECT_LOG_INFO is enough here * as this is the typical case where * we terminate the connection ourself. */ smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN);
smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "cancelling and disable disconnect_work\n"); disable_work_sync(&sc->disconnect_work);
smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "destroying rdma session\n"); if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) smbdirect_socket_cleanup_work(&sc->disconnect_work);
/* * We don't wait for RDMA_CM_EVENT_DISCONNECTED, * rdma_disconnect() was already called by * smbdirect_socket_cleanup_work() if needed * and smbdirect_socket_destroy() drains the qp, * destroys it and calls rdma_destroy_id(), which * only waits for a currently running event handler * and makes sure no further events are delivered. * The rest of the disconnect protocol is handled * by the rdma core asynchronously. * * Waiting for RDMA_CM_EVENT_DISCONNECTED could * take very long or forever, e.g. if the peer * just disappeared. */ smbdirect_socket_destroy(sc); smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, "status=%s first_error=%1pe", smbdirect_socket_status_string(sc->status), SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));}
int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr){ int ret;
if (sc->status != SMBDIRECT_SOCKET_CREATED) return -EINVAL;
ret = rdma_bind_addr(sc->rdma.cm_id, addr); if (ret) return ret;
return 0;}EXPORT_SYMBOL_GPL(smbdirect_socket_bind);
void smbdirect_socket_shutdown(struct smbdirect_socket *sc){ smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN);}EXPORT_SYMBOL_GPL(smbdirect_socket_shutdown);
static void smbdirect_socket_release_disconnect(struct kref *kref){ struct smbdirect_socket *sc = container_of(kref, struct smbdirect_socket, refs.disconnect);
/* * For now do a sync disconnect/destroy */ smbdirect_socket_destroy_sync(sc);}
static void smbdirect_socket_release_destroy(struct kref *kref){ struct smbdirect_socket *sc = container_of(kref, struct smbdirect_socket, refs.destroy);
/* * Do a sync disconnect/destroy... * hopefully a no-op, as it should be already * in DESTROYED state, before we free the memory. */ smbdirect_socket_destroy_sync(sc);
/* * Only a listener (backlog != -1) is ever dereferenced * via sc->accept.listener under rcu_read_lock(), see * smbdirect_listen_orphan_socket(). Other sockets can be * freed immediately. */ if (sc->listen.backlog != -1) /* was a listener */ kfree_rcu(sc, refs.rcu); else kfree(sc);}
void smbdirect_socket_release(struct smbdirect_socket *sc){ /* * We expect only 1 disconnect reference * and if it is already 0, it's a use after free! */ WARN_ON_ONCE(kref_read(&sc->refs.disconnect) != 1); WARN_ON(!kref_put(&sc->refs.disconnect, smbdirect_socket_release_disconnect));
/* * This may not trigger smbdirect_socket_release_destroy(), * if struct smbdirect_socket is embedded in another structure * indicated by REFCOUNT_MAX. */ kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy);}EXPORT_SYMBOL_GPL(smbdirect_socket_release);
int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, enum smbdirect_socket_status expected_status, int unexpected_errno, wait_queue_head_t *waitq, atomic_t *total_credits, int needed){ int ret;
if (WARN_ON_ONCE(needed < 0)) return -EINVAL;
do { if (atomic_sub_return(needed, total_credits) >= 0) return 0;
atomic_add(needed, total_credits); ret = wait_event_interruptible(*waitq, atomic_read(total_credits) >= needed || sc->status != expected_status);
if (sc->status != expected_status) return unexpected_errno; else if (ret < 0) return ret; } while (true);}