2 * Copyright (c) 2005 Voltaire Inc. All rights reserved.
3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/completion.h>
38 #include <linux/in6.h>
39 #include <linux/mutex.h>
40 #include <linux/random.h>
41 #include <linux/idr.h>
42 #include <linux/inetdevice.h>
47 #include <rdma/rdma_cm.h>
48 #include <rdma/rdma_cm_ib.h>
49 #include <rdma/ib_cache.h>
50 #include <rdma/ib_cm.h>
51 #include <rdma/ib_sa.h>
52 #include <rdma/iw_cm.h>
54 MODULE_AUTHOR("Sean Hefty");
55 MODULE_DESCRIPTION("Generic RDMA CM Agent");
56 MODULE_LICENSE("Dual BSD/GPL");
58 #define CMA_CM_RESPONSE_TIMEOUT 20
59 #define CMA_MAX_CM_RETRIES 15
60 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
62 static void cma_add_one(struct ib_device *device);
63 static void cma_remove_one(struct ib_device *device);
65 static struct ib_client cma_client = {
68 .remove = cma_remove_one
71 static struct ib_sa_client sa_client;
72 static struct rdma_addr_client addr_client;
73 static LIST_HEAD(dev_list);
74 static LIST_HEAD(listen_any_list);
75 static DEFINE_MUTEX(lock);
76 static struct workqueue_struct *cma_wq;
77 static DEFINE_IDR(sdp_ps);
78 static DEFINE_IDR(tcp_ps);
79 static DEFINE_IDR(udp_ps);
80 static DEFINE_IDR(ipoib_ps);
84 struct list_head list;
85 struct ib_device *device;
86 struct completion comp;
88 struct list_head id_list;
105 struct rdma_bind_list {
107 struct hlist_head owners;
112 * Device removal can occur at anytime, so we need extra handling to
113 * serialize notifying the user of device removal with other callbacks.
114 * We do this by disabling removal notification while a callback is in process,
115 * and reporting it after the callback completes.
117 struct rdma_id_private {
118 struct rdma_cm_id id;
120 struct rdma_bind_list *bind_list;
121 struct hlist_node node;
122 struct list_head list; /* listen_any_list or cma_device.list */
123 struct list_head listen_list; /* per device listens */
124 struct cma_device *cma_dev;
125 struct list_head mc_list;
128 enum cma_state state;
130 struct mutex qp_mutex;
132 struct completion comp;
134 struct mutex handler_mutex;
138 struct ib_sa_query *query;
152 struct cma_multicast {
153 struct rdma_id_private *id_priv;
155 struct ib_sa_multicast *ib;
157 struct list_head list;
159 struct sockaddr_storage addr;
163 struct work_struct work;
164 struct rdma_id_private *id;
165 enum cma_state old_state;
166 enum cma_state new_state;
167 struct rdma_cm_event event;
170 struct cma_ndev_work {
171 struct work_struct work;
172 struct rdma_id_private *id;
173 struct rdma_cm_event event;
186 u8 ip_version; /* IP version: 7:4 */
188 union cma_ip_addr src_addr;
189 union cma_ip_addr dst_addr;
194 u8 sdp_version; /* Major version: 7:4 */
195 u8 ip_version; /* IP version: 7:4 */
196 u8 sdp_specific1[10];
198 __be16 sdp_specific2;
199 union cma_ip_addr src_addr;
200 union cma_ip_addr dst_addr;
208 #define CMA_VERSION 0x00
209 #define SDP_MAJ_VERSION 0x2
211 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp)
216 spin_lock_irqsave(&id_priv->lock, flags);
217 ret = (id_priv->state == comp);
218 spin_unlock_irqrestore(&id_priv->lock, flags);
222 static int cma_comp_exch(struct rdma_id_private *id_priv,
223 enum cma_state comp, enum cma_state exch)
228 spin_lock_irqsave(&id_priv->lock, flags);
229 if ((ret = (id_priv->state == comp)))
230 id_priv->state = exch;
231 spin_unlock_irqrestore(&id_priv->lock, flags);
235 static enum cma_state cma_exch(struct rdma_id_private *id_priv,
241 spin_lock_irqsave(&id_priv->lock, flags);
242 old = id_priv->state;
243 id_priv->state = exch;
244 spin_unlock_irqrestore(&id_priv->lock, flags);
248 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
250 return hdr->ip_version >> 4;
253 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
255 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
258 static inline u8 sdp_get_majv(u8 sdp_version)
260 return sdp_version >> 4;
263 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
265 return hh->ip_version >> 4;
268 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
270 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
273 static inline int cma_is_ud_ps(enum rdma_port_space ps)
275 return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
278 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
279 struct cma_device *cma_dev)
281 atomic_inc(&cma_dev->refcount);
282 id_priv->cma_dev = cma_dev;
283 id_priv->id.device = cma_dev->device;
284 list_add_tail(&id_priv->list, &cma_dev->id_list);
287 static inline void cma_deref_dev(struct cma_device *cma_dev)
289 if (atomic_dec_and_test(&cma_dev->refcount))
290 complete(&cma_dev->comp);
293 static void cma_detach_from_dev(struct rdma_id_private *id_priv)
295 list_del(&id_priv->list);
296 cma_deref_dev(id_priv->cma_dev);
297 id_priv->cma_dev = NULL;
300 static int cma_set_qkey(struct rdma_id_private *id_priv)
302 struct ib_sa_mcmember_rec rec;
308 switch (id_priv->id.ps) {
310 id_priv->qkey = RDMA_UDP_QKEY;
313 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
314 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
315 id_priv->id.port_num, &rec.mgid,
318 id_priv->qkey = be32_to_cpu(rec.qkey);
326 static int cma_acquire_dev(struct rdma_id_private *id_priv)
328 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
329 struct cma_device *cma_dev;
333 switch (rdma_node_get_transport(dev_addr->dev_type)) {
334 case RDMA_TRANSPORT_IB:
335 ib_addr_get_sgid(dev_addr, &gid);
337 case RDMA_TRANSPORT_IWARP:
338 iw_addr_get_sgid(dev_addr, &gid);
344 list_for_each_entry(cma_dev, &dev_list, list) {
345 ret = ib_find_cached_gid(cma_dev->device, &gid,
346 &id_priv->id.port_num, NULL);
348 cma_attach_to_dev(id_priv, cma_dev);
355 static void cma_deref_id(struct rdma_id_private *id_priv)
357 if (atomic_dec_and_test(&id_priv->refcount))
358 complete(&id_priv->comp);
361 static int cma_disable_callback(struct rdma_id_private *id_priv,
362 enum cma_state state)
364 mutex_lock(&id_priv->handler_mutex);
365 if (id_priv->state != state) {
366 mutex_unlock(&id_priv->handler_mutex);
372 static int cma_has_cm_dev(struct rdma_id_private *id_priv)
374 return (id_priv->id.device && id_priv->cm_id.ib);
377 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
378 void *context, enum rdma_port_space ps)
380 struct rdma_id_private *id_priv;
382 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
384 return ERR_PTR(-ENOMEM);
386 id_priv->state = CMA_IDLE;
387 id_priv->id.context = context;
388 id_priv->id.event_handler = event_handler;
390 spin_lock_init(&id_priv->lock);
391 mutex_init(&id_priv->qp_mutex);
392 init_completion(&id_priv->comp);
393 atomic_set(&id_priv->refcount, 1);
394 mutex_init(&id_priv->handler_mutex);
395 INIT_LIST_HEAD(&id_priv->listen_list);
396 INIT_LIST_HEAD(&id_priv->mc_list);
397 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
401 EXPORT_SYMBOL(rdma_create_id);
403 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
405 struct ib_qp_attr qp_attr;
406 int qp_attr_mask, ret;
408 qp_attr.qp_state = IB_QPS_INIT;
409 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
413 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
417 qp_attr.qp_state = IB_QPS_RTR;
418 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
422 qp_attr.qp_state = IB_QPS_RTS;
424 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
429 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
431 struct ib_qp_attr qp_attr;
432 int qp_attr_mask, ret;
434 qp_attr.qp_state = IB_QPS_INIT;
435 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
439 return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
442 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
443 struct ib_qp_init_attr *qp_init_attr)
445 struct rdma_id_private *id_priv;
449 id_priv = container_of(id, struct rdma_id_private, id);
450 if (id->device != pd->device)
453 qp = ib_create_qp(pd, qp_init_attr);
457 if (cma_is_ud_ps(id_priv->id.ps))
458 ret = cma_init_ud_qp(id_priv, qp);
460 ret = cma_init_conn_qp(id_priv, qp);
465 id_priv->qp_num = qp->qp_num;
466 id_priv->srq = (qp->srq != NULL);
472 EXPORT_SYMBOL(rdma_create_qp);
474 void rdma_destroy_qp(struct rdma_cm_id *id)
476 struct rdma_id_private *id_priv;
478 id_priv = container_of(id, struct rdma_id_private, id);
479 mutex_lock(&id_priv->qp_mutex);
480 ib_destroy_qp(id_priv->id.qp);
481 id_priv->id.qp = NULL;
482 mutex_unlock(&id_priv->qp_mutex);
484 EXPORT_SYMBOL(rdma_destroy_qp);
486 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
487 struct rdma_conn_param *conn_param)
489 struct ib_qp_attr qp_attr;
490 int qp_attr_mask, ret;
492 mutex_lock(&id_priv->qp_mutex);
493 if (!id_priv->id.qp) {
498 /* Need to update QP attributes from default values. */
499 qp_attr.qp_state = IB_QPS_INIT;
500 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
504 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
508 qp_attr.qp_state = IB_QPS_RTR;
509 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
514 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
515 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
517 mutex_unlock(&id_priv->qp_mutex);
521 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
522 struct rdma_conn_param *conn_param)
524 struct ib_qp_attr qp_attr;
525 int qp_attr_mask, ret;
527 mutex_lock(&id_priv->qp_mutex);
528 if (!id_priv->id.qp) {
533 qp_attr.qp_state = IB_QPS_RTS;
534 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
539 qp_attr.max_rd_atomic = conn_param->initiator_depth;
540 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
542 mutex_unlock(&id_priv->qp_mutex);
546 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
548 struct ib_qp_attr qp_attr;
551 mutex_lock(&id_priv->qp_mutex);
552 if (!id_priv->id.qp) {
557 qp_attr.qp_state = IB_QPS_ERR;
558 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
560 mutex_unlock(&id_priv->qp_mutex);
564 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
565 struct ib_qp_attr *qp_attr, int *qp_attr_mask)
567 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
570 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
571 ib_addr_get_pkey(dev_addr),
572 &qp_attr->pkey_index);
576 qp_attr->port_num = id_priv->id.port_num;
577 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
579 if (cma_is_ud_ps(id_priv->id.ps)) {
580 ret = cma_set_qkey(id_priv);
584 qp_attr->qkey = id_priv->qkey;
585 *qp_attr_mask |= IB_QP_QKEY;
587 qp_attr->qp_access_flags = 0;
588 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
593 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
596 struct rdma_id_private *id_priv;
599 id_priv = container_of(id, struct rdma_id_private, id);
600 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
601 case RDMA_TRANSPORT_IB:
602 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps))
603 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
605 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
607 if (qp_attr->qp_state == IB_QPS_RTR)
608 qp_attr->rq_psn = id_priv->seq_num;
610 case RDMA_TRANSPORT_IWARP:
611 if (!id_priv->cm_id.iw) {
612 qp_attr->qp_access_flags = 0;
613 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
615 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
625 EXPORT_SYMBOL(rdma_init_qp_attr);
627 static inline int cma_zero_addr(struct sockaddr *addr)
629 struct in6_addr *ip6;
631 if (addr->sa_family == AF_INET)
632 return ipv4_is_zeronet(
633 ((struct sockaddr_in *)addr)->sin_addr.s_addr);
635 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
636 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
637 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
641 static inline int cma_loopback_addr(struct sockaddr *addr)
643 if (addr->sa_family == AF_INET)
644 return ipv4_is_loopback(
645 ((struct sockaddr_in *) addr)->sin_addr.s_addr);
647 return ipv6_addr_loopback(
648 &((struct sockaddr_in6 *) addr)->sin6_addr);
651 static inline int cma_any_addr(struct sockaddr *addr)
653 return cma_zero_addr(addr) || cma_loopback_addr(addr);
656 static inline __be16 cma_port(struct sockaddr *addr)
658 if (addr->sa_family == AF_INET)
659 return ((struct sockaddr_in *) addr)->sin_port;
661 return ((struct sockaddr_in6 *) addr)->sin6_port;
664 static inline int cma_any_port(struct sockaddr *addr)
666 return !cma_port(addr);
669 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
670 u8 *ip_ver, __be16 *port,
671 union cma_ip_addr **src, union cma_ip_addr **dst)
675 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
679 *ip_ver = sdp_get_ip_ver(hdr);
680 *port = ((struct sdp_hh *) hdr)->port;
681 *src = &((struct sdp_hh *) hdr)->src_addr;
682 *dst = &((struct sdp_hh *) hdr)->dst_addr;
685 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
688 *ip_ver = cma_get_ip_ver(hdr);
689 *port = ((struct cma_hdr *) hdr)->port;
690 *src = &((struct cma_hdr *) hdr)->src_addr;
691 *dst = &((struct cma_hdr *) hdr)->dst_addr;
695 if (*ip_ver != 4 && *ip_ver != 6)
700 static void cma_save_net_info(struct rdma_addr *addr,
701 struct rdma_addr *listen_addr,
702 u8 ip_ver, __be16 port,
703 union cma_ip_addr *src, union cma_ip_addr *dst)
705 struct sockaddr_in *listen4, *ip4;
706 struct sockaddr_in6 *listen6, *ip6;
710 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
711 ip4 = (struct sockaddr_in *) &addr->src_addr;
712 ip4->sin_family = listen4->sin_family;
713 ip4->sin_addr.s_addr = dst->ip4.addr;
714 ip4->sin_port = listen4->sin_port;
716 ip4 = (struct sockaddr_in *) &addr->dst_addr;
717 ip4->sin_family = listen4->sin_family;
718 ip4->sin_addr.s_addr = src->ip4.addr;
719 ip4->sin_port = port;
722 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
723 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
724 ip6->sin6_family = listen6->sin6_family;
725 ip6->sin6_addr = dst->ip6;
726 ip6->sin6_port = listen6->sin6_port;
728 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
729 ip6->sin6_family = listen6->sin6_family;
730 ip6->sin6_addr = src->ip6;
731 ip6->sin6_port = port;
738 static inline int cma_user_data_offset(enum rdma_port_space ps)
744 return sizeof(struct cma_hdr);
748 static void cma_cancel_route(struct rdma_id_private *id_priv)
750 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
751 case RDMA_TRANSPORT_IB:
753 ib_sa_cancel_query(id_priv->query_id, id_priv->query);
760 static void cma_cancel_listens(struct rdma_id_private *id_priv)
762 struct rdma_id_private *dev_id_priv;
765 * Remove from listen_any_list to prevent added devices from spawning
766 * additional listen requests.
769 list_del(&id_priv->list);
771 while (!list_empty(&id_priv->listen_list)) {
772 dev_id_priv = list_entry(id_priv->listen_list.next,
773 struct rdma_id_private, listen_list);
774 /* sync with device removal to avoid duplicate destruction */
775 list_del_init(&dev_id_priv->list);
776 list_del(&dev_id_priv->listen_list);
779 rdma_destroy_id(&dev_id_priv->id);
785 static void cma_cancel_operation(struct rdma_id_private *id_priv,
786 enum cma_state state)
790 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
792 case CMA_ROUTE_QUERY:
793 cma_cancel_route(id_priv);
796 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
797 && !id_priv->cma_dev)
798 cma_cancel_listens(id_priv);
805 static void cma_release_port(struct rdma_id_private *id_priv)
807 struct rdma_bind_list *bind_list = id_priv->bind_list;
813 hlist_del(&id_priv->node);
814 if (hlist_empty(&bind_list->owners)) {
815 idr_remove(bind_list->ps, bind_list->port);
821 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
823 struct cma_multicast *mc;
825 while (!list_empty(&id_priv->mc_list)) {
826 mc = container_of(id_priv->mc_list.next,
827 struct cma_multicast, list);
829 ib_sa_free_multicast(mc->multicast.ib);
834 void rdma_destroy_id(struct rdma_cm_id *id)
836 struct rdma_id_private *id_priv;
837 enum cma_state state;
839 id_priv = container_of(id, struct rdma_id_private, id);
840 state = cma_exch(id_priv, CMA_DESTROYING);
841 cma_cancel_operation(id_priv, state);
844 if (id_priv->cma_dev) {
846 switch (rdma_node_get_transport(id->device->node_type)) {
847 case RDMA_TRANSPORT_IB:
848 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
849 ib_destroy_cm_id(id_priv->cm_id.ib);
851 case RDMA_TRANSPORT_IWARP:
852 if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
853 iw_destroy_cm_id(id_priv->cm_id.iw);
858 cma_leave_mc_groups(id_priv);
860 cma_detach_from_dev(id_priv);
864 cma_release_port(id_priv);
865 cma_deref_id(id_priv);
866 wait_for_completion(&id_priv->comp);
868 if (id_priv->internal_id)
869 cma_deref_id(id_priv->id.context);
871 kfree(id_priv->id.route.path_rec);
874 EXPORT_SYMBOL(rdma_destroy_id);
876 static int cma_rep_recv(struct rdma_id_private *id_priv)
880 ret = cma_modify_qp_rtr(id_priv, NULL);
884 ret = cma_modify_qp_rts(id_priv, NULL);
888 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
894 cma_modify_qp_err(id_priv);
895 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
900 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
902 if (id_priv->id.ps == RDMA_PS_SDP &&
903 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
910 static void cma_set_rep_event_data(struct rdma_cm_event *event,
911 struct ib_cm_rep_event_param *rep_data,
914 event->param.conn.private_data = private_data;
915 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
916 event->param.conn.responder_resources = rep_data->responder_resources;
917 event->param.conn.initiator_depth = rep_data->initiator_depth;
918 event->param.conn.flow_control = rep_data->flow_control;
919 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
920 event->param.conn.srq = rep_data->srq;
921 event->param.conn.qp_num = rep_data->remote_qpn;
924 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
926 struct rdma_id_private *id_priv = cm_id->context;
927 struct rdma_cm_event event;
930 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
931 cma_disable_callback(id_priv, CMA_CONNECT)) ||
932 (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
933 cma_disable_callback(id_priv, CMA_DISCONNECT)))
936 memset(&event, 0, sizeof event);
937 switch (ib_event->event) {
938 case IB_CM_REQ_ERROR:
939 case IB_CM_REP_ERROR:
940 event.event = RDMA_CM_EVENT_UNREACHABLE;
941 event.status = -ETIMEDOUT;
943 case IB_CM_REP_RECEIVED:
944 event.status = cma_verify_rep(id_priv, ib_event->private_data);
946 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
947 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
948 event.status = cma_rep_recv(id_priv);
949 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
950 RDMA_CM_EVENT_ESTABLISHED;
952 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
953 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
954 ib_event->private_data);
956 case IB_CM_RTU_RECEIVED:
957 case IB_CM_USER_ESTABLISHED:
958 event.event = RDMA_CM_EVENT_ESTABLISHED;
960 case IB_CM_DREQ_ERROR:
961 event.status = -ETIMEDOUT; /* fall through */
962 case IB_CM_DREQ_RECEIVED:
963 case IB_CM_DREP_RECEIVED:
964 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT))
966 event.event = RDMA_CM_EVENT_DISCONNECTED;
968 case IB_CM_TIMEWAIT_EXIT:
969 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
971 case IB_CM_MRA_RECEIVED:
974 case IB_CM_REJ_RECEIVED:
975 cma_modify_qp_err(id_priv);
976 event.status = ib_event->param.rej_rcvd.reason;
977 event.event = RDMA_CM_EVENT_REJECTED;
978 event.param.conn.private_data = ib_event->private_data;
979 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
982 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
987 ret = id_priv->id.event_handler(&id_priv->id, &event);
989 /* Destroy the CM ID by returning a non-zero value. */
990 id_priv->cm_id.ib = NULL;
991 cma_exch(id_priv, CMA_DESTROYING);
992 mutex_unlock(&id_priv->handler_mutex);
993 rdma_destroy_id(&id_priv->id);
997 mutex_unlock(&id_priv->handler_mutex);
1001 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1002 struct ib_cm_event *ib_event)
1004 struct rdma_id_private *id_priv;
1005 struct rdma_cm_id *id;
1006 struct rdma_route *rt;
1007 union cma_ip_addr *src, *dst;
1012 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1013 &ip_ver, &port, &src, &dst))
1016 id = rdma_create_id(listen_id->event_handler, listen_id->context,
1021 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1022 ip_ver, port, src, dst);
1025 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1026 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1031 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1032 if (rt->num_paths == 2)
1033 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1035 ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1036 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
1037 &id->route.addr.dev_addr);
1041 id_priv = container_of(id, struct rdma_id_private, id);
1042 id_priv->state = CMA_CONNECT;
1046 rdma_destroy_id(id);
1051 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1052 struct ib_cm_event *ib_event)
1054 struct rdma_id_private *id_priv;
1055 struct rdma_cm_id *id;
1056 union cma_ip_addr *src, *dst;
1061 id = rdma_create_id(listen_id->event_handler, listen_id->context,
1067 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1068 &ip_ver, &port, &src, &dst))
1071 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1072 ip_ver, port, src, dst);
1074 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
1075 &id->route.addr.dev_addr);
1079 id_priv = container_of(id, struct rdma_id_private, id);
1080 id_priv->state = CMA_CONNECT;
1083 rdma_destroy_id(id);
1087 static void cma_set_req_event_data(struct rdma_cm_event *event,
1088 struct ib_cm_req_event_param *req_data,
1089 void *private_data, int offset)
1091 event->param.conn.private_data = private_data + offset;
1092 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1093 event->param.conn.responder_resources = req_data->responder_resources;
1094 event->param.conn.initiator_depth = req_data->initiator_depth;
1095 event->param.conn.flow_control = req_data->flow_control;
1096 event->param.conn.retry_count = req_data->retry_count;
1097 event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1098 event->param.conn.srq = req_data->srq;
1099 event->param.conn.qp_num = req_data->remote_qpn;
1102 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1104 struct rdma_id_private *listen_id, *conn_id;
1105 struct rdma_cm_event event;
1108 listen_id = cm_id->context;
1109 if (cma_disable_callback(listen_id, CMA_LISTEN))
1110 return -ECONNABORTED;
1112 memset(&event, 0, sizeof event);
1113 offset = cma_user_data_offset(listen_id->id.ps);
1114 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1115 if (cma_is_ud_ps(listen_id->id.ps)) {
1116 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1117 event.param.ud.private_data = ib_event->private_data + offset;
1118 event.param.ud.private_data_len =
1119 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1121 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1122 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1123 ib_event->private_data, offset);
1130 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1132 ret = cma_acquire_dev(conn_id);
1133 mutex_unlock(&lock);
1135 goto release_conn_id;
1137 conn_id->cm_id.ib = cm_id;
1138 cm_id->context = conn_id;
1139 cm_id->cm_handler = cma_ib_handler;
1142 * Protect against the user destroying conn_id from another thread
1143 * until we're done accessing it.
1145 atomic_inc(&conn_id->refcount);
1146 ret = conn_id->id.event_handler(&conn_id->id, &event);
1149 * Acquire mutex to prevent user executing rdma_destroy_id()
1150 * while we're accessing the cm_id.
1153 if (cma_comp(conn_id, CMA_CONNECT) &&
1154 !cma_is_ud_ps(conn_id->id.ps))
1155 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1156 mutex_unlock(&lock);
1157 mutex_unlock(&conn_id->handler_mutex);
1158 cma_deref_id(conn_id);
1161 cma_deref_id(conn_id);
1163 /* Destroy the CM ID by returning a non-zero value. */
1164 conn_id->cm_id.ib = NULL;
1167 cma_exch(conn_id, CMA_DESTROYING);
1168 mutex_unlock(&conn_id->handler_mutex);
1169 rdma_destroy_id(&conn_id->id);
1172 mutex_unlock(&listen_id->handler_mutex);
1176 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1178 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1181 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1182 struct ib_cm_compare_data *compare)
1184 struct cma_hdr *cma_data, *cma_mask;
1185 struct sdp_hh *sdp_data, *sdp_mask;
1187 struct in6_addr ip6_addr;
1189 memset(compare, 0, sizeof *compare);
1190 cma_data = (void *) compare->data;
1191 cma_mask = (void *) compare->mask;
1192 sdp_data = (void *) compare->data;
1193 sdp_mask = (void *) compare->mask;
1195 switch (addr->sa_family) {
1197 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1198 if (ps == RDMA_PS_SDP) {
1199 sdp_set_ip_ver(sdp_data, 4);
1200 sdp_set_ip_ver(sdp_mask, 0xF);
1201 sdp_data->dst_addr.ip4.addr = ip4_addr;
1202 sdp_mask->dst_addr.ip4.addr = htonl(~0);
1204 cma_set_ip_ver(cma_data, 4);
1205 cma_set_ip_ver(cma_mask, 0xF);
1206 cma_data->dst_addr.ip4.addr = ip4_addr;
1207 cma_mask->dst_addr.ip4.addr = htonl(~0);
1211 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1212 if (ps == RDMA_PS_SDP) {
1213 sdp_set_ip_ver(sdp_data, 6);
1214 sdp_set_ip_ver(sdp_mask, 0xF);
1215 sdp_data->dst_addr.ip6 = ip6_addr;
1216 memset(&sdp_mask->dst_addr.ip6, 0xFF,
1217 sizeof sdp_mask->dst_addr.ip6);
1219 cma_set_ip_ver(cma_data, 6);
1220 cma_set_ip_ver(cma_mask, 0xF);
1221 cma_data->dst_addr.ip6 = ip6_addr;
1222 memset(&cma_mask->dst_addr.ip6, 0xFF,
1223 sizeof cma_mask->dst_addr.ip6);
1231 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1233 struct rdma_id_private *id_priv = iw_id->context;
1234 struct rdma_cm_event event;
1235 struct sockaddr_in *sin;
1238 if (cma_disable_callback(id_priv, CMA_CONNECT))
1241 memset(&event, 0, sizeof event);
1242 switch (iw_event->event) {
1243 case IW_CM_EVENT_CLOSE:
1244 event.event = RDMA_CM_EVENT_DISCONNECTED;
1246 case IW_CM_EVENT_CONNECT_REPLY:
1247 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1248 *sin = iw_event->local_addr;
1249 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1250 *sin = iw_event->remote_addr;
1251 switch (iw_event->status) {
1253 event.event = RDMA_CM_EVENT_ESTABLISHED;
1257 event.event = RDMA_CM_EVENT_REJECTED;
1260 event.event = RDMA_CM_EVENT_UNREACHABLE;
1263 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1267 case IW_CM_EVENT_ESTABLISHED:
1268 event.event = RDMA_CM_EVENT_ESTABLISHED;
1274 event.status = iw_event->status;
1275 event.param.conn.private_data = iw_event->private_data;
1276 event.param.conn.private_data_len = iw_event->private_data_len;
1277 ret = id_priv->id.event_handler(&id_priv->id, &event);
1279 /* Destroy the CM ID by returning a non-zero value. */
1280 id_priv->cm_id.iw = NULL;
1281 cma_exch(id_priv, CMA_DESTROYING);
1282 mutex_unlock(&id_priv->handler_mutex);
1283 rdma_destroy_id(&id_priv->id);
1287 mutex_unlock(&id_priv->handler_mutex);
1291 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1292 struct iw_cm_event *iw_event)
1294 struct rdma_cm_id *new_cm_id;
1295 struct rdma_id_private *listen_id, *conn_id;
1296 struct sockaddr_in *sin;
1297 struct net_device *dev = NULL;
1298 struct rdma_cm_event event;
1300 struct ib_device_attr attr;
1302 listen_id = cm_id->context;
1303 if (cma_disable_callback(listen_id, CMA_LISTEN))
1304 return -ECONNABORTED;
1306 /* Create a new RDMA id for the new IW CM ID */
1307 new_cm_id = rdma_create_id(listen_id->id.event_handler,
1308 listen_id->id.context,
1310 if (IS_ERR(new_cm_id)) {
1314 conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1315 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1316 conn_id->state = CMA_CONNECT;
1318 dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1320 ret = -EADDRNOTAVAIL;
1321 mutex_unlock(&conn_id->handler_mutex);
1322 rdma_destroy_id(new_cm_id);
1325 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1327 mutex_unlock(&conn_id->handler_mutex);
1328 rdma_destroy_id(new_cm_id);
1333 ret = cma_acquire_dev(conn_id);
1334 mutex_unlock(&lock);
1336 mutex_unlock(&conn_id->handler_mutex);
1337 rdma_destroy_id(new_cm_id);
1341 conn_id->cm_id.iw = cm_id;
1342 cm_id->context = conn_id;
1343 cm_id->cm_handler = cma_iw_handler;
1345 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1346 *sin = iw_event->local_addr;
1347 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1348 *sin = iw_event->remote_addr;
1350 ret = ib_query_device(conn_id->id.device, &attr);
1352 mutex_unlock(&conn_id->handler_mutex);
1353 rdma_destroy_id(new_cm_id);
1357 memset(&event, 0, sizeof event);
1358 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1359 event.param.conn.private_data = iw_event->private_data;
1360 event.param.conn.private_data_len = iw_event->private_data_len;
1361 event.param.conn.initiator_depth = attr.max_qp_init_rd_atom;
1362 event.param.conn.responder_resources = attr.max_qp_rd_atom;
1365 * Protect against the user destroying conn_id from another thread
1366 * until we're done accessing it.
1368 atomic_inc(&conn_id->refcount);
1369 ret = conn_id->id.event_handler(&conn_id->id, &event);
1371 /* User wants to destroy the CM ID */
1372 conn_id->cm_id.iw = NULL;
1373 cma_exch(conn_id, CMA_DESTROYING);
1374 mutex_unlock(&conn_id->handler_mutex);
1375 cma_deref_id(conn_id);
1376 rdma_destroy_id(&conn_id->id);
1380 mutex_unlock(&conn_id->handler_mutex);
1381 cma_deref_id(conn_id);
1386 mutex_unlock(&listen_id->handler_mutex);
1390 static int cma_ib_listen(struct rdma_id_private *id_priv)
1392 struct ib_cm_compare_data compare_data;
1393 struct sockaddr *addr;
1397 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler,
1399 if (IS_ERR(id_priv->cm_id.ib))
1400 return PTR_ERR(id_priv->cm_id.ib);
1402 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
1403 svc_id = cma_get_service_id(id_priv->id.ps, addr);
1404 if (cma_any_addr(addr))
1405 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1407 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1408 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1412 ib_destroy_cm_id(id_priv->cm_id.ib);
1413 id_priv->cm_id.ib = NULL;
1419 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1422 struct sockaddr_in *sin;
1424 id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
1425 iw_conn_req_handler,
1427 if (IS_ERR(id_priv->cm_id.iw))
1428 return PTR_ERR(id_priv->cm_id.iw);
1430 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1431 id_priv->cm_id.iw->local_addr = *sin;
1433 ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1436 iw_destroy_cm_id(id_priv->cm_id.iw);
1437 id_priv->cm_id.iw = NULL;
1443 static int cma_listen_handler(struct rdma_cm_id *id,
1444 struct rdma_cm_event *event)
1446 struct rdma_id_private *id_priv = id->context;
1448 id->context = id_priv->id.context;
1449 id->event_handler = id_priv->id.event_handler;
1450 return id_priv->id.event_handler(id, event);
1453 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1454 struct cma_device *cma_dev)
1456 struct rdma_id_private *dev_id_priv;
1457 struct rdma_cm_id *id;
1460 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps);
1464 dev_id_priv = container_of(id, struct rdma_id_private, id);
1466 dev_id_priv->state = CMA_ADDR_BOUND;
1467 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1468 ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
1470 cma_attach_to_dev(dev_id_priv, cma_dev);
1471 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1472 atomic_inc(&id_priv->refcount);
1473 dev_id_priv->internal_id = 1;
1475 ret = rdma_listen(id, id_priv->backlog);
1477 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1478 "listening on device %s\n", ret, cma_dev->device->name);
1481 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1483 struct cma_device *cma_dev;
1486 list_add_tail(&id_priv->list, &listen_any_list);
1487 list_for_each_entry(cma_dev, &dev_list, list)
1488 cma_listen_on_dev(id_priv, cma_dev);
1489 mutex_unlock(&lock);
1492 static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af)
1494 struct sockaddr_storage addr_in;
1496 memset(&addr_in, 0, sizeof addr_in);
1497 addr_in.ss_family = af;
1498 return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
1501 int rdma_listen(struct rdma_cm_id *id, int backlog)
1503 struct rdma_id_private *id_priv;
1506 id_priv = container_of(id, struct rdma_id_private, id);
1507 if (id_priv->state == CMA_IDLE) {
1508 ret = cma_bind_any(id, AF_INET);
1513 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN))
1516 id_priv->backlog = backlog;
1518 switch (rdma_node_get_transport(id->device->node_type)) {
1519 case RDMA_TRANSPORT_IB:
1520 ret = cma_ib_listen(id_priv);
1524 case RDMA_TRANSPORT_IWARP:
1525 ret = cma_iw_listen(id_priv, backlog);
1534 cma_listen_on_all(id_priv);
1538 id_priv->backlog = 0;
1539 cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND);
1542 EXPORT_SYMBOL(rdma_listen);
1544 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1546 struct rdma_id_private *id_priv;
1548 id_priv = container_of(id, struct rdma_id_private, id);
1549 id_priv->tos = (u8) tos;
1551 EXPORT_SYMBOL(rdma_set_service_type);
1553 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1556 struct cma_work *work = context;
1557 struct rdma_route *route;
1559 route = &work->id->id.route;
1562 route->num_paths = 1;
1563 *route->path_rec = *path_rec;
1565 work->old_state = CMA_ROUTE_QUERY;
1566 work->new_state = CMA_ADDR_RESOLVED;
1567 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1568 work->event.status = status;
1571 queue_work(cma_wq, &work->work);
1574 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1575 struct cma_work *work)
1577 struct rdma_addr *addr = &id_priv->id.route.addr;
1578 struct ib_sa_path_rec path_rec;
1579 ib_sa_comp_mask comp_mask;
1580 struct sockaddr_in6 *sin6;
1582 memset(&path_rec, 0, sizeof path_rec);
1583 ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1584 ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1585 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1586 path_rec.numb_path = 1;
1587 path_rec.reversible = 1;
1588 path_rec.service_id = cma_get_service_id(id_priv->id.ps,
1589 (struct sockaddr *) &addr->dst_addr);
1591 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1592 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1593 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1595 if (addr->src_addr.ss_family == AF_INET) {
1596 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1597 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1599 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1600 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1601 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1604 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1605 id_priv->id.port_num, &path_rec,
1606 comp_mask, timeout_ms,
1607 GFP_KERNEL, cma_query_handler,
1608 work, &id_priv->query);
1610 return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1613 static void cma_work_handler(struct work_struct *_work)
1615 struct cma_work *work = container_of(_work, struct cma_work, work);
1616 struct rdma_id_private *id_priv = work->id;
1619 mutex_lock(&id_priv->handler_mutex);
1620 if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1623 if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1624 cma_exch(id_priv, CMA_DESTROYING);
1628 mutex_unlock(&id_priv->handler_mutex);
1629 cma_deref_id(id_priv);
1631 rdma_destroy_id(&id_priv->id);
1635 static void cma_ndev_work_handler(struct work_struct *_work)
1637 struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1638 struct rdma_id_private *id_priv = work->id;
1641 mutex_lock(&id_priv->handler_mutex);
1642 if (id_priv->state == CMA_DESTROYING ||
1643 id_priv->state == CMA_DEVICE_REMOVAL)
1646 if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1647 cma_exch(id_priv, CMA_DESTROYING);
1652 mutex_unlock(&id_priv->handler_mutex);
1653 cma_deref_id(id_priv);
1655 rdma_destroy_id(&id_priv->id);
1659 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1661 struct rdma_route *route = &id_priv->id.route;
1662 struct cma_work *work;
1665 work = kzalloc(sizeof *work, GFP_KERNEL);
1670 INIT_WORK(&work->work, cma_work_handler);
1671 work->old_state = CMA_ROUTE_QUERY;
1672 work->new_state = CMA_ROUTE_RESOLVED;
1673 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1675 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1676 if (!route->path_rec) {
1681 ret = cma_query_ib_route(id_priv, timeout_ms, work);
1687 kfree(route->path_rec);
1688 route->path_rec = NULL;
1694 int rdma_set_ib_paths(struct rdma_cm_id *id,
1695 struct ib_sa_path_rec *path_rec, int num_paths)
1697 struct rdma_id_private *id_priv;
1700 id_priv = container_of(id, struct rdma_id_private, id);
1701 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED))
1704 id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL);
1705 if (!id->route.path_rec) {
1710 memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths);
1713 cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED);
1716 EXPORT_SYMBOL(rdma_set_ib_paths);
1718 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1720 struct cma_work *work;
1722 work = kzalloc(sizeof *work, GFP_KERNEL);
1727 INIT_WORK(&work->work, cma_work_handler);
1728 work->old_state = CMA_ROUTE_QUERY;
1729 work->new_state = CMA_ROUTE_RESOLVED;
1730 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1731 queue_work(cma_wq, &work->work);
1735 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1737 struct rdma_id_private *id_priv;
1740 id_priv = container_of(id, struct rdma_id_private, id);
1741 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
1744 atomic_inc(&id_priv->refcount);
1745 switch (rdma_node_get_transport(id->device->node_type)) {
1746 case RDMA_TRANSPORT_IB:
1747 ret = cma_resolve_ib_route(id_priv, timeout_ms);
1749 case RDMA_TRANSPORT_IWARP:
1750 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1761 cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED);
1762 cma_deref_id(id_priv);
1765 EXPORT_SYMBOL(rdma_resolve_route);
1767 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1769 struct cma_device *cma_dev;
1770 struct ib_port_attr port_attr;
1777 if (list_empty(&dev_list)) {
1781 list_for_each_entry(cma_dev, &dev_list, list)
1782 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1783 if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1784 port_attr.state == IB_PORT_ACTIVE)
1788 cma_dev = list_entry(dev_list.next, struct cma_device, list);
1791 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1795 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1799 ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1800 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1801 id_priv->id.port_num = p;
1802 cma_attach_to_dev(id_priv, cma_dev);
1804 mutex_unlock(&lock);
1808 static void addr_handler(int status, struct sockaddr *src_addr,
1809 struct rdma_dev_addr *dev_addr, void *context)
1811 struct rdma_id_private *id_priv = context;
1812 struct rdma_cm_event event;
1814 memset(&event, 0, sizeof event);
1815 mutex_lock(&id_priv->handler_mutex);
1818 * Grab mutex to block rdma_destroy_id() from removing the device while
1819 * we're trying to acquire it.
1822 if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) {
1823 mutex_unlock(&lock);
1827 if (!status && !id_priv->cma_dev)
1828 status = cma_acquire_dev(id_priv);
1829 mutex_unlock(&lock);
1832 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
1834 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1835 event.status = status;
1837 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1838 ip_addr_size(src_addr));
1839 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1842 if (id_priv->id.event_handler(&id_priv->id, &event)) {
1843 cma_exch(id_priv, CMA_DESTROYING);
1844 mutex_unlock(&id_priv->handler_mutex);
1845 cma_deref_id(id_priv);
1846 rdma_destroy_id(&id_priv->id);
1850 mutex_unlock(&id_priv->handler_mutex);
1851 cma_deref_id(id_priv);
1854 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
1856 struct cma_work *work;
1857 struct sockaddr_in *src_in, *dst_in;
1861 work = kzalloc(sizeof *work, GFP_KERNEL);
1865 if (!id_priv->cma_dev) {
1866 ret = cma_bind_loopback(id_priv);
1871 ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1872 ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
1874 if (cma_zero_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)) {
1875 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr;
1876 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr;
1877 src_in->sin_family = dst_in->sin_family;
1878 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr;
1882 INIT_WORK(&work->work, cma_work_handler);
1883 work->old_state = CMA_ADDR_QUERY;
1884 work->new_state = CMA_ADDR_RESOLVED;
1885 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1886 queue_work(cma_wq, &work->work);
1893 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1894 struct sockaddr *dst_addr)
1896 if (src_addr && src_addr->sa_family)
1897 return rdma_bind_addr(id, src_addr);
1899 return cma_bind_any(id, dst_addr->sa_family);
1902 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1903 struct sockaddr *dst_addr, int timeout_ms)
1905 struct rdma_id_private *id_priv;
1908 id_priv = container_of(id, struct rdma_id_private, id);
1909 if (id_priv->state == CMA_IDLE) {
1910 ret = cma_bind_addr(id, src_addr, dst_addr);
1915 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
1918 atomic_inc(&id_priv->refcount);
1919 memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
1920 if (cma_any_addr(dst_addr))
1921 ret = cma_resolve_loopback(id_priv);
1923 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
1924 dst_addr, &id->route.addr.dev_addr,
1925 timeout_ms, addr_handler, id_priv);
1931 cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND);
1932 cma_deref_id(id_priv);
1935 EXPORT_SYMBOL(rdma_resolve_addr);
1937 static void cma_bind_port(struct rdma_bind_list *bind_list,
1938 struct rdma_id_private *id_priv)
1940 struct sockaddr_in *sin;
1942 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1943 sin->sin_port = htons(bind_list->port);
1944 id_priv->bind_list = bind_list;
1945 hlist_add_head(&id_priv->node, &bind_list->owners);
1948 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
1949 unsigned short snum)
1951 struct rdma_bind_list *bind_list;
1954 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1959 ret = idr_get_new_above(ps, bind_list, snum, &port);
1960 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1966 ret = -EADDRNOTAVAIL;
1971 bind_list->port = (unsigned short) port;
1972 cma_bind_port(bind_list, id_priv);
1975 idr_remove(ps, port);
1981 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
1983 struct rdma_bind_list *bind_list;
1984 int port, ret, low, high;
1986 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1991 /* FIXME: add proper port randomization per like inet_csk_get_port */
1993 ret = idr_get_new_above(ps, bind_list, next_port, &port);
1994 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1999 inet_get_local_port_range(&low, &high);
2001 if (next_port != low) {
2002 idr_remove(ps, port);
2006 ret = -EADDRNOTAVAIL;
2013 next_port = port + 1;
2016 bind_list->port = (unsigned short) port;
2017 cma_bind_port(bind_list, id_priv);
2020 idr_remove(ps, port);
2026 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2028 struct rdma_id_private *cur_id;
2029 struct sockaddr_in *sin, *cur_sin;
2030 struct rdma_bind_list *bind_list;
2031 struct hlist_node *node;
2032 unsigned short snum;
2034 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
2035 snum = ntohs(sin->sin_port);
2036 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
2039 bind_list = idr_find(ps, snum);
2041 return cma_alloc_port(ps, id_priv, snum);
2044 * We don't support binding to any address if anyone is bound to
2045 * a specific address on the same port.
2047 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr))
2048 return -EADDRNOTAVAIL;
2050 hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2051 if (cma_any_addr((struct sockaddr *) &cur_id->id.route.addr.src_addr))
2052 return -EADDRNOTAVAIL;
2054 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr;
2055 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr)
2059 cma_bind_port(bind_list, id_priv);
2063 static int cma_get_port(struct rdma_id_private *id_priv)
2068 switch (id_priv->id.ps) {
2082 return -EPROTONOSUPPORT;
2086 if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
2087 ret = cma_alloc_any_port(ps, id_priv);
2089 ret = cma_use_port(ps, id_priv);
2090 mutex_unlock(&lock);
2095 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2097 struct rdma_id_private *id_priv;
2100 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
2101 return -EAFNOSUPPORT;
2103 id_priv = container_of(id, struct rdma_id_private, id);
2104 if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
2107 if (!cma_any_addr(addr)) {
2108 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
2113 ret = cma_acquire_dev(id_priv);
2114 mutex_unlock(&lock);
2119 memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2120 ret = cma_get_port(id_priv);
2126 if (!cma_any_addr(addr)) {
2128 cma_detach_from_dev(id_priv);
2129 mutex_unlock(&lock);
2132 cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE);
2135 EXPORT_SYMBOL(rdma_bind_addr);
2137 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2138 struct rdma_route *route)
2140 struct cma_hdr *cma_hdr;
2141 struct sdp_hh *sdp_hdr;
2143 if (route->addr.src_addr.ss_family == AF_INET) {
2144 struct sockaddr_in *src4, *dst4;
2146 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2147 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2152 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2154 sdp_set_ip_ver(sdp_hdr, 4);
2155 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2156 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2157 sdp_hdr->port = src4->sin_port;
2161 cma_hdr->cma_version = CMA_VERSION;
2162 cma_set_ip_ver(cma_hdr, 4);
2163 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2164 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2165 cma_hdr->port = src4->sin_port;
2169 struct sockaddr_in6 *src6, *dst6;
2171 src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
2172 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
2177 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2179 sdp_set_ip_ver(sdp_hdr, 6);
2180 sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2181 sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2182 sdp_hdr->port = src6->sin6_port;
2186 cma_hdr->cma_version = CMA_VERSION;
2187 cma_set_ip_ver(cma_hdr, 6);
2188 cma_hdr->src_addr.ip6 = src6->sin6_addr;
2189 cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2190 cma_hdr->port = src6->sin6_port;
2197 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2198 struct ib_cm_event *ib_event)
2200 struct rdma_id_private *id_priv = cm_id->context;
2201 struct rdma_cm_event event;
2202 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2205 if (cma_disable_callback(id_priv, CMA_CONNECT))
2208 memset(&event, 0, sizeof event);
2209 switch (ib_event->event) {
2210 case IB_CM_SIDR_REQ_ERROR:
2211 event.event = RDMA_CM_EVENT_UNREACHABLE;
2212 event.status = -ETIMEDOUT;
2214 case IB_CM_SIDR_REP_RECEIVED:
2215 event.param.ud.private_data = ib_event->private_data;
2216 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2217 if (rep->status != IB_SIDR_SUCCESS) {
2218 event.event = RDMA_CM_EVENT_UNREACHABLE;
2219 event.status = ib_event->param.sidr_rep_rcvd.status;
2222 ret = cma_set_qkey(id_priv);
2224 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2225 event.status = -EINVAL;
2228 if (id_priv->qkey != rep->qkey) {
2229 event.event = RDMA_CM_EVENT_UNREACHABLE;
2230 event.status = -EINVAL;
2233 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2234 id_priv->id.route.path_rec,
2235 &event.param.ud.ah_attr);
2236 event.param.ud.qp_num = rep->qpn;
2237 event.param.ud.qkey = rep->qkey;
2238 event.event = RDMA_CM_EVENT_ESTABLISHED;
2242 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2247 ret = id_priv->id.event_handler(&id_priv->id, &event);
2249 /* Destroy the CM ID by returning a non-zero value. */
2250 id_priv->cm_id.ib = NULL;
2251 cma_exch(id_priv, CMA_DESTROYING);
2252 mutex_unlock(&id_priv->handler_mutex);
2253 rdma_destroy_id(&id_priv->id);
2257 mutex_unlock(&id_priv->handler_mutex);
2261 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2262 struct rdma_conn_param *conn_param)
2264 struct ib_cm_sidr_req_param req;
2265 struct rdma_route *route;
2268 req.private_data_len = sizeof(struct cma_hdr) +
2269 conn_param->private_data_len;
2270 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2271 if (!req.private_data)
2274 if (conn_param->private_data && conn_param->private_data_len)
2275 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2276 conn_param->private_data, conn_param->private_data_len);
2278 route = &id_priv->id.route;
2279 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2283 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device,
2284 cma_sidr_rep_handler, id_priv);
2285 if (IS_ERR(id_priv->cm_id.ib)) {
2286 ret = PTR_ERR(id_priv->cm_id.ib);
2290 req.path = route->path_rec;
2291 req.service_id = cma_get_service_id(id_priv->id.ps,
2292 (struct sockaddr *) &route->addr.dst_addr);
2293 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2294 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2296 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2298 ib_destroy_cm_id(id_priv->cm_id.ib);
2299 id_priv->cm_id.ib = NULL;
2302 kfree(req.private_data);
2306 static int cma_connect_ib(struct rdma_id_private *id_priv,
2307 struct rdma_conn_param *conn_param)
2309 struct ib_cm_req_param req;
2310 struct rdma_route *route;
2314 memset(&req, 0, sizeof req);
2315 offset = cma_user_data_offset(id_priv->id.ps);
2316 req.private_data_len = offset + conn_param->private_data_len;
2317 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2321 if (conn_param->private_data && conn_param->private_data_len)
2322 memcpy(private_data + offset, conn_param->private_data,
2323 conn_param->private_data_len);
2325 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler,
2327 if (IS_ERR(id_priv->cm_id.ib)) {
2328 ret = PTR_ERR(id_priv->cm_id.ib);
2332 route = &id_priv->id.route;
2333 ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2336 req.private_data = private_data;
2338 req.primary_path = &route->path_rec[0];
2339 if (route->num_paths == 2)
2340 req.alternate_path = &route->path_rec[1];
2342 req.service_id = cma_get_service_id(id_priv->id.ps,
2343 (struct sockaddr *) &route->addr.dst_addr);
2344 req.qp_num = id_priv->qp_num;
2345 req.qp_type = IB_QPT_RC;
2346 req.starting_psn = id_priv->seq_num;
2347 req.responder_resources = conn_param->responder_resources;
2348 req.initiator_depth = conn_param->initiator_depth;
2349 req.flow_control = conn_param->flow_control;
2350 req.retry_count = conn_param->retry_count;
2351 req.rnr_retry_count = conn_param->rnr_retry_count;
2352 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2353 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2354 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2355 req.srq = id_priv->srq ? 1 : 0;
2357 ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2359 if (ret && !IS_ERR(id_priv->cm_id.ib)) {
2360 ib_destroy_cm_id(id_priv->cm_id.ib);
2361 id_priv->cm_id.ib = NULL;
2364 kfree(private_data);
2368 static int cma_connect_iw(struct rdma_id_private *id_priv,
2369 struct rdma_conn_param *conn_param)
2371 struct iw_cm_id *cm_id;
2372 struct sockaddr_in* sin;
2374 struct iw_cm_conn_param iw_param;
2376 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2377 if (IS_ERR(cm_id)) {
2378 ret = PTR_ERR(cm_id);
2382 id_priv->cm_id.iw = cm_id;
2384 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2385 cm_id->local_addr = *sin;
2387 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2388 cm_id->remote_addr = *sin;
2390 ret = cma_modify_qp_rtr(id_priv, conn_param);
2394 iw_param.ord = conn_param->initiator_depth;
2395 iw_param.ird = conn_param->responder_resources;
2396 iw_param.private_data = conn_param->private_data;
2397 iw_param.private_data_len = conn_param->private_data_len;
2399 iw_param.qpn = id_priv->qp_num;
2401 iw_param.qpn = conn_param->qp_num;
2402 ret = iw_cm_connect(cm_id, &iw_param);
2404 if (ret && !IS_ERR(cm_id)) {
2405 iw_destroy_cm_id(cm_id);
2406 id_priv->cm_id.iw = NULL;
2411 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2413 struct rdma_id_private *id_priv;
2416 id_priv = container_of(id, struct rdma_id_private, id);
2417 if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
2421 id_priv->qp_num = conn_param->qp_num;
2422 id_priv->srq = conn_param->srq;
2425 switch (rdma_node_get_transport(id->device->node_type)) {
2426 case RDMA_TRANSPORT_IB:
2427 if (cma_is_ud_ps(id->ps))
2428 ret = cma_resolve_ib_udp(id_priv, conn_param);
2430 ret = cma_connect_ib(id_priv, conn_param);
2432 case RDMA_TRANSPORT_IWARP:
2433 ret = cma_connect_iw(id_priv, conn_param);
2444 cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED);
2447 EXPORT_SYMBOL(rdma_connect);
2449 static int cma_accept_ib(struct rdma_id_private *id_priv,
2450 struct rdma_conn_param *conn_param)
2452 struct ib_cm_rep_param rep;
2455 ret = cma_modify_qp_rtr(id_priv, conn_param);
2459 ret = cma_modify_qp_rts(id_priv, conn_param);
2463 memset(&rep, 0, sizeof rep);
2464 rep.qp_num = id_priv->qp_num;
2465 rep.starting_psn = id_priv->seq_num;
2466 rep.private_data = conn_param->private_data;
2467 rep.private_data_len = conn_param->private_data_len;
2468 rep.responder_resources = conn_param->responder_resources;
2469 rep.initiator_depth = conn_param->initiator_depth;
2470 rep.failover_accepted = 0;
2471 rep.flow_control = conn_param->flow_control;
2472 rep.rnr_retry_count = conn_param->rnr_retry_count;
2473 rep.srq = id_priv->srq ? 1 : 0;
2475 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2480 static int cma_accept_iw(struct rdma_id_private *id_priv,
2481 struct rdma_conn_param *conn_param)
2483 struct iw_cm_conn_param iw_param;
2486 ret = cma_modify_qp_rtr(id_priv, conn_param);
2490 iw_param.ord = conn_param->initiator_depth;
2491 iw_param.ird = conn_param->responder_resources;
2492 iw_param.private_data = conn_param->private_data;
2493 iw_param.private_data_len = conn_param->private_data_len;
2494 if (id_priv->id.qp) {
2495 iw_param.qpn = id_priv->qp_num;
2497 iw_param.qpn = conn_param->qp_num;
2499 return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2502 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2503 enum ib_cm_sidr_status status,
2504 const void *private_data, int private_data_len)
2506 struct ib_cm_sidr_rep_param rep;
2509 memset(&rep, 0, sizeof rep);
2510 rep.status = status;
2511 if (status == IB_SIDR_SUCCESS) {
2512 ret = cma_set_qkey(id_priv);
2515 rep.qp_num = id_priv->qp_num;
2516 rep.qkey = id_priv->qkey;
2518 rep.private_data = private_data;
2519 rep.private_data_len = private_data_len;
2521 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2524 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2526 struct rdma_id_private *id_priv;
2529 id_priv = container_of(id, struct rdma_id_private, id);
2530 if (!cma_comp(id_priv, CMA_CONNECT))
2533 if (!id->qp && conn_param) {
2534 id_priv->qp_num = conn_param->qp_num;
2535 id_priv->srq = conn_param->srq;
2538 switch (rdma_node_get_transport(id->device->node_type)) {
2539 case RDMA_TRANSPORT_IB:
2540 if (cma_is_ud_ps(id->ps))
2541 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2542 conn_param->private_data,
2543 conn_param->private_data_len);
2544 else if (conn_param)
2545 ret = cma_accept_ib(id_priv, conn_param);
2547 ret = cma_rep_recv(id_priv);
2549 case RDMA_TRANSPORT_IWARP:
2550 ret = cma_accept_iw(id_priv, conn_param);
2562 cma_modify_qp_err(id_priv);
2563 rdma_reject(id, NULL, 0);
2566 EXPORT_SYMBOL(rdma_accept);
2568 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2570 struct rdma_id_private *id_priv;
2573 id_priv = container_of(id, struct rdma_id_private, id);
2574 if (!cma_has_cm_dev(id_priv))
2577 switch (id->device->node_type) {
2578 case RDMA_NODE_IB_CA:
2579 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2587 EXPORT_SYMBOL(rdma_notify);
2589 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2590 u8 private_data_len)
2592 struct rdma_id_private *id_priv;
2595 id_priv = container_of(id, struct rdma_id_private, id);
2596 if (!cma_has_cm_dev(id_priv))
2599 switch (rdma_node_get_transport(id->device->node_type)) {
2600 case RDMA_TRANSPORT_IB:
2601 if (cma_is_ud_ps(id->ps))
2602 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2603 private_data, private_data_len);
2605 ret = ib_send_cm_rej(id_priv->cm_id.ib,
2606 IB_CM_REJ_CONSUMER_DEFINED, NULL,
2607 0, private_data, private_data_len);
2609 case RDMA_TRANSPORT_IWARP:
2610 ret = iw_cm_reject(id_priv->cm_id.iw,
2611 private_data, private_data_len);
2619 EXPORT_SYMBOL(rdma_reject);
2621 int rdma_disconnect(struct rdma_cm_id *id)
2623 struct rdma_id_private *id_priv;
2626 id_priv = container_of(id, struct rdma_id_private, id);
2627 if (!cma_has_cm_dev(id_priv))
2630 switch (rdma_node_get_transport(id->device->node_type)) {
2631 case RDMA_TRANSPORT_IB:
2632 ret = cma_modify_qp_err(id_priv);
2635 /* Initiate or respond to a disconnect. */
2636 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2637 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2639 case RDMA_TRANSPORT_IWARP:
2640 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2649 EXPORT_SYMBOL(rdma_disconnect);
2651 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2653 struct rdma_id_private *id_priv;
2654 struct cma_multicast *mc = multicast->context;
2655 struct rdma_cm_event event;
2658 id_priv = mc->id_priv;
2659 if (cma_disable_callback(id_priv, CMA_ADDR_BOUND) &&
2660 cma_disable_callback(id_priv, CMA_ADDR_RESOLVED))
2663 mutex_lock(&id_priv->qp_mutex);
2664 if (!status && id_priv->id.qp)
2665 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2666 multicast->rec.mlid);
2667 mutex_unlock(&id_priv->qp_mutex);
2669 memset(&event, 0, sizeof event);
2670 event.status = status;
2671 event.param.ud.private_data = mc->context;
2673 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2674 ib_init_ah_from_mcmember(id_priv->id.device,
2675 id_priv->id.port_num, &multicast->rec,
2676 &event.param.ud.ah_attr);
2677 event.param.ud.qp_num = 0xFFFFFF;
2678 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2680 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2682 ret = id_priv->id.event_handler(&id_priv->id, &event);
2684 cma_exch(id_priv, CMA_DESTROYING);
2685 mutex_unlock(&id_priv->handler_mutex);
2686 rdma_destroy_id(&id_priv->id);
2690 mutex_unlock(&id_priv->handler_mutex);
2694 static void cma_set_mgid(struct rdma_id_private *id_priv,
2695 struct sockaddr *addr, union ib_gid *mgid)
2697 unsigned char mc_map[MAX_ADDR_LEN];
2698 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2699 struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2700 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2702 if (cma_any_addr(addr)) {
2703 memset(mgid, 0, sizeof *mgid);
2704 } else if ((addr->sa_family == AF_INET6) &&
2705 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) ==
2707 /* IPv6 address is an SA assigned MGID. */
2708 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2710 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
2711 if (id_priv->id.ps == RDMA_PS_UDP)
2712 mc_map[7] = 0x01; /* Use RDMA CM signature */
2713 *mgid = *(union ib_gid *) (mc_map + 4);
2717 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2718 struct cma_multicast *mc)
2720 struct ib_sa_mcmember_rec rec;
2721 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2722 ib_sa_comp_mask comp_mask;
2725 ib_addr_get_mgid(dev_addr, &rec.mgid);
2726 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2731 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
2732 if (id_priv->id.ps == RDMA_PS_UDP)
2733 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2734 ib_addr_get_sgid(dev_addr, &rec.port_gid);
2735 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2738 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
2739 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
2740 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
2741 IB_SA_MCMEMBER_REC_FLOW_LABEL |
2742 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
2744 if (id_priv->id.ps == RDMA_PS_IPOIB)
2745 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
2746 IB_SA_MCMEMBER_REC_RATE_SELECTOR;
2748 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
2749 id_priv->id.port_num, &rec,
2750 comp_mask, GFP_KERNEL,
2751 cma_ib_mc_handler, mc);
2752 if (IS_ERR(mc->multicast.ib))
2753 return PTR_ERR(mc->multicast.ib);
2758 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
2761 struct rdma_id_private *id_priv;
2762 struct cma_multicast *mc;
2765 id_priv = container_of(id, struct rdma_id_private, id);
2766 if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2767 !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2770 mc = kmalloc(sizeof *mc, GFP_KERNEL);
2774 memcpy(&mc->addr, addr, ip_addr_size(addr));
2775 mc->context = context;
2776 mc->id_priv = id_priv;
2778 spin_lock(&id_priv->lock);
2779 list_add(&mc->list, &id_priv->mc_list);
2780 spin_unlock(&id_priv->lock);
2782 switch (rdma_node_get_transport(id->device->node_type)) {
2783 case RDMA_TRANSPORT_IB:
2784 ret = cma_join_ib_multicast(id_priv, mc);
2792 spin_lock_irq(&id_priv->lock);
2793 list_del(&mc->list);
2794 spin_unlock_irq(&id_priv->lock);
2799 EXPORT_SYMBOL(rdma_join_multicast);
2801 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
2803 struct rdma_id_private *id_priv;
2804 struct cma_multicast *mc;
2806 id_priv = container_of(id, struct rdma_id_private, id);
2807 spin_lock_irq(&id_priv->lock);
2808 list_for_each_entry(mc, &id_priv->mc_list, list) {
2809 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
2810 list_del(&mc->list);
2811 spin_unlock_irq(&id_priv->lock);
2814 ib_detach_mcast(id->qp,
2815 &mc->multicast.ib->rec.mgid,
2816 mc->multicast.ib->rec.mlid);
2817 ib_sa_free_multicast(mc->multicast.ib);
2822 spin_unlock_irq(&id_priv->lock);
2824 EXPORT_SYMBOL(rdma_leave_multicast);
2826 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
2828 struct rdma_dev_addr *dev_addr;
2829 struct cma_ndev_work *work;
2831 dev_addr = &id_priv->id.route.addr.dev_addr;
2833 if ((dev_addr->src_dev == ndev) &&
2834 memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
2835 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
2836 ndev->name, &id_priv->id);
2837 work = kzalloc(sizeof *work, GFP_KERNEL);
2841 INIT_WORK(&work->work, cma_ndev_work_handler);
2843 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
2844 atomic_inc(&id_priv->refcount);
2845 queue_work(cma_wq, &work->work);
2851 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
2854 struct net_device *ndev = (struct net_device *)ctx;
2855 struct cma_device *cma_dev;
2856 struct rdma_id_private *id_priv;
2857 int ret = NOTIFY_DONE;
2859 if (dev_net(ndev) != &init_net)
2862 if (event != NETDEV_BONDING_FAILOVER)
2865 if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
2869 list_for_each_entry(cma_dev, &dev_list, list)
2870 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
2871 ret = cma_netdev_change(ndev, id_priv);
2877 mutex_unlock(&lock);
2881 static struct notifier_block cma_nb = {
2882 .notifier_call = cma_netdev_callback
2885 static void cma_add_one(struct ib_device *device)
2887 struct cma_device *cma_dev;
2888 struct rdma_id_private *id_priv;
2890 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
2894 cma_dev->device = device;
2896 init_completion(&cma_dev->comp);
2897 atomic_set(&cma_dev->refcount, 1);
2898 INIT_LIST_HEAD(&cma_dev->id_list);
2899 ib_set_client_data(device, &cma_client, cma_dev);
2902 list_add_tail(&cma_dev->list, &dev_list);
2903 list_for_each_entry(id_priv, &listen_any_list, list)
2904 cma_listen_on_dev(id_priv, cma_dev);
2905 mutex_unlock(&lock);
2908 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
2910 struct rdma_cm_event event;
2911 enum cma_state state;
2914 /* Record that we want to remove the device */
2915 state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
2916 if (state == CMA_DESTROYING)
2919 cma_cancel_operation(id_priv, state);
2920 mutex_lock(&id_priv->handler_mutex);
2922 /* Check for destruction from another callback. */
2923 if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
2926 memset(&event, 0, sizeof event);
2927 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
2928 ret = id_priv->id.event_handler(&id_priv->id, &event);
2930 mutex_unlock(&id_priv->handler_mutex);
2934 static void cma_process_remove(struct cma_device *cma_dev)
2936 struct rdma_id_private *id_priv;
2940 while (!list_empty(&cma_dev->id_list)) {
2941 id_priv = list_entry(cma_dev->id_list.next,
2942 struct rdma_id_private, list);
2944 list_del(&id_priv->listen_list);
2945 list_del_init(&id_priv->list);
2946 atomic_inc(&id_priv->refcount);
2947 mutex_unlock(&lock);
2949 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
2950 cma_deref_id(id_priv);
2952 rdma_destroy_id(&id_priv->id);
2956 mutex_unlock(&lock);
2958 cma_deref_dev(cma_dev);
2959 wait_for_completion(&cma_dev->comp);
2962 static void cma_remove_one(struct ib_device *device)
2964 struct cma_device *cma_dev;
2966 cma_dev = ib_get_client_data(device, &cma_client);
2971 list_del(&cma_dev->list);
2972 mutex_unlock(&lock);
2974 cma_process_remove(cma_dev);
2978 static int __init cma_init(void)
2980 int ret, low, high, remaining;
2982 get_random_bytes(&next_port, sizeof next_port);
2983 inet_get_local_port_range(&low, &high);
2984 remaining = (high - low) + 1;
2985 next_port = ((unsigned int) next_port % remaining) + low;
2987 cma_wq = create_singlethread_workqueue("rdma_cm");
2991 ib_sa_register_client(&sa_client);
2992 rdma_addr_register_client(&addr_client);
2993 register_netdevice_notifier(&cma_nb);
2995 ret = ib_register_client(&cma_client);
3001 unregister_netdevice_notifier(&cma_nb);
3002 rdma_addr_unregister_client(&addr_client);
3003 ib_sa_unregister_client(&sa_client);
3004 destroy_workqueue(cma_wq);
3008 static void __exit cma_cleanup(void)
3010 ib_unregister_client(&cma_client);
3011 unregister_netdevice_notifier(&cma_nb);
3012 rdma_addr_unregister_client(&addr_client);
3013 ib_sa_unregister_client(&sa_client);
3014 destroy_workqueue(cma_wq);
3015 idr_destroy(&sdp_ps);
3016 idr_destroy(&tcp_ps);
3017 idr_destroy(&udp_ps);
3018 idr_destroy(&ipoib_ps);
3021 module_init(cma_init);
3022 module_exit(cma_cleanup);