net/sunrpc/xprtrdma/verbs.c

   1 /*
   2  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the BSD-type
   8  * license below:
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  *      Redistributions of source code must retain the above copyright
  15  *      notice, this list of conditions and the following disclaimer.
  16  *
  17  *      Redistributions in binary form must reproduce the above
  18  *      copyright notice, this list of conditions and the following
  19  *      disclaimer in the documentation and/or other materials provided
  20  *      with the distribution.
  21  *
  22  *      Neither the name of the Network Appliance, Inc. nor the names of
  23  *      its contributors may be used to endorse or promote products
  24  *      derived from this software without specific prior written
  25  *      permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  32  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  33  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  34  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  35  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  36  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  37  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38  */
  39
  40 /*
  41  * verbs.c
  42  *
  43  * Encapsulates the major functions managing:
  44  *  o adapters
  45  *  o endpoints
  46  *  o connections
  47  *  o buffer memory
  48  */
  49
  50 #include <linux/interrupt.h>
  51 #include <linux/pci.h>  /* for Tavor hack below */
  52 #include <linux/slab.h>
  53
  54 #include "xprt_rdma.h"
  55
  56 /*
  57  * Globals/Macros
  58  */
  59
  60 #ifdef RPC_DEBUG
  61 # define RPCDBG_FACILITY        RPCDBG_TRANS
  62 #endif
  63
  64 /*
  65  * internal functions
  66  */
  67
  68 /*
  69  * handle replies in tasklet context, using a single, global list
  70  * rdma tasklet function -- just turn around and call the func
  71  * for all replies on the list
  72  */
  73
  74 static DEFINE_SPINLOCK(rpcrdma_tk_lock_g);
  75 static LIST_HEAD(rpcrdma_tasklets_g);
  76
  77 static void
  78 rpcrdma_run_tasklet(unsigned long data)
  79 {
  80         struct rpcrdma_rep *rep;
  81         void (*func)(struct rpcrdma_rep *);
  82         unsigned long flags;
  83
  84         data = data;
  85         spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
  86         while (!list_empty(&rpcrdma_tasklets_g)) {
  87                 rep = list_entry(rpcrdma_tasklets_g.next,
  88                                  struct rpcrdma_rep, rr_list);
  89                 list_del(&rep->rr_list);
  90                 func = rep->rr_func;
  91                 rep->rr_func = NULL;
  92                 spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
  93
  94                 if (func)
  95                         func(rep);
  96                 else
  97                         rpcrdma_recv_buffer_put(rep);
  98
  99                 spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
 100         }
 101         spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
 102 }
 103
 104 static DECLARE_TASKLET(rpcrdma_tasklet_g, rpcrdma_run_tasklet, 0UL);
 105
 106 static inline void
 107 rpcrdma_schedule_tasklet(struct rpcrdma_rep *rep)
 108 {
 109         unsigned long flags;
 110
 111         spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
 112         list_add_tail(&rep->rr_list, &rpcrdma_tasklets_g);
 113         spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
 114         tasklet_schedule(&rpcrdma_tasklet_g);
 115 }
 116
 117 static void
 118 rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context)
 119 {
 120         struct rpcrdma_ep *ep = context;
 121
 122         dprintk("RPC:       %s: QP error %X on device %s ep %p\n",
 123                 __func__, event->event, event->device->name, context);
 124         if (ep->rep_connected == 1) {
 125                 ep->rep_connected = -EIO;
 126                 ep->rep_func(ep);
 127                 wake_up_all(&ep->rep_connect_wait);
 128         }
 129 }
 130
 131 static void
 132 rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context)
 133 {
 134         struct rpcrdma_ep *ep = context;
 135
 136         dprintk("RPC:       %s: CQ error %X on device %s ep %p\n",
 137                 __func__, event->event, event->device->name, context);
 138         if (ep->rep_connected == 1) {
 139                 ep->rep_connected = -EIO;
 140                 ep->rep_func(ep);
 141                 wake_up_all(&ep->rep_connect_wait);
 142         }
 143 }
 144
 145 static inline
 146 void rpcrdma_event_process(struct ib_wc *wc)
 147 {
 148         struct rpcrdma_mw *frmr;
 149         struct rpcrdma_rep *rep =
 150                         (struct rpcrdma_rep *)(unsigned long) wc->wr_id;
 151
 152         dprintk("RPC:       %s: event rep %p status %X opcode %X length %u\n",
 153                 __func__, rep, wc->status, wc->opcode, wc->byte_len);
 154
 155         if (!rep) /* send or bind completion that we don't care about */
 156                 return;
 157
 158         if (IB_WC_SUCCESS != wc->status) {
 159                 dprintk("RPC:       %s: WC opcode %d status %X, connection lost\n",
 160                         __func__, wc->opcode, wc->status);
 161                 rep->rr_len = ~0U;
 162                 if (wc->opcode != IB_WC_FAST_REG_MR && wc->opcode != IB_WC_LOCAL_INV)
 163                         rpcrdma_schedule_tasklet(rep);
 164                 return;
 165         }
 166
 167         switch (wc->opcode) {
 168         case IB_WC_FAST_REG_MR:
 169                 frmr = (struct rpcrdma_mw *)(unsigned long)wc->wr_id;
 170                 frmr->r.frmr.state = FRMR_IS_VALID;
 171                 break;
 172         case IB_WC_LOCAL_INV:
 173                 frmr = (struct rpcrdma_mw *)(unsigned long)wc->wr_id;
 174                 frmr->r.frmr.state = FRMR_IS_INVALID;
 175                 break;
 176         case IB_WC_RECV:
 177                 rep->rr_len = wc->byte_len;
 178                 ib_dma_sync_single_for_cpu(
 179                         rdmab_to_ia(rep->rr_buffer)->ri_id->device,
 180                         rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE);
 181                 /* Keep (only) the most recent credits, after check validity */
 182                 if (rep->rr_len >= 16) {
 183                         struct rpcrdma_msg *p =
 184                                         (struct rpcrdma_msg *) rep->rr_base;
 185                         unsigned int credits = ntohl(p->rm_credit);
 186                         if (credits == 0) {
 187                                 dprintk("RPC:       %s: server"
 188                                         " dropped credits to 0!\n", __func__);
 189                                 /* don't deadlock */
 190                                 credits = 1;
 191                         } else if (credits > rep->rr_buffer->rb_max_requests) {
 192                                 dprintk("RPC:       %s: server"
 193                                         " over-crediting: %d (%d)\n",
 194                                         __func__, credits,
 195                                         rep->rr_buffer->rb_max_requests);
 196                                 credits = rep->rr_buffer->rb_max_requests;
 197                         }
 198                         atomic_set(&rep->rr_buffer->rb_credits, credits);
 199                 }
 200                 /* fall through */
 201         case IB_WC_BIND_MW:
 202                 rpcrdma_schedule_tasklet(rep);
 203                 break;
 204         default:
 205                 dprintk("RPC:       %s: unexpected WC event %X\n",
 206                         __func__, wc->opcode);
 207                 break;
 208         }
 209 }
 210
 211 static inline int
 212 rpcrdma_cq_poll(struct ib_cq *cq)
 213 {
 214         struct ib_wc wc;
 215         int rc;
 216
 217         for (;;) {
 218                 rc = ib_poll_cq(cq, 1, &wc);
 219                 if (rc < 0) {
 220                         dprintk("RPC:       %s: ib_poll_cq failed %i\n",
 221                                 __func__, rc);
 222                         return rc;
 223                 }
 224                 if (rc == 0)
 225                         break;
 226
 227                 rpcrdma_event_process(&wc);
 228         }
 229
 230         return 0;
 231 }
 232
 233 /*
 234  * rpcrdma_cq_event_upcall
 235  *
 236  * This upcall handles recv, send, bind and unbind events.
 237  * It is reentrant but processes single events in order to maintain
 238  * ordering of receives to keep server credits.
 239  *
 240  * It is the responsibility of the scheduled tasklet to return
 241  * recv buffers to the pool. NOTE: this affects synchronization of
 242  * connection shutdown. That is, the structures required for
 243  * the completion of the reply handler must remain intact until
 244  * all memory has been reclaimed.
 245  *
 246  * Note that send events are suppressed and do not result in an upcall.
 247  */
 248 static void
 249 rpcrdma_cq_event_upcall(struct ib_cq *cq, void *context)
 250 {
 251         int rc;
 252
 253         rc = rpcrdma_cq_poll(cq);
 254         if (rc)
 255                 return;
 256
 257         rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
 258         if (rc) {
 259                 dprintk("RPC:       %s: ib_req_notify_cq failed %i\n",
 260                         __func__, rc);
 261                 return;
 262         }
 263
 264         rpcrdma_cq_poll(cq);
 265 }
 266
 267 #ifdef RPC_DEBUG
 268 static const char * const conn[] = {
 269         "address resolved",
 270         "address error",
 271         "route resolved",
 272         "route error",
 273         "connect request",
 274         "connect response",
 275         "connect error",
 276         "unreachable",
 277         "rejected",
 278         "established",
 279         "disconnected",
 280         "device removal"
 281 };
 282 #endif
 283
 284 static int
 285 rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
 286 {
 287         struct rpcrdma_xprt *xprt = id->context;
 288         struct rpcrdma_ia *ia = &xprt->rx_ia;
 289         struct rpcrdma_ep *ep = &xprt->rx_ep;
 290 #ifdef RPC_DEBUG
 291         struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr;
 292 #endif
 293         struct ib_qp_attr attr;
 294         struct ib_qp_init_attr iattr;
 295         int connstate = 0;
 296
 297         switch (event->event) {
 298         case RDMA_CM_EVENT_ADDR_RESOLVED:
 299         case RDMA_CM_EVENT_ROUTE_RESOLVED:
 300                 ia->ri_async_rc = 0;
 301                 complete(&ia->ri_done);
 302                 break;
 303         case RDMA_CM_EVENT_ADDR_ERROR:
 304                 ia->ri_async_rc = -EHOSTUNREACH;
 305                 dprintk("RPC:       %s: CM address resolution error, ep 0x%p\n",
 306                         __func__, ep);
 307                 complete(&ia->ri_done);
 308                 break;
 309         case RDMA_CM_EVENT_ROUTE_ERROR:
 310                 ia->ri_async_rc = -ENETUNREACH;
 311                 dprintk("RPC:       %s: CM route resolution error, ep 0x%p\n",
 312                         __func__, ep);
 313                 complete(&ia->ri_done);
 314                 break;
 315         case RDMA_CM_EVENT_ESTABLISHED:
 316                 connstate = 1;
 317                 ib_query_qp(ia->ri_id->qp, &attr,
 318                         IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC,
 319                         &iattr);
 320                 dprintk("RPC:       %s: %d responder resources"
 321                         " (%d initiator)\n",
 322                         __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic);
 323                 goto connected;
 324         case RDMA_CM_EVENT_CONNECT_ERROR:
 325                 connstate = -ENOTCONN;
 326                 goto connected;
 327         case RDMA_CM_EVENT_UNREACHABLE:
 328                 connstate = -ENETDOWN;
 329                 goto connected;
 330         case RDMA_CM_EVENT_REJECTED:
 331                 connstate = -ECONNREFUSED;
 332                 goto connected;
 333         case RDMA_CM_EVENT_DISCONNECTED:
 334                 connstate = -ECONNABORTED;
 335                 goto connected;
 336         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 337                 connstate = -ENODEV;
 338 connected:
 339                 dprintk("RPC:       %s: %s: %pI4:%u (ep 0x%p event 0x%x)\n",
 340                         __func__,
 341                         (event->event <= 11) ? conn[event->event] :
 342                                                 "unknown connection error",
 343                         &addr->sin_addr.s_addr,
 344                         ntohs(addr->sin_port),
 345                         ep, event->event);
 346                 atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1);
 347                 dprintk("RPC:       %s: %sconnected\n",
 348                                         __func__, connstate > 0 ? "" : "dis");
 349                 ep->rep_connected = connstate;
 350                 ep->rep_func(ep);
 351                 wake_up_all(&ep->rep_connect_wait);
 352                 break;
 353         default:
 354                 dprintk("RPC:       %s: unexpected CM event %d\n",
 355                         __func__, event->event);
 356                 break;
 357         }
 358
 359 #ifdef RPC_DEBUG
 360         if (connstate == 1) {
 361                 int ird = attr.max_dest_rd_atomic;
 362                 int tird = ep->rep_remote_cma.responder_resources;
 363                 printk(KERN_INFO "rpcrdma: connection to %pI4:%u "
 364                         "on %s, memreg %d slots %d ird %d%s\n",
 365                         &addr->sin_addr.s_addr,
 366                         ntohs(addr->sin_port),
 367                         ia->ri_id->device->name,
 368                         ia->ri_memreg_strategy,
 369                         xprt->rx_buf.rb_max_requests,
 370                         ird, ird < 4 && ird < tird / 2 ? " (low!)" : "");
 371         } else if (connstate < 0) {
 372                 printk(KERN_INFO "rpcrdma: connection to %pI4:%u closed (%d)\n",
 373                         &addr->sin_addr.s_addr,
 374                         ntohs(addr->sin_port),
 375                         connstate);
 376         }
 377 #endif
 378
 379         return 0;
 380 }
 381
 382 static struct rdma_cm_id *
 383 rpcrdma_create_id(struct rpcrdma_xprt *xprt,
 384                         struct rpcrdma_ia *ia, struct sockaddr *addr)
 385 {
 386         struct rdma_cm_id *id;
 387         int rc;
 388
 389         init_completion(&ia->ri_done);
 390
 391         id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP, IB_QPT_RC);
 392         if (IS_ERR(id)) {
 393                 rc = PTR_ERR(id);
 394                 dprintk("RPC:       %s: rdma_create_id() failed %i\n",
 395                         __func__, rc);
 396                 return id;
 397         }
 398
 399         ia->ri_async_rc = -ETIMEDOUT;
 400         rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT);
 401         if (rc) {
 402                 dprintk("RPC:       %s: rdma_resolve_addr() failed %i\n",
 403                         __func__, rc);
 404                 goto out;
 405         }
 406         wait_for_completion_interruptible_timeout(&ia->ri_done,
 407                                 msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
 408         rc = ia->ri_async_rc;
 409         if (rc)
 410                 goto out;
 411
 412         ia->ri_async_rc = -ETIMEDOUT;
 413         rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT);
 414         if (rc) {
 415                 dprintk("RPC:       %s: rdma_resolve_route() failed %i\n",
 416                         __func__, rc);
 417                 goto out;
 418         }
 419         wait_for_completion_interruptible_timeout(&ia->ri_done,
 420                                 msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
 421         rc = ia->ri_async_rc;
 422         if (rc)
 423                 goto out;
 424
 425         return id;
 426
 427 out:
 428         rdma_destroy_id(id);
 429         return ERR_PTR(rc);
 430 }
 431
 432 /*
 433  * Drain any cq, prior to teardown.
 434  */
 435 static void
 436 rpcrdma_clean_cq(struct ib_cq *cq)
 437 {
 438         struct ib_wc wc;
 439         int count = 0;
 440
 441         while (1 == ib_poll_cq(cq, 1, &wc))
 442                 ++count;
 443
 444         if (count)
 445                 dprintk("RPC:       %s: flushed %d events (last 0x%x)\n",
 446                         __func__, count, wc.opcode);
 447 }
 448
 449 /*
 450  * Exported functions.
 451  */
 452
 453 /*
 454  * Open and initialize an Interface Adapter.
 455  *  o initializes fields of struct rpcrdma_ia, including
 456  *    interface and provider attributes and protection zone.
 457  */
 458 int
 459 rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
 460 {
 461         int rc, mem_priv;
 462         struct ib_device_attr devattr;
 463         struct rpcrdma_ia *ia = &xprt->rx_ia;
 464
 465         ia->ri_id = rpcrdma_create_id(xprt, ia, addr);
 466         if (IS_ERR(ia->ri_id)) {
 467                 rc = PTR_ERR(ia->ri_id);
 468                 goto out1;
 469         }
 470
 471         ia->ri_pd = ib_alloc_pd(ia->ri_id->device);
 472         if (IS_ERR(ia->ri_pd)) {
 473                 rc = PTR_ERR(ia->ri_pd);
 474                 dprintk("RPC:       %s: ib_alloc_pd() failed %i\n",
 475                         __func__, rc);
 476                 goto out2;
 477         }
 478
 479         /*
 480          * Query the device to determine if the requested memory
 481          * registration strategy is supported. If it isn't, set the
 482          * strategy to a globally supported model.
 483          */
 484         rc = ib_query_device(ia->ri_id->device, &devattr);
 485         if (rc) {
 486                 dprintk("RPC:       %s: ib_query_device failed %d\n",
 487                         __func__, rc);
 488                 goto out2;
 489         }
 490
 491         if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
 492                 ia->ri_have_dma_lkey = 1;
 493                 ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey;
 494         }
 495
 496         switch (memreg) {
 497         case RPCRDMA_MEMWINDOWS:
 498         case RPCRDMA_MEMWINDOWS_ASYNC:
 499                 if (!(devattr.device_cap_flags & IB_DEVICE_MEM_WINDOW)) {
 500                         dprintk("RPC:       %s: MEMWINDOWS registration "
 501                                 "specified but not supported by adapter, "
 502                                 "using slower RPCRDMA_REGISTER\n",
 503                                 __func__);
 504                         memreg = RPCRDMA_REGISTER;
 505                 }
 506                 break;
 507         case RPCRDMA_MTHCAFMR:
 508                 if (!ia->ri_id->device->alloc_fmr) {
 509 #if RPCRDMA_PERSISTENT_REGISTRATION
 510                         dprintk("RPC:       %s: MTHCAFMR registration "
 511                                 "specified but not supported by adapter, "
 512                                 "using riskier RPCRDMA_ALLPHYSICAL\n",
 513                                 __func__);
 514                         memreg = RPCRDMA_ALLPHYSICAL;
 515 #else
 516                         dprintk("RPC:       %s: MTHCAFMR registration "
 517                                 "specified but not supported by adapter, "
 518                                 "using slower RPCRDMA_REGISTER\n",
 519                                 __func__);
 520                         memreg = RPCRDMA_REGISTER;
 521 #endif
 522                 }
 523                 break;
 524         case RPCRDMA_FRMR:
 525                 /* Requires both frmr reg and local dma lkey */
 526                 if ((devattr.device_cap_flags &
 527                      (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) !=
 528                     (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) {
 529 #if RPCRDMA_PERSISTENT_REGISTRATION
 530                         dprintk("RPC:       %s: FRMR registration "
 531                                 "specified but not supported by adapter, "
 532                                 "using riskier RPCRDMA_ALLPHYSICAL\n",
 533                                 __func__);
 534                         memreg = RPCRDMA_ALLPHYSICAL;
 535 #else
 536                         dprintk("RPC:       %s: FRMR registration "
 537                                 "specified but not supported by adapter, "
 538                                 "using slower RPCRDMA_REGISTER\n",
 539                                 __func__);
 540                         memreg = RPCRDMA_REGISTER;
 541 #endif
 542                 }
 543                 break;
 544         }
 545
 546         /*
 547          * Optionally obtain an underlying physical identity mapping in
 548          * order to do a memory window-based bind. This base registration
 549          * is protected from remote access - that is enabled only by binding
 550          * for the specific bytes targeted during each RPC operation, and
 551          * revoked after the corresponding completion similar to a storage
 552          * adapter.
 553          */
 554         switch (memreg) {
 555         case RPCRDMA_BOUNCEBUFFERS:
 556         case RPCRDMA_REGISTER:
 557         case RPCRDMA_FRMR:
 558                 break;
 559 #if RPCRDMA_PERSISTENT_REGISTRATION
 560         case RPCRDMA_ALLPHYSICAL:
 561                 mem_priv = IB_ACCESS_LOCAL_WRITE |
 562                                 IB_ACCESS_REMOTE_WRITE |
 563                                 IB_ACCESS_REMOTE_READ;
 564                 goto register_setup;
 565 #endif
 566         case RPCRDMA_MEMWINDOWS_ASYNC:
 567         case RPCRDMA_MEMWINDOWS:
 568                 mem_priv = IB_ACCESS_LOCAL_WRITE |
 569                                 IB_ACCESS_MW_BIND;
 570                 goto register_setup;
 571         case RPCRDMA_MTHCAFMR:
 572                 if (ia->ri_have_dma_lkey)
 573                         break;
 574                 mem_priv = IB_ACCESS_LOCAL_WRITE;
 575         register_setup:
 576                 ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv);
 577                 if (IS_ERR(ia->ri_bind_mem)) {
 578                         printk(KERN_ALERT "%s: ib_get_dma_mr for "
 579                                 "phys register failed with %lX\n\t"
 580                                 "Will continue with degraded performance\n",
 581                                 __func__, PTR_ERR(ia->ri_bind_mem));
 582                         memreg = RPCRDMA_REGISTER;
 583                         ia->ri_bind_mem = NULL;
 584                 }
 585                 break;
 586         default:
 587                 printk(KERN_ERR "%s: invalid memory registration mode %d\n",
 588                                 __func__, memreg);
 589                 rc = -EINVAL;
 590                 goto out2;
 591         }
 592         dprintk("RPC:       %s: memory registration strategy is %d\n",
 593                 __func__, memreg);
 594
 595         /* Else will do memory reg/dereg for each chunk */
 596         ia->ri_memreg_strategy = memreg;
 597
 598         return 0;
 599 out2:
 600         rdma_destroy_id(ia->ri_id);
 601         ia->ri_id = NULL;
 602 out1:
 603         return rc;
 604 }
 605
 606 /*
 607  * Clean up/close an IA.
 608  *   o if event handles and PD have been initialized, free them.
 609  *   o close the IA
 610  */
 611 void
 612 rpcrdma_ia_close(struct rpcrdma_ia *ia)
 613 {
 614         int rc;
 615
 616         dprintk("RPC:       %s: entering\n", __func__);
 617         if (ia->ri_bind_mem != NULL) {
 618                 rc = ib_dereg_mr(ia->ri_bind_mem);
 619                 dprintk("RPC:       %s: ib_dereg_mr returned %i\n",
 620                         __func__, rc);
 621         }
 622         if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) {
 623                 if (ia->ri_id->qp)
 624                         rdma_destroy_qp(ia->ri_id);
 625                 rdma_destroy_id(ia->ri_id);
 626                 ia->ri_id = NULL;
 627         }
 628         if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) {
 629                 rc = ib_dealloc_pd(ia->ri_pd);
 630                 dprintk("RPC:       %s: ib_dealloc_pd returned %i\n",
 631                         __func__, rc);
 632         }
 633 }
 634
 635 /*
 636  * Create unconnected endpoint.
 637  */
 638 int
 639 rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 640                                 struct rpcrdma_create_data_internal *cdata)
 641 {
 642         struct ib_device_attr devattr;
 643         int rc, err;
 644
 645         rc = ib_query_device(ia->ri_id->device, &devattr);
 646         if (rc) {
 647                 dprintk("RPC:       %s: ib_query_device failed %d\n",
 648                         __func__, rc);
 649                 return rc;
 650         }
 651
 652         /* check provider's send/recv wr limits */
 653         if (cdata->max_requests > devattr.max_qp_wr)
 654                 cdata->max_requests = devattr.max_qp_wr;
 655
 656         ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall;
 657         ep->rep_attr.qp_context = ep;
 658         /* send_cq and recv_cq initialized below */
 659         ep->rep_attr.srq = NULL;
 660         ep->rep_attr.cap.max_send_wr = cdata->max_requests;
 661         switch (ia->ri_memreg_strategy) {
 662         case RPCRDMA_FRMR:
 663                 /* Add room for frmr register and invalidate WRs.
 664                  * 1. FRMR reg WR for head
 665                  * 2. FRMR invalidate WR for head
 666                  * 3. FRMR reg WR for pagelist
 667                  * 4. FRMR invalidate WR for pagelist
 668                  * 5. FRMR reg WR for tail
 669                  * 6. FRMR invalidate WR for tail
 670                  * 7. The RDMA_SEND WR
 671                  */
 672                 ep->rep_attr.cap.max_send_wr *= 7;
 673                 if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr) {
 674                         cdata->max_requests = devattr.max_qp_wr / 7;
 675                         if (!cdata->max_requests)
 676                                 return -EINVAL;
 677                         ep->rep_attr.cap.max_send_wr = cdata->max_requests * 7;
 678                 }
 679                 break;
 680         case RPCRDMA_MEMWINDOWS_ASYNC:
 681         case RPCRDMA_MEMWINDOWS:
 682                 /* Add room for mw_binds+unbinds - overkill! */
 683                 ep->rep_attr.cap.max_send_wr++;
 684                 ep->rep_attr.cap.max_send_wr *= (2 * RPCRDMA_MAX_SEGS);
 685                 if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr)
 686                         return -EINVAL;
 687                 break;
 688         default:
 689                 break;
 690         }
 691         ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
 692         ep->rep_attr.cap.max_send_sge = (cdata->padding ? 4 : 2);
 693         ep->rep_attr.cap.max_recv_sge = 1;
 694         ep->rep_attr.cap.max_inline_data = 0;
 695         ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 696         ep->rep_attr.qp_type = IB_QPT_RC;
 697         ep->rep_attr.port_num = ~0;
 698
 699         dprintk("RPC:       %s: requested max: dtos: send %d recv %d; "
 700                 "iovs: send %d recv %d\n",
 701                 __func__,
 702                 ep->rep_attr.cap.max_send_wr,
 703                 ep->rep_attr.cap.max_recv_wr,
 704                 ep->rep_attr.cap.max_send_sge,
 705                 ep->rep_attr.cap.max_recv_sge);
 706
 707         /* set trigger for requesting send completion */
 708         ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 /*  - 1*/;
 709         switch (ia->ri_memreg_strategy) {
 710         case RPCRDMA_MEMWINDOWS_ASYNC:
 711         case RPCRDMA_MEMWINDOWS:
 712                 ep->rep_cqinit -= RPCRDMA_MAX_SEGS;
 713                 break;
 714         default:
 715                 break;
 716         }
 717         if (ep->rep_cqinit <= 2)
 718                 ep->rep_cqinit = 0;
 719         INIT_CQCOUNT(ep);
 720         ep->rep_ia = ia;
 721         init_waitqueue_head(&ep->rep_connect_wait);
 722
 723         /*
 724          * Create a single cq for receive dto and mw_bind (only ever
 725          * care about unbind, really). Send completions are suppressed.
 726          * Use single threaded tasklet upcalls to maintain ordering.
 727          */
 728         ep->rep_cq = ib_create_cq(ia->ri_id->device, rpcrdma_cq_event_upcall,
 729                                   rpcrdma_cq_async_error_upcall, NULL,
 730                                   ep->rep_attr.cap.max_recv_wr +
 731                                   ep->rep_attr.cap.max_send_wr + 1, 0);
 732         if (IS_ERR(ep->rep_cq)) {
 733                 rc = PTR_ERR(ep->rep_cq);
 734                 dprintk("RPC:       %s: ib_create_cq failed: %i\n",
 735                         __func__, rc);
 736                 goto out1;
 737         }
 738
 739         rc = ib_req_notify_cq(ep->rep_cq, IB_CQ_NEXT_COMP);
 740         if (rc) {
 741                 dprintk("RPC:       %s: ib_req_notify_cq failed: %i\n",
 742                         __func__, rc);
 743                 goto out2;
 744         }
 745
 746         ep->rep_attr.send_cq = ep->rep_cq;
 747         ep->rep_attr.recv_cq = ep->rep_cq;
 748
 749         /* Initialize cma parameters */
 750
 751         /* RPC/RDMA does not use private data */
 752         ep->rep_remote_cma.private_data = NULL;
 753         ep->rep_remote_cma.private_data_len = 0;
 754
 755         /* Client offers RDMA Read but does not initiate */
 756         ep->rep_remote_cma.initiator_depth = 0;
 757         if (ia->ri_memreg_strategy == RPCRDMA_BOUNCEBUFFERS)
 758                 ep->rep_remote_cma.responder_resources = 0;
 759         else if (devattr.max_qp_rd_atom > 32)   /* arbitrary but <= 255 */
 760                 ep->rep_remote_cma.responder_resources = 32;
 761         else
 762                 ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom;
 763
 764         ep->rep_remote_cma.retry_count = 7;
 765         ep->rep_remote_cma.flow_control = 0;
 766         ep->rep_remote_cma.rnr_retry_count = 0;
 767
 768         return 0;
 769
 770 out2:
 771         err = ib_destroy_cq(ep->rep_cq);
 772         if (err)
 773                 dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
 774                         __func__, err);
 775 out1:
 776         return rc;
 777 }
 778
 779 /*
 780  * rpcrdma_ep_destroy
 781  *
 782  * Disconnect and destroy endpoint. After this, the only
 783  * valid operations on the ep are to free it (if dynamically
 784  * allocated) or re-create it.
 785  *
 786  * The caller's error handling must be sure to not leak the endpoint
 787  * if this function fails.
 788  */
 789 int
 790 rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 791 {
 792         int rc;
 793
 794         dprintk("RPC:       %s: entering, connected is %d\n",
 795                 __func__, ep->rep_connected);
 796
 797         if (ia->ri_id->qp) {
 798                 rc = rpcrdma_ep_disconnect(ep, ia);
 799                 if (rc)
 800                         dprintk("RPC:       %s: rpcrdma_ep_disconnect"
 801                                 " returned %i\n", __func__, rc);
 802                 rdma_destroy_qp(ia->ri_id);
 803                 ia->ri_id->qp = NULL;
 804         }
 805
 806         /* padding - could be done in rpcrdma_buffer_destroy... */
 807         if (ep->rep_pad_mr) {
 808                 rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad);
 809                 ep->rep_pad_mr = NULL;
 810         }
 811
 812         rpcrdma_clean_cq(ep->rep_cq);
 813         rc = ib_destroy_cq(ep->rep_cq);
 814         if (rc)
 815                 dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
 816                         __func__, rc);
 817
 818         return rc;
 819 }
 820
 821 /*
 822  * Connect unconnected endpoint.
 823  */
 824 int
 825 rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 826 {
 827         struct rdma_cm_id *id;
 828         int rc = 0;
 829         int retry_count = 0;
 830
 831         if (ep->rep_connected != 0) {
 832                 struct rpcrdma_xprt *xprt;
 833 retry:
 834                 rc = rpcrdma_ep_disconnect(ep, ia);
 835                 if (rc && rc != -ENOTCONN)
 836                         dprintk("RPC:       %s: rpcrdma_ep_disconnect"
 837                                 " status %i\n", __func__, rc);
 838                 rpcrdma_clean_cq(ep->rep_cq);
 839
 840                 xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);
 841                 id = rpcrdma_create_id(xprt, ia,
 842                                 (struct sockaddr *)&xprt->rx_data.addr);
 843                 if (IS_ERR(id)) {
 844                         rc = PTR_ERR(id);
 845                         goto out;
 846                 }
 847                 /* TEMP TEMP TEMP - fail if new device:
 848                  * Deregister/remarshal *all* requests!
 849                  * Close and recreate adapter, pd, etc!
 850                  * Re-determine all attributes still sane!
 851                  * More stuff I haven't thought of!
 852                  * Rrrgh!
 853                  */
 854                 if (ia->ri_id->device != id->device) {
 855                         printk("RPC:       %s: can't reconnect on "
 856                                 "different device!\n", __func__);
 857                         rdma_destroy_id(id);
 858                         rc = -ENETDOWN;
 859                         goto out;
 860                 }
 861                 /* END TEMP */
 862                 rdma_destroy_qp(ia->ri_id);
 863                 rdma_destroy_id(ia->ri_id);
 864                 ia->ri_id = id;
 865         }
 866
 867         rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);
 868         if (rc) {
 869                 dprintk("RPC:       %s: rdma_create_qp failed %i\n",
 870                         __func__, rc);
 871                 goto out;
 872         }
 873
 874 /* XXX Tavor device performs badly with 2K MTU! */
 875 if (strnicmp(ia->ri_id->device->dma_device->bus->name, "pci", 3) == 0) {
 876         struct pci_dev *pcid = to_pci_dev(ia->ri_id->device->dma_device);
 877         if (pcid->device == PCI_DEVICE_ID_MELLANOX_TAVOR &&
 878             (pcid->vendor == PCI_VENDOR_ID_MELLANOX ||
 879              pcid->vendor == PCI_VENDOR_ID_TOPSPIN)) {
 880                 struct ib_qp_attr attr = {
 881                         .path_mtu = IB_MTU_1024
 882                 };
 883                 rc = ib_modify_qp(ia->ri_id->qp, &attr, IB_QP_PATH_MTU);
 884         }
 885 }
 886
 887         ep->rep_connected = 0;
 888
 889         rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);
 890         if (rc) {
 891                 dprintk("RPC:       %s: rdma_connect() failed with %i\n",
 892                                 __func__, rc);
 893                 goto out;
 894         }
 895
 896         wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0);
 897
 898         /*
 899          * Check state. A non-peer reject indicates no listener
 900          * (ECONNREFUSED), which may be a transient state. All
 901          * others indicate a transport condition which has already
 902          * undergone a best-effort.
 903          */
 904         if (ep->rep_connected == -ECONNREFUSED &&
 905             ++retry_count <= RDMA_CONNECT_RETRY_MAX) {
 906                 dprintk("RPC:       %s: non-peer_reject, retry\n", __func__);
 907                 goto retry;
 908         }
 909         if (ep->rep_connected <= 0) {
 910                 /* Sometimes, the only way to reliably connect to remote
 911                  * CMs is to use same nonzero values for ORD and IRD. */
 912                 if (retry_count++ <= RDMA_CONNECT_RETRY_MAX + 1 &&
 913                     (ep->rep_remote_cma.responder_resources == 0 ||
 914                      ep->rep_remote_cma.initiator_depth !=
 915                                 ep->rep_remote_cma.responder_resources)) {
 916                         if (ep->rep_remote_cma.responder_resources == 0)
 917                                 ep->rep_remote_cma.responder_resources = 1;
 918                         ep->rep_remote_cma.initiator_depth =
 919                                 ep->rep_remote_cma.responder_resources;
 920                         goto retry;
 921                 }
 922                 rc = ep->rep_connected;
 923         } else {
 924                 dprintk("RPC:       %s: connected\n", __func__);
 925         }
 926
 927 out:
 928         if (rc)
 929                 ep->rep_connected = rc;
 930         return rc;
 931 }
 932
 933 /*
 934  * rpcrdma_ep_disconnect
 935  *
 936  * This is separate from destroy to facilitate the ability
 937  * to reconnect without recreating the endpoint.
 938  *
 939  * This call is not reentrant, and must not be made in parallel
 940  * on the same endpoint.
 941  */
 942 int
 943 rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 944 {
 945         int rc;
 946
 947         rpcrdma_clean_cq(ep->rep_cq);
 948         rc = rdma_disconnect(ia->ri_id);
 949         if (!rc) {
 950                 /* returns without wait if not connected */
 951                 wait_event_interruptible(ep->rep_connect_wait,
 952                                                         ep->rep_connected != 1);
 953                 dprintk("RPC:       %s: after wait, %sconnected\n", __func__,
 954                         (ep->rep_connected == 1) ? "still " : "dis");
 955         } else {
 956                 dprintk("RPC:       %s: rdma_disconnect %i\n", __func__, rc);
 957                 ep->rep_connected = rc;
 958         }
 959         return rc;
 960 }
 961
 962 /*
 963  * Initialize buffer memory
 964  */
 965 int
 966 rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
 967         struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata)
 968 {
 969         char *p;
 970         size_t len;
 971         int i, rc;
 972         struct rpcrdma_mw *r;
 973
 974         buf->rb_max_requests = cdata->max_requests;
 975         spin_lock_init(&buf->rb_lock);
 976         atomic_set(&buf->rb_credits, 1);
 977
 978         /* Need to allocate:
 979          *   1.  arrays for send and recv pointers
 980          *   2.  arrays of struct rpcrdma_req to fill in pointers
 981          *   3.  array of struct rpcrdma_rep for replies
 982          *   4.  padding, if any
 983          *   5.  mw's, fmr's or frmr's, if any
 984          * Send/recv buffers in req/rep need to be registered
 985          */
 986
 987         len = buf->rb_max_requests *
 988                 (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *));
 989         len += cdata->padding;
 990         switch (ia->ri_memreg_strategy) {
 991         case RPCRDMA_FRMR:
 992                 len += buf->rb_max_requests * RPCRDMA_MAX_SEGS *
 993                                 sizeof(struct rpcrdma_mw);
 994                 break;
 995         case RPCRDMA_MTHCAFMR:
 996                 /* TBD we are perhaps overallocating here */
 997                 len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS *
 998                                 sizeof(struct rpcrdma_mw);
 999                 break;
1000         case RPCRDMA_MEMWINDOWS_ASYNC:
1001         case RPCRDMA_MEMWINDOWS:
1002                 len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS *
1003                                 sizeof(struct rpcrdma_mw);
1004                 break;
1005         default:
1006                 break;
1007         }
1008
1009         /* allocate 1, 4 and 5 in one shot */
1010         p = kzalloc(len, GFP_KERNEL);
1011         if (p == NULL) {
1012                 dprintk("RPC:       %s: req_t/rep_t/pad kzalloc(%zd) failed\n",
1013                         __func__, len);
1014                 rc = -ENOMEM;
1015                 goto out;
1016         }
1017         buf->rb_pool = p;       /* for freeing it later */
1018
1019         buf->rb_send_bufs = (struct rpcrdma_req **) p;
1020         p = (char *) &buf->rb_send_bufs[buf->rb_max_requests];
1021         buf->rb_recv_bufs = (struct rpcrdma_rep **) p;
1022         p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests];
1023
1024         /*
1025          * Register the zeroed pad buffer, if any.
1026          */
1027         if (cdata->padding) {
1028                 rc = rpcrdma_register_internal(ia, p, cdata->padding,
1029                                             &ep->rep_pad_mr, &ep->rep_pad);
1030                 if (rc)
1031                         goto out;
1032         }
1033         p += cdata->padding;
1034
1035         /*
1036          * Allocate the fmr's, or mw's for mw_bind chunk registration.
1037          * We "cycle" the mw's in order to minimize rkey reuse,
1038          * and also reduce unbind-to-bind collision.
1039          */
1040         INIT_LIST_HEAD(&buf->rb_mws);
1041         r = (struct rpcrdma_mw *)p;
1042         switch (ia->ri_memreg_strategy) {
1043         case RPCRDMA_FRMR:
1044                 for (i = buf->rb_max_requests * RPCRDMA_MAX_SEGS; i; i--) {
1045                         r->r.frmr.fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd,
1046                                                          RPCRDMA_MAX_SEGS);
1047                         if (IS_ERR(r->r.frmr.fr_mr)) {
1048                                 rc = PTR_ERR(r->r.frmr.fr_mr);
1049                                 dprintk("RPC:       %s: ib_alloc_fast_reg_mr"
1050                                         " failed %i\n", __func__, rc);
1051                                 goto out;
1052                         }
1053                         r->r.frmr.fr_pgl =
1054                                 ib_alloc_fast_reg_page_list(ia->ri_id->device,
1055                                                             RPCRDMA_MAX_SEGS);
1056                         if (IS_ERR(r->r.frmr.fr_pgl)) {
1057                                 rc = PTR_ERR(r->r.frmr.fr_pgl);
1058                                 dprintk("RPC:       %s: "
1059                                         "ib_alloc_fast_reg_page_list "
1060                                         "failed %i\n", __func__, rc);
1061                                 goto out;
1062                         }
1063                         list_add(&r->mw_list, &buf->rb_mws);
1064                         ++r;
1065                 }
1066                 break;
1067         case RPCRDMA_MTHCAFMR:
1068                 /* TBD we are perhaps overallocating here */
1069                 for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
1070                         static struct ib_fmr_attr fa =
1071                                 { RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT };
1072                         r->r.fmr = ib_alloc_fmr(ia->ri_pd,
1073                                 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ,
1074                                 &fa);
1075                         if (IS_ERR(r->r.fmr)) {
1076                                 rc = PTR_ERR(r->r.fmr);
1077                                 dprintk("RPC:       %s: ib_alloc_fmr"
1078                                         " failed %i\n", __func__, rc);
1079                                 goto out;
1080                         }
1081                         list_add(&r->mw_list, &buf->rb_mws);
1082                         ++r;
1083                 }
1084                 break;
1085         case RPCRDMA_MEMWINDOWS_ASYNC:
1086         case RPCRDMA_MEMWINDOWS:
1087                 /* Allocate one extra request's worth, for full cycling */
1088                 for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
1089                         r->r.mw = ib_alloc_mw(ia->ri_pd, IB_MW_TYPE_1);
1090                         if (IS_ERR(r->r.mw)) {
1091                                 rc = PTR_ERR(r->r.mw);
1092                                 dprintk("RPC:       %s: ib_alloc_mw"
1093                                         " failed %i\n", __func__, rc);
1094                                 goto out;
1095                         }
1096                         list_add(&r->mw_list, &buf->rb_mws);
1097                         ++r;
1098                 }
1099                 break;
1100         default:
1101                 break;
1102         }
1103
1104         /*
1105          * Allocate/init the request/reply buffers. Doing this
1106          * using kmalloc for now -- one for each buf.
1107          */
1108         for (i = 0; i < buf->rb_max_requests; i++) {
1109                 struct rpcrdma_req *req;
1110                 struct rpcrdma_rep *rep;
1111
1112                 len = cdata->inline_wsize + sizeof(struct rpcrdma_req);
1113                 /* RPC layer requests *double* size + 1K RPC_SLACK_SPACE! */
1114                 /* Typical ~2400b, so rounding up saves work later */
1115                 if (len < 4096)
1116                         len = 4096;
1117                 req = kmalloc(len, GFP_KERNEL);
1118                 if (req == NULL) {
1119                         dprintk("RPC:       %s: request buffer %d alloc"
1120                                 " failed\n", __func__, i);
1121                         rc = -ENOMEM;
1122                         goto out;
1123                 }
1124                 memset(req, 0, sizeof(struct rpcrdma_req));
1125                 buf->rb_send_bufs[i] = req;
1126                 buf->rb_send_bufs[i]->rl_buffer = buf;
1127
1128                 rc = rpcrdma_register_internal(ia, req->rl_base,
1129                                 len - offsetof(struct rpcrdma_req, rl_base),
1130                                 &buf->rb_send_bufs[i]->rl_handle,
1131                                 &buf->rb_send_bufs[i]->rl_iov);
1132                 if (rc)
1133                         goto out;
1134
1135                 buf->rb_send_bufs[i]->rl_size = len-sizeof(struct rpcrdma_req);
1136
1137                 len = cdata->inline_rsize + sizeof(struct rpcrdma_rep);
1138                 rep = kmalloc(len, GFP_KERNEL);
1139                 if (rep == NULL) {
1140                         dprintk("RPC:       %s: reply buffer %d alloc failed\n",
1141                                 __func__, i);
1142                         rc = -ENOMEM;
1143                         goto out;
1144                 }
1145                 memset(rep, 0, sizeof(struct rpcrdma_rep));
1146                 buf->rb_recv_bufs[i] = rep;
1147                 buf->rb_recv_bufs[i]->rr_buffer = buf;
1148                 init_waitqueue_head(&rep->rr_unbind);
1149
1150                 rc = rpcrdma_register_internal(ia, rep->rr_base,
1151                                 len - offsetof(struct rpcrdma_rep, rr_base),
1152                                 &buf->rb_recv_bufs[i]->rr_handle,
1153                                 &buf->rb_recv_bufs[i]->rr_iov);
1154                 if (rc)
1155                         goto out;
1156
1157         }
1158         dprintk("RPC:       %s: max_requests %d\n",
1159                 __func__, buf->rb_max_requests);
1160         /* done */
1161         return 0;
1162 out:
1163         rpcrdma_buffer_destroy(buf);
1164         return rc;
1165 }
1166
1167 /*
1168  * Unregister and destroy buffer memory. Need to deal with
1169  * partial initialization, so it's callable from failed create.
1170  * Must be called before destroying endpoint, as registrations
1171  * reference it.
1172  */
1173 void
1174 rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
1175 {
1176         int rc, i;
1177         struct rpcrdma_ia *ia = rdmab_to_ia(buf);
1178         struct rpcrdma_mw *r;
1179
1180         /* clean up in reverse order from create
1181          *   1.  recv mr memory (mr free, then kfree)
1182          *   1a. bind mw memory
1183          *   2.  send mr memory (mr free, then kfree)
1184          *   3.  padding (if any) [moved to rpcrdma_ep_destroy]
1185          *   4.  arrays
1186          */
1187         dprintk("RPC:       %s: entering\n", __func__);
1188
1189         for (i = 0; i < buf->rb_max_requests; i++) {
1190                 if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) {
1191                         rpcrdma_deregister_internal(ia,
1192                                         buf->rb_recv_bufs[i]->rr_handle,
1193                                         &buf->rb_recv_bufs[i]->rr_iov);
1194                         kfree(buf->rb_recv_bufs[i]);
1195                 }
1196                 if (buf->rb_send_bufs && buf->rb_send_bufs[i]) {
1197                         while (!list_empty(&buf->rb_mws)) {
1198                                 r = list_entry(buf->rb_mws.next,
1199                                         struct rpcrdma_mw, mw_list);
1200                                 list_del(&r->mw_list);
1201                                 switch (ia->ri_memreg_strategy) {
1202                                 case RPCRDMA_FRMR:
1203                                         rc = ib_dereg_mr(r->r.frmr.fr_mr);
1204                                         if (rc)
1205                                                 dprintk("RPC:       %s:"
1206                                                         " ib_dereg_mr"
1207                                                         " failed %i\n",
1208                                                         __func__, rc);
1209                                         ib_free_fast_reg_page_list(r->r.frmr.fr_pgl);
1210                                         break;
1211                                 case RPCRDMA_MTHCAFMR:
1212                                         rc = ib_dealloc_fmr(r->r.fmr);
1213                                         if (rc)
1214                                                 dprintk("RPC:       %s:"
1215                                                         " ib_dealloc_fmr"
1216                                                         " failed %i\n",
1217                                                         __func__, rc);
1218                                         break;
1219                                 case RPCRDMA_MEMWINDOWS_ASYNC:
1220                                 case RPCRDMA_MEMWINDOWS:
1221                                         rc = ib_dealloc_mw(r->r.mw);
1222                                         if (rc)
1223                                                 dprintk("RPC:       %s:"
1224                                                         " ib_dealloc_mw"
1225                                                         " failed %i\n",
1226                                                         __func__, rc);
1227                                         break;
1228                                 default:
1229                                         break;
1230                                 }
1231                         }
1232                         rpcrdma_deregister_internal(ia,
1233                                         buf->rb_send_bufs[i]->rl_handle,
1234                                         &buf->rb_send_bufs[i]->rl_iov);
1235                         kfree(buf->rb_send_bufs[i]);
1236                 }
1237         }
1238
1239         kfree(buf->rb_pool);
1240 }
1241
1242 /*
1243  * Get a set of request/reply buffers.
1244  *
1245  * Reply buffer (if needed) is attached to send buffer upon return.
1246  * Rule:
1247  *    rb_send_index and rb_recv_index MUST always be pointing to the
1248  *    *next* available buffer (non-NULL). They are incremented after
1249  *    removing buffers, and decremented *before* returning them.
1250  */
1251 struct rpcrdma_req *
1252 rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
1253 {
1254         struct rpcrdma_req *req;
1255         unsigned long flags;
1256         int i;
1257         struct rpcrdma_mw *r;
1258
1259         spin_lock_irqsave(&buffers->rb_lock, flags);
1260         if (buffers->rb_send_index == buffers->rb_max_requests) {
1261                 spin_unlock_irqrestore(&buffers->rb_lock, flags);
1262                 dprintk("RPC:       %s: out of request buffers\n", __func__);
1263                 return ((struct rpcrdma_req *)NULL);
1264         }
1265
1266         req = buffers->rb_send_bufs[buffers->rb_send_index];
1267         if (buffers->rb_send_index < buffers->rb_recv_index) {
1268                 dprintk("RPC:       %s: %d extra receives outstanding (ok)\n",
1269                         __func__,
1270                         buffers->rb_recv_index - buffers->rb_send_index);
1271                 req->rl_reply = NULL;
1272         } else {
1273                 req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index];
1274                 buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL;
1275         }
1276         buffers->rb_send_bufs[buffers->rb_send_index++] = NULL;
1277         if (!list_empty(&buffers->rb_mws)) {
1278                 i = RPCRDMA_MAX_SEGS - 1;
1279                 do {
1280                         r = list_entry(buffers->rb_mws.next,
1281                                         struct rpcrdma_mw, mw_list);
1282                         list_del(&r->mw_list);
1283                         req->rl_segments[i].mr_chunk.rl_mw = r;
1284                 } while (--i >= 0);
1285         }
1286         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1287         return req;
1288 }
1289
1290 /*
1291  * Put request/reply buffers back into pool.
1292  * Pre-decrement counter/array index.
1293  */
1294 void
1295 rpcrdma_buffer_put(struct rpcrdma_req *req)
1296 {
1297         struct rpcrdma_buffer *buffers = req->rl_buffer;
1298         struct rpcrdma_ia *ia = rdmab_to_ia(buffers);
1299         int i;
1300         unsigned long flags;
1301
1302         BUG_ON(req->rl_nchunks != 0);
1303         spin_lock_irqsave(&buffers->rb_lock, flags);
1304         buffers->rb_send_bufs[--buffers->rb_send_index] = req;
1305         req->rl_niovs = 0;
1306         if (req->rl_reply) {
1307                 buffers->rb_recv_bufs[--buffers->rb_recv_index] = req->rl_reply;
1308                 init_waitqueue_head(&req->rl_reply->rr_unbind);
1309                 req->rl_reply->rr_func = NULL;
1310                 req->rl_reply = NULL;
1311         }
1312         switch (ia->ri_memreg_strategy) {
1313         case RPCRDMA_FRMR:
1314         case RPCRDMA_MTHCAFMR:
1315         case RPCRDMA_MEMWINDOWS_ASYNC:
1316         case RPCRDMA_MEMWINDOWS:
1317                 /*
1318                  * Cycle mw's back in reverse order, and "spin" them.
1319                  * This delays and scrambles reuse as much as possible.
1320                  */
1321                 i = 1;
1322                 do {
1323                         struct rpcrdma_mw **mw;
1324                         mw = &req->rl_segments[i].mr_chunk.rl_mw;
1325                         list_add_tail(&(*mw)->mw_list, &buffers->rb_mws);
1326                         *mw = NULL;
1327                 } while (++i < RPCRDMA_MAX_SEGS);
1328                 list_add_tail(&req->rl_segments[0].mr_chunk.rl_mw->mw_list,
1329                                         &buffers->rb_mws);
1330                 req->rl_segments[0].mr_chunk.rl_mw = NULL;
1331                 break;
1332         default:
1333                 break;
1334         }
1335         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1336 }
1337
1338 /*
1339  * Recover reply buffers from pool.
1340  * This happens when recovering from error conditions.
1341  * Post-increment counter/array index.
1342  */
1343 void
1344 rpcrdma_recv_buffer_get(struct rpcrdma_req *req)
1345 {
1346         struct rpcrdma_buffer *buffers = req->rl_buffer;
1347         unsigned long flags;
1348
1349         if (req->rl_iov.length == 0)    /* special case xprt_rdma_allocate() */
1350                 buffers = ((struct rpcrdma_req *) buffers)->rl_buffer;
1351         spin_lock_irqsave(&buffers->rb_lock, flags);
1352         if (buffers->rb_recv_index < buffers->rb_max_requests) {
1353                 req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index];
1354                 buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL;
1355         }
1356         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1357 }
1358
1359 /*
1360  * Put reply buffers back into pool when not attached to
1361  * request. This happens in error conditions, and when
1362  * aborting unbinds. Pre-decrement counter/array index.
1363  */
1364 void
1365 rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
1366 {
1367         struct rpcrdma_buffer *buffers = rep->rr_buffer;
1368         unsigned long flags;
1369
1370         rep->rr_func = NULL;
1371         spin_lock_irqsave(&buffers->rb_lock, flags);
1372         buffers->rb_recv_bufs[--buffers->rb_recv_index] = rep;
1373         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1374 }
1375
1376 /*
1377  * Wrappers for internal-use kmalloc memory registration, used by buffer code.
1378  */
1379
1380 int
1381 rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len,
1382                                 struct ib_mr **mrp, struct ib_sge *iov)
1383 {
1384         struct ib_phys_buf ipb;
1385         struct ib_mr *mr;
1386         int rc;
1387
1388         /*
1389          * All memory passed here was kmalloc'ed, therefore phys-contiguous.
1390          */
1391         iov->addr = ib_dma_map_single(ia->ri_id->device,
1392                         va, len, DMA_BIDIRECTIONAL);
1393         iov->length = len;
1394
1395         if (ia->ri_have_dma_lkey) {
1396                 *mrp = NULL;
1397                 iov->lkey = ia->ri_dma_lkey;
1398                 return 0;
1399         } else if (ia->ri_bind_mem != NULL) {
1400                 *mrp = NULL;
1401                 iov->lkey = ia->ri_bind_mem->lkey;
1402                 return 0;
1403         }
1404
1405         ipb.addr = iov->addr;
1406         ipb.size = iov->length;
1407         mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1,
1408                         IB_ACCESS_LOCAL_WRITE, &iov->addr);
1409
1410         dprintk("RPC:       %s: phys convert: 0x%llx "
1411                         "registered 0x%llx length %d\n",
1412                         __func__, (unsigned long long)ipb.addr,
1413                         (unsigned long long)iov->addr, len);
1414
1415         if (IS_ERR(mr)) {
1416                 *mrp = NULL;
1417                 rc = PTR_ERR(mr);
1418                 dprintk("RPC:       %s: failed with %i\n", __func__, rc);
1419         } else {
1420                 *mrp = mr;
1421                 iov->lkey = mr->lkey;
1422                 rc = 0;
1423         }
1424
1425         return rc;
1426 }
1427
1428 int
1429 rpcrdma_deregister_internal(struct rpcrdma_ia *ia,
1430                                 struct ib_mr *mr, struct ib_sge *iov)
1431 {
1432         int rc;
1433
1434         ib_dma_unmap_single(ia->ri_id->device,
1435                         iov->addr, iov->length, DMA_BIDIRECTIONAL);
1436
1437         if (NULL == mr)
1438                 return 0;
1439
1440         rc = ib_dereg_mr(mr);
1441         if (rc)
1442                 dprintk("RPC:       %s: ib_dereg_mr failed %i\n", __func__, rc);
1443         return rc;
1444 }
1445
1446 /*
1447  * Wrappers for chunk registration, shared by read/write chunk code.
1448  */
1449
1450 static void
1451 rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, int writing)
1452 {
1453         seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1454         seg->mr_dmalen = seg->mr_len;
1455         if (seg->mr_page)
1456                 seg->mr_dma = ib_dma_map_page(ia->ri_id->device,
1457                                 seg->mr_page, offset_in_page(seg->mr_offset),
1458                                 seg->mr_dmalen, seg->mr_dir);
1459         else
1460                 seg->mr_dma = ib_dma_map_single(ia->ri_id->device,
1461                                 seg->mr_offset,
1462                                 seg->mr_dmalen, seg->mr_dir);
1463         if (ib_dma_mapping_error(ia->ri_id->device, seg->mr_dma)) {
1464                 dprintk("RPC:       %s: mr_dma %llx mr_offset %p mr_dma_len %zu\n",
1465                         __func__,
1466                         (unsigned long long)seg->mr_dma,
1467                         seg->mr_offset, seg->mr_dmalen);
1468         }
1469 }
1470
1471 static void
1472 rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg)
1473 {
1474         if (seg->mr_page)
1475                 ib_dma_unmap_page(ia->ri_id->device,
1476                                 seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1477         else
1478                 ib_dma_unmap_single(ia->ri_id->device,
1479                                 seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1480 }
1481
1482 static int
1483 rpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg,
1484                         int *nsegs, int writing, struct rpcrdma_ia *ia,
1485                         struct rpcrdma_xprt *r_xprt)
1486 {
1487         struct rpcrdma_mr_seg *seg1 = seg;
1488         struct ib_send_wr invalidate_wr, frmr_wr, *bad_wr, *post_wr;
1489
1490         u8 key;
1491         int len, pageoff;
1492         int i, rc;
1493         int seg_len;
1494         u64 pa;
1495         int page_no;
1496
1497         pageoff = offset_in_page(seg1->mr_offset);
1498         seg1->mr_offset -= pageoff;     /* start of page */
1499         seg1->mr_len += pageoff;
1500         len = -pageoff;
1501         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1502                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1503         for (page_no = i = 0; i < *nsegs;) {
1504                 rpcrdma_map_one(ia, seg, writing);
1505                 pa = seg->mr_dma;
1506                 for (seg_len = seg->mr_len; seg_len > 0; seg_len -= PAGE_SIZE) {
1507                         seg1->mr_chunk.rl_mw->r.frmr.fr_pgl->
1508                                 page_list[page_no++] = pa;
1509                         pa += PAGE_SIZE;
1510                 }
1511                 len += seg->mr_len;
1512                 ++seg;
1513                 ++i;
1514                 /* Check for holes */
1515                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1516                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
1517                         break;
1518         }
1519         dprintk("RPC:       %s: Using frmr %p to map %d segments\n",
1520                 __func__, seg1->mr_chunk.rl_mw, i);
1521
1522         if (unlikely(seg1->mr_chunk.rl_mw->r.frmr.state == FRMR_IS_VALID)) {
1523                 dprintk("RPC:       %s: frmr %x left valid, posting invalidate.\n",
1524                         __func__,
1525                         seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey);
1526                 /* Invalidate before using. */
1527                 memset(&invalidate_wr, 0, sizeof invalidate_wr);
1528                 invalidate_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw;
1529                 invalidate_wr.next = &frmr_wr;
1530                 invalidate_wr.opcode = IB_WR_LOCAL_INV;
1531                 invalidate_wr.send_flags = IB_SEND_SIGNALED;
1532                 invalidate_wr.ex.invalidate_rkey =
1533                         seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1534                 DECR_CQCOUNT(&r_xprt->rx_ep);
1535                 post_wr = &invalidate_wr;
1536         } else
1537                 post_wr = &frmr_wr;
1538
1539         /* Bump the key */
1540         key = (u8)(seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey & 0x000000FF);
1541         ib_update_fast_reg_key(seg1->mr_chunk.rl_mw->r.frmr.fr_mr, ++key);
1542
1543         /* Prepare FRMR WR */
1544         memset(&frmr_wr, 0, sizeof frmr_wr);
1545         frmr_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw;
1546         frmr_wr.opcode = IB_WR_FAST_REG_MR;
1547         frmr_wr.send_flags = IB_SEND_SIGNALED;
1548         frmr_wr.wr.fast_reg.iova_start = seg1->mr_dma;
1549         frmr_wr.wr.fast_reg.page_list = seg1->mr_chunk.rl_mw->r.frmr.fr_pgl;
1550         frmr_wr.wr.fast_reg.page_list_len = page_no;
1551         frmr_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
1552         frmr_wr.wr.fast_reg.length = page_no << PAGE_SHIFT;
1553         BUG_ON(frmr_wr.wr.fast_reg.length < len);
1554         frmr_wr.wr.fast_reg.access_flags = (writing ?
1555                                 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :
1556                                 IB_ACCESS_REMOTE_READ);
1557         frmr_wr.wr.fast_reg.rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1558         DECR_CQCOUNT(&r_xprt->rx_ep);
1559
1560         rc = ib_post_send(ia->ri_id->qp, post_wr, &bad_wr);
1561
1562         if (rc) {
1563                 dprintk("RPC:       %s: failed ib_post_send for register,"
1564                         " status %i\n", __func__, rc);
1565                 while (i--)
1566                         rpcrdma_unmap_one(ia, --seg);
1567         } else {
1568                 seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1569                 seg1->mr_base = seg1->mr_dma + pageoff;
1570                 seg1->mr_nsegs = i;
1571                 seg1->mr_len = len;
1572         }
1573         *nsegs = i;
1574         return rc;
1575 }
1576
1577 static int
1578 rpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg,
1579                         struct rpcrdma_ia *ia, struct rpcrdma_xprt *r_xprt)
1580 {
1581         struct rpcrdma_mr_seg *seg1 = seg;
1582         struct ib_send_wr invalidate_wr, *bad_wr;
1583         int rc;
1584
1585         while (seg1->mr_nsegs--)
1586                 rpcrdma_unmap_one(ia, seg++);
1587
1588         memset(&invalidate_wr, 0, sizeof invalidate_wr);
1589         invalidate_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw;
1590         invalidate_wr.opcode = IB_WR_LOCAL_INV;
1591         invalidate_wr.send_flags = IB_SEND_SIGNALED;
1592         invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1593         DECR_CQCOUNT(&r_xprt->rx_ep);
1594
1595         rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr);
1596         if (rc)
1597                 dprintk("RPC:       %s: failed ib_post_send for invalidate,"
1598                         " status %i\n", __func__, rc);
1599         return rc;
1600 }
1601
1602 static int
1603 rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,
1604                         int *nsegs, int writing, struct rpcrdma_ia *ia)
1605 {
1606         struct rpcrdma_mr_seg *seg1 = seg;
1607         u64 physaddrs[RPCRDMA_MAX_DATA_SEGS];
1608         int len, pageoff, i, rc;
1609
1610         pageoff = offset_in_page(seg1->mr_offset);
1611         seg1->mr_offset -= pageoff;     /* start of page */
1612         seg1->mr_len += pageoff;
1613         len = -pageoff;
1614         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1615                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1616         for (i = 0; i < *nsegs;) {
1617                 rpcrdma_map_one(ia, seg, writing);
1618                 physaddrs[i] = seg->mr_dma;
1619                 len += seg->mr_len;
1620                 ++seg;
1621                 ++i;
1622                 /* Check for holes */
1623                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1624                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
1625                         break;
1626         }
1627         rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr,
1628                                 physaddrs, i, seg1->mr_dma);
1629         if (rc) {
1630                 dprintk("RPC:       %s: failed ib_map_phys_fmr "
1631                         "%u@0x%llx+%i (%d)... status %i\n", __func__,
1632                         len, (unsigned long long)seg1->mr_dma,
1633                         pageoff, i, rc);
1634                 while (i--)
1635                         rpcrdma_unmap_one(ia, --seg);
1636         } else {
1637                 seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey;
1638                 seg1->mr_base = seg1->mr_dma + pageoff;
1639                 seg1->mr_nsegs = i;
1640                 seg1->mr_len = len;
1641         }
1642         *nsegs = i;
1643         return rc;
1644 }
1645
1646 static int
1647 rpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg,
1648                         struct rpcrdma_ia *ia)
1649 {
1650         struct rpcrdma_mr_seg *seg1 = seg;
1651         LIST_HEAD(l);
1652         int rc;
1653
1654         list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l);
1655         rc = ib_unmap_fmr(&l);
1656         while (seg1->mr_nsegs--)
1657                 rpcrdma_unmap_one(ia, seg++);
1658         if (rc)
1659                 dprintk("RPC:       %s: failed ib_unmap_fmr,"
1660                         " status %i\n", __func__, rc);
1661         return rc;
1662 }
1663
1664 static int
1665 rpcrdma_register_memwin_external(struct rpcrdma_mr_seg *seg,
1666                         int *nsegs, int writing, struct rpcrdma_ia *ia,
1667                         struct rpcrdma_xprt *r_xprt)
1668 {
1669         int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
1670                                   IB_ACCESS_REMOTE_READ);
1671         struct ib_mw_bind param;
1672         int rc;
1673
1674         *nsegs = 1;
1675         rpcrdma_map_one(ia, seg, writing);
1676         param.bind_info.mr = ia->ri_bind_mem;
1677         param.wr_id = 0ULL;     /* no send cookie */
1678         param.bind_info.addr = seg->mr_dma;
1679         param.bind_info.length = seg->mr_len;
1680         param.send_flags = 0;
1681         param.bind_info.mw_access_flags = mem_priv;
1682
1683         DECR_CQCOUNT(&r_xprt->rx_ep);
1684         rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
1685         if (rc) {
1686                 dprintk("RPC:       %s: failed ib_bind_mw "
1687                         "%u@0x%llx status %i\n",
1688                         __func__, seg->mr_len,
1689                         (unsigned long long)seg->mr_dma, rc);
1690                 rpcrdma_unmap_one(ia, seg);
1691         } else {
1692                 seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey;
1693                 seg->mr_base = param.bind_info.addr;
1694                 seg->mr_nsegs = 1;
1695         }
1696         return rc;
1697 }
1698
1699 static int
1700 rpcrdma_deregister_memwin_external(struct rpcrdma_mr_seg *seg,
1701                         struct rpcrdma_ia *ia,
1702                         struct rpcrdma_xprt *r_xprt, void **r)
1703 {
1704         struct ib_mw_bind param;
1705         LIST_HEAD(l);
1706         int rc;
1707
1708         BUG_ON(seg->mr_nsegs != 1);
1709         param.bind_info.mr = ia->ri_bind_mem;
1710         param.bind_info.addr = 0ULL;    /* unbind */
1711         param.bind_info.length = 0;
1712         param.bind_info.mw_access_flags = 0;
1713         if (*r) {
1714                 param.wr_id = (u64) (unsigned long) *r;
1715                 param.send_flags = IB_SEND_SIGNALED;
1716                 INIT_CQCOUNT(&r_xprt->rx_ep);
1717         } else {
1718                 param.wr_id = 0ULL;
1719                 param.send_flags = 0;
1720                 DECR_CQCOUNT(&r_xprt->rx_ep);
1721         }
1722         rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
1723         rpcrdma_unmap_one(ia, seg);
1724         if (rc)
1725                 dprintk("RPC:       %s: failed ib_(un)bind_mw,"
1726                         " status %i\n", __func__, rc);
1727         else
1728                 *r = NULL;      /* will upcall on completion */
1729         return rc;
1730 }
1731
1732 static int
1733 rpcrdma_register_default_external(struct rpcrdma_mr_seg *seg,
1734                         int *nsegs, int writing, struct rpcrdma_ia *ia)
1735 {
1736         int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
1737                                   IB_ACCESS_REMOTE_READ);
1738         struct rpcrdma_mr_seg *seg1 = seg;
1739         struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS];
1740         int len, i, rc = 0;
1741
1742         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1743                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1744         for (len = 0, i = 0; i < *nsegs;) {
1745                 rpcrdma_map_one(ia, seg, writing);
1746                 ipb[i].addr = seg->mr_dma;
1747                 ipb[i].size = seg->mr_len;
1748                 len += seg->mr_len;
1749                 ++seg;
1750                 ++i;
1751                 /* Check for holes */
1752                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1753                     offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len))
1754                         break;
1755         }
1756         seg1->mr_base = seg1->mr_dma;
1757         seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd,
1758                                 ipb, i, mem_priv, &seg1->mr_base);
1759         if (IS_ERR(seg1->mr_chunk.rl_mr)) {
1760                 rc = PTR_ERR(seg1->mr_chunk.rl_mr);
1761                 dprintk("RPC:       %s: failed ib_reg_phys_mr "
1762                         "%u@0x%llx (%d)... status %i\n",
1763                         __func__, len,
1764                         (unsigned long long)seg1->mr_dma, i, rc);
1765                 while (i--)
1766                         rpcrdma_unmap_one(ia, --seg);
1767         } else {
1768                 seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey;
1769                 seg1->mr_nsegs = i;
1770                 seg1->mr_len = len;
1771         }
1772         *nsegs = i;
1773         return rc;
1774 }
1775
1776 static int
1777 rpcrdma_deregister_default_external(struct rpcrdma_mr_seg *seg,
1778                         struct rpcrdma_ia *ia)
1779 {
1780         struct rpcrdma_mr_seg *seg1 = seg;
1781         int rc;
1782
1783         rc = ib_dereg_mr(seg1->mr_chunk.rl_mr);
1784         seg1->mr_chunk.rl_mr = NULL;
1785         while (seg1->mr_nsegs--)
1786                 rpcrdma_unmap_one(ia, seg++);
1787         if (rc)
1788                 dprintk("RPC:       %s: failed ib_dereg_mr,"
1789                         " status %i\n", __func__, rc);
1790         return rc;
1791 }
1792
1793 int
1794 rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
1795                         int nsegs, int writing, struct rpcrdma_xprt *r_xprt)
1796 {
1797         struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1798         int rc = 0;
1799
1800         switch (ia->ri_memreg_strategy) {
1801
1802 #if RPCRDMA_PERSISTENT_REGISTRATION
1803         case RPCRDMA_ALLPHYSICAL:
1804                 rpcrdma_map_one(ia, seg, writing);
1805                 seg->mr_rkey = ia->ri_bind_mem->rkey;
1806                 seg->mr_base = seg->mr_dma;
1807                 seg->mr_nsegs = 1;
1808                 nsegs = 1;
1809                 break;
1810 #endif
1811
1812         /* Registration using frmr registration */
1813         case RPCRDMA_FRMR:
1814                 rc = rpcrdma_register_frmr_external(seg, &nsegs, writing, ia, r_xprt);
1815                 break;
1816
1817         /* Registration using fmr memory registration */
1818         case RPCRDMA_MTHCAFMR:
1819                 rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia);
1820                 break;
1821
1822         /* Registration using memory windows */
1823         case RPCRDMA_MEMWINDOWS_ASYNC:
1824         case RPCRDMA_MEMWINDOWS:
1825                 rc = rpcrdma_register_memwin_external(seg, &nsegs, writing, ia, r_xprt);
1826                 break;
1827
1828         /* Default registration each time */
1829         default:
1830                 rc = rpcrdma_register_default_external(seg, &nsegs, writing, ia);
1831                 break;
1832         }
1833         if (rc)
1834                 return -1;
1835
1836         return nsegs;
1837 }
1838
1839 int
1840 rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg,
1841                 struct rpcrdma_xprt *r_xprt, void *r)
1842 {
1843         struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1844         int nsegs = seg->mr_nsegs, rc;
1845
1846         switch (ia->ri_memreg_strategy) {
1847
1848 #if RPCRDMA_PERSISTENT_REGISTRATION
1849         case RPCRDMA_ALLPHYSICAL:
1850                 BUG_ON(nsegs != 1);
1851                 rpcrdma_unmap_one(ia, seg);
1852                 rc = 0;
1853                 break;
1854 #endif
1855
1856         case RPCRDMA_FRMR:
1857                 rc = rpcrdma_deregister_frmr_external(seg, ia, r_xprt);
1858                 break;
1859
1860         case RPCRDMA_MTHCAFMR:
1861                 rc = rpcrdma_deregister_fmr_external(seg, ia);
1862                 break;
1863
1864         case RPCRDMA_MEMWINDOWS_ASYNC:
1865         case RPCRDMA_MEMWINDOWS:
1866                 rc = rpcrdma_deregister_memwin_external(seg, ia, r_xprt, &r);
1867                 break;
1868
1869         default:
1870                 rc = rpcrdma_deregister_default_external(seg, ia);
1871                 break;
1872         }
1873         if (r) {
1874                 struct rpcrdma_rep *rep = r;
1875                 void (*func)(struct rpcrdma_rep *) = rep->rr_func;
1876                 rep->rr_func = NULL;
1877                 func(rep);      /* dereg done, callback now */
1878         }
1879         return nsegs;
1880 }
1881
1882 /*
1883  * Prepost any receive buffer, then post send.
1884  *
1885  * Receive buffer is donated to hardware, reclaimed upon recv completion.
1886  */
1887 int
1888 rpcrdma_ep_post(struct rpcrdma_ia *ia,
1889                 struct rpcrdma_ep *ep,
1890                 struct rpcrdma_req *req)
1891 {
1892         struct ib_send_wr send_wr, *send_wr_fail;
1893         struct rpcrdma_rep *rep = req->rl_reply;
1894         int rc;
1895
1896         if (rep) {
1897                 rc = rpcrdma_ep_post_recv(ia, ep, rep);
1898                 if (rc)
1899                         goto out;
1900                 req->rl_reply = NULL;
1901         }
1902
1903         send_wr.next = NULL;
1904         send_wr.wr_id = 0ULL;   /* no send cookie */
1905         send_wr.sg_list = req->rl_send_iov;
1906         send_wr.num_sge = req->rl_niovs;
1907         send_wr.opcode = IB_WR_SEND;
1908         if (send_wr.num_sge == 4)       /* no need to sync any pad (constant) */
1909                 ib_dma_sync_single_for_device(ia->ri_id->device,
1910                         req->rl_send_iov[3].addr, req->rl_send_iov[3].length,
1911                         DMA_TO_DEVICE);
1912         ib_dma_sync_single_for_device(ia->ri_id->device,
1913                 req->rl_send_iov[1].addr, req->rl_send_iov[1].length,
1914                 DMA_TO_DEVICE);
1915         ib_dma_sync_single_for_device(ia->ri_id->device,
1916                 req->rl_send_iov[0].addr, req->rl_send_iov[0].length,
1917                 DMA_TO_DEVICE);
1918
1919         if (DECR_CQCOUNT(ep) > 0)
1920                 send_wr.send_flags = 0;
1921         else { /* Provider must take a send completion every now and then */
1922                 INIT_CQCOUNT(ep);
1923                 send_wr.send_flags = IB_SEND_SIGNALED;
1924         }
1925
1926         rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail);
1927         if (rc)
1928                 dprintk("RPC:       %s: ib_post_send returned %i\n", __func__,
1929                         rc);
1930 out:
1931         return rc;
1932 }
1933
1934 /*
1935  * (Re)post a receive buffer.
1936  */
1937 int
1938 rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
1939                      struct rpcrdma_ep *ep,
1940                      struct rpcrdma_rep *rep)
1941 {
1942         struct ib_recv_wr recv_wr, *recv_wr_fail;
1943         int rc;
1944
1945         recv_wr.next = NULL;
1946         recv_wr.wr_id = (u64) (unsigned long) rep;
1947         recv_wr.sg_list = &rep->rr_iov;
1948         recv_wr.num_sge = 1;
1949
1950         ib_dma_sync_single_for_cpu(ia->ri_id->device,
1951                 rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL);
1952
1953         DECR_CQCOUNT(ep);
1954         rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);
1955
1956         if (rc)
1957                 dprintk("RPC:       %s: ib_post_recv returned %i\n", __func__,
1958                         rc);
1959         return rc;
1960 }