IB/mthca: Use mmiowb after doorbell ring

We discovered a problem when running IPoIB applications on multiple
CPUs on an Altix system. Many messages such as:

ib_mthca 0002:01:00.0: SQ 000014 full (19941644 head, 19941707 tail, 64 max, 0 nreq)

appear in syslog, and the driver wedges up.

Apparently this is because writes to the doorbells from different CPUs
reach the device out of order. The following patch adds mmiowb() calls
after doorbell rings to ensure the doorbell writes are ordered.

Signed-off-by: Arthur Kepner <akepner@sgi.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index e393681ba7d46a1988508c3f5d25e27b2cd4261d..149b3690123968ced42384ca1173be972ef710be 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -39,6 +39,8 @@
 #include <linux/init.h>
 #include <linux/hardirq.h>
 
+#include <asm/io.h>
+
 #include <rdma/ib_pack.h>
 
 #include "mthca_dev.h"
@@ -210,6 +212,11 @@ static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
                mthca_write64(doorbell,
                              dev->kar + MTHCA_CQ_DOORBELL,
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+               /*
+                * Make sure doorbells don't leak out of CQ spinlock
+                * and reach the HCA out of order:
+                */
+               mmiowb();
        }
 }
 

diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 5e5c58b9920b5c3d53b6d0a61a3afcaee037e281..6a7822e0fc19a94ca67fc306d5207aabdc865466 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -39,6 +39,8 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
+#include <asm/io.h>
+
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_cache.h>
 #include <rdma/ib_pack.h>
@@ -1732,6 +1734,11 @@ out:
                mthca_write64(doorbell,
                              dev->kar + MTHCA_SEND_DOORBELL,
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+               /*
+                * Make sure doorbells don't leak out of SQ spinlock
+                * and reach the HCA out of order:
+                */
+               mmiowb();
        }
 
        qp->sq.next_ind = ind;
@@ -1851,6 +1858,12 @@ out:
        qp->rq.next_ind = ind;
        qp->rq.head    += nreq;
 
+       /*
+        * Make sure doorbells don't leak out of RQ spinlock and reach
+        * the HCA out of order:
+        */
+       mmiowb();
+
        spin_unlock_irqrestore(&qp->rq.lock, flags);
        return err;
 }
@@ -2112,6 +2125,12 @@ out:
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
        }
 
+       /*
+        * Make sure doorbells don't leak out of SQ spinlock and reach
+        * the HCA out of order:
+        */
+       mmiowb();
+
        spin_unlock_irqrestore(&qp->sq.lock, flags);
        return err;
 }

diff --git a/drivers/infiniband/hw/mthca/mthca_srq.c b/drivers/infiniband/hw/mthca/mthca_srq.c
index 92a72f5215286493250283b89b7aed2ab096af9a..f5d7677d107969ee59b68b815d4926d7a5685cbf 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_srq.c
+++ b/drivers/infiniband/hw/mthca/mthca_srq.c
@@ -35,6 +35,8 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 
+#include <asm/io.h>
+
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
 #include "mthca_memfree.h"
@@ -595,6 +597,12 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
        }
 
+       /*
+        * Make sure doorbells don't leak out of SRQ spinlock and
+        * reach the HCA out of order:
+        */
+       mmiowb();
+
        spin_unlock_irqrestore(&srq->lock, flags);
        return err;
 }