enic: Add Accelerated RFS support
authorGovindarajulu Varadarajan <_govind@gmx.com>
Mon, 23 Jun 2014 10:38:02 +0000 (16:08 +0530)
committerDavid S. Miller <davem@davemloft.net>
Mon, 23 Jun 2014 21:32:19 +0000 (14:32 -0700)
This patch adds supports for Accelerated Receive Flow Steering.

When the desired rx is different from current rq, for a flow, kernel calls the
driver function enic_rx_flow_steer(). enic_rx_flow_steer adds a IP-TCP/UDP
hardware filter.

Driver registers a timer function enic_flow_may_expire. This function is called
every HZ/4 seconds. In this function we check if the added filter has expired
by calling rps_may_expire_flow(). If the flow has expired, it removes the hw
filter.

As of now adaptor supports only IPv4 - TCP/UDP filters.

Signed-off-by: Govindarajulu Varadarajan <_govind@gmx.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
drivers/net/ethernet/cisco/enic/enic.h
drivers/net/ethernet/cisco/enic/enic_clsf.c
drivers/net/ethernet/cisco/enic/enic_clsf.h
drivers/net/ethernet/cisco/enic/enic_main.c
drivers/net/ethernet/cisco/enic/enic_res.c
drivers/net/ethernet/cisco/enic/vnic_enet.h

index 14f465f239d65d8c791618c3961b03cd6ef79fed..b9b9178e174ef005895df63c61ee03e7a937d12e 100644 (file)
@@ -99,6 +99,44 @@ struct enic_port_profile {
        u8 mac_addr[ETH_ALEN];
 };
 
+#ifdef CONFIG_RFS_ACCEL
+/* enic_rfs_fltr_node - rfs filter node in hash table
+ *     @@keys: IPv4 5 tuple
+ *     @flow_id: flow_id of clsf filter provided by kernel
+ *     @fltr_id: filter id of clsf filter returned by adaptor
+ *     @rq_id: desired rq index
+ *     @node: hlist_node
+ */
+struct enic_rfs_fltr_node {
+       struct flow_keys keys;
+       u32 flow_id;
+       u16 fltr_id;
+       u16 rq_id;
+       struct hlist_node node;
+};
+
+/* enic_rfs_flw_tbl - rfs flow table
+ *     @max: Maximum number of filters vNIC supports
+ *     @free: Number of free filters available
+ *     @toclean: hash table index to clean next
+ *     @ht_head: hash table list head
+ *     @lock: spin lock
+ *     @rfs_may_expire: timer function for enic_rps_may_expire_flow
+ */
+struct enic_rfs_flw_tbl {
+       u16 max;
+       int free;
+
+#define ENIC_RFS_FLW_BITSHIFT  (10)
+#define ENIC_RFS_FLW_MASK      ((1 << ENIC_RFS_FLW_BITSHIFT) - 1)
+       u16 toclean:ENIC_RFS_FLW_BITSHIFT;
+       struct hlist_head ht_head[1 << ENIC_RFS_FLW_BITSHIFT];
+       spinlock_t lock;
+       struct timer_list rfs_may_expire;
+};
+
+#endif /* CONFIG_RFS_ACCEL */
+
 /* Per-instance private data structure */
 struct enic {
        struct net_device *netdev;
@@ -150,6 +188,9 @@ struct enic {
        /* completion queue cache line section */
        ____cacheline_aligned struct vnic_cq cq[ENIC_CQ_MAX];
        unsigned int cq_count;
+#ifdef CONFIG_RFS_ACCEL
+       struct enic_rfs_flw_tbl rfs_h;
+#endif
 };
 
 static inline struct device *enic_get_dev(struct enic *enic)
index f6703c4f76a94700b9885995cc09784002963d2a..7f27a4c7fbfdc5cb1dfd721ef612bfffe79da8ed 100644 (file)
@@ -64,3 +64,216 @@ int enic_delfltr(struct enic *enic, u16 filter_id)
 
        return ret;
 }
+
+#ifdef CONFIG_RFS_ACCEL
+void enic_flow_may_expire(unsigned long data)
+{
+       struct enic *enic = (struct enic *)data;
+       bool res;
+       int j;
+
+       spin_lock(&enic->rfs_h.lock);
+       for (j = 0; j < ENIC_CLSF_EXPIRE_COUNT; j++) {
+               struct hlist_head *hhead;
+               struct hlist_node *tmp;
+               struct enic_rfs_fltr_node *n;
+
+               hhead = &enic->rfs_h.ht_head[enic->rfs_h.toclean++];
+               hlist_for_each_entry_safe(n, tmp, hhead, node) {
+                       res = rps_may_expire_flow(enic->netdev, n->rq_id,
+                                                 n->flow_id, n->fltr_id);
+                       if (res) {
+                               res = enic_delfltr(enic, n->fltr_id);
+                               if (unlikely(res))
+                                       continue;
+                               hlist_del(&n->node);
+                               kfree(n);
+                               enic->rfs_h.free++;
+                       }
+               }
+       }
+       spin_unlock(&enic->rfs_h.lock);
+       mod_timer(&enic->rfs_h.rfs_may_expire, jiffies + HZ/4);
+}
+
+/* enic_rfs_flw_tbl_init - initialize enic->rfs_h members
+ *     @enic: enic data
+ */
+void enic_rfs_flw_tbl_init(struct enic *enic)
+{
+       int i;
+
+       spin_lock_init(&enic->rfs_h.lock);
+       for (i = 0; i <= ENIC_RFS_FLW_MASK; i++)
+               INIT_HLIST_HEAD(&enic->rfs_h.ht_head[i]);
+       enic->rfs_h.max = enic->config.num_arfs;
+       enic->rfs_h.free = enic->rfs_h.max;
+       enic->rfs_h.toclean = 0;
+       init_timer(&enic->rfs_h.rfs_may_expire);
+       enic->rfs_h.rfs_may_expire.function = enic_flow_may_expire;
+       enic->rfs_h.rfs_may_expire.data = (unsigned long)enic;
+       mod_timer(&enic->rfs_h.rfs_may_expire, jiffies + HZ/4);
+}
+
+void enic_rfs_flw_tbl_free(struct enic *enic)
+{
+       int i, res;
+
+       del_timer_sync(&enic->rfs_h.rfs_may_expire);
+       spin_lock(&enic->rfs_h.lock);
+       enic->rfs_h.free = 0;
+       for (i = 0; i < (1 << ENIC_RFS_FLW_BITSHIFT); i++) {
+               struct hlist_head *hhead;
+               struct hlist_node *tmp;
+               struct enic_rfs_fltr_node *n;
+
+               hhead = &enic->rfs_h.ht_head[i];
+               hlist_for_each_entry_safe(n, tmp, hhead, node) {
+                       enic_delfltr(enic, n->fltr_id);
+                       hlist_del(&n->node);
+                       kfree(n);
+               }
+       }
+       spin_unlock(&enic->rfs_h.lock);
+}
+
+static struct enic_rfs_fltr_node *htbl_key_search(struct hlist_head *h,
+                                                 struct flow_keys *k)
+{
+       struct enic_rfs_fltr_node *tpos;
+
+       hlist_for_each_entry(tpos, h, node)
+               if (tpos->keys.src == k->src &&
+                   tpos->keys.dst == k->dst &&
+                   tpos->keys.ports == k->ports &&
+                   tpos->keys.ip_proto == k->ip_proto &&
+                   tpos->keys.n_proto == k->n_proto)
+                       return tpos;
+       return NULL;
+}
+
+int enic_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
+                      u16 rxq_index, u32 flow_id)
+{
+       struct flow_keys keys;
+       struct enic_rfs_fltr_node *n;
+       struct enic *enic;
+       u16 tbl_idx;
+       int res, i;
+
+       enic = netdev_priv(dev);
+       res = skb_flow_dissect(skb, &keys);
+       if (!res || keys.n_proto != htons(ETH_P_IP) ||
+           (keys.ip_proto != IPPROTO_TCP && keys.ip_proto != IPPROTO_UDP))
+               return -EPROTONOSUPPORT;
+
+       tbl_idx = skb_get_hash_raw(skb) & ENIC_RFS_FLW_MASK;
+       spin_lock(&enic->rfs_h.lock);
+       n = htbl_key_search(&enic->rfs_h.ht_head[tbl_idx], &keys);
+
+       if (n) { /* entry already present  */
+               if (rxq_index == n->rq_id) {
+                       res = -EEXIST;
+                       goto ret_unlock;
+               }
+
+               /* desired rq changed for the flow, we need to delete
+                * old fltr and add new one
+                *
+                * The moment we delete the fltr, the upcoming pkts
+                * are put it default rq based on rss. When we add
+                * new filter, upcoming pkts are put in desired queue.
+                * This could cause ooo pkts.
+                *
+                * Lets 1st try adding new fltr and then del old one.
+                */
+               i = --enic->rfs_h.free;
+               /* clsf tbl is full, we have to del old fltr first*/
+               if (unlikely(i < 0)) {
+                       enic->rfs_h.free++;
+                       res = enic_delfltr(enic, n->fltr_id);
+                       if (unlikely(res < 0))
+                               goto ret_unlock;
+                       res = enic_addfltr_5t(enic, &keys, rxq_index);
+                       if (res < 0) {
+                               hlist_del(&n->node);
+                               enic->rfs_h.free++;
+                               goto ret_unlock;
+                       }
+               /* add new fltr 1st then del old fltr */
+               } else {
+                       int ret;
+
+                       res = enic_addfltr_5t(enic, &keys, rxq_index);
+                       if (res < 0) {
+                               enic->rfs_h.free++;
+                               goto ret_unlock;
+                       }
+                       ret = enic_delfltr(enic, n->fltr_id);
+                       /* deleting old fltr failed. Add old fltr to list.
+                        * enic_flow_may_expire() will try to delete it later.
+                        */
+                       if (unlikely(ret < 0)) {
+                               struct enic_rfs_fltr_node *d;
+                               struct hlist_head *head;
+
+                               head = &enic->rfs_h.ht_head[tbl_idx];
+                               d = kmalloc(sizeof(*d), GFP_ATOMIC);
+                               if (d) {
+                                       d->fltr_id = n->fltr_id;
+                                       INIT_HLIST_NODE(&d->node);
+                                       hlist_add_head(&d->node, head);
+                               }
+                       } else {
+                               enic->rfs_h.free++;
+                       }
+               }
+               n->rq_id = rxq_index;
+               n->fltr_id = res;
+               n->flow_id = flow_id;
+       /* entry not present */
+       } else {
+               i = --enic->rfs_h.free;
+               if (i <= 0) {
+                       enic->rfs_h.free++;
+                       res = -EBUSY;
+                       goto ret_unlock;
+               }
+
+               n = kmalloc(sizeof(*n), GFP_ATOMIC);
+               if (!n) {
+                       res = -ENOMEM;
+                       enic->rfs_h.free++;
+                       goto ret_unlock;
+               }
+
+               res = enic_addfltr_5t(enic, &keys, rxq_index);
+               if (res < 0) {
+                       kfree(n);
+                       enic->rfs_h.free++;
+                       goto ret_unlock;
+               }
+               n->rq_id = rxq_index;
+               n->fltr_id = res;
+               n->flow_id = flow_id;
+               n->keys = keys;
+               INIT_HLIST_NODE(&n->node);
+               hlist_add_head(&n->node, &enic->rfs_h.ht_head[tbl_idx]);
+       }
+
+ret_unlock:
+       spin_unlock(&enic->rfs_h.lock);
+       return res;
+}
+
+#else
+
+void enic_rfs_flw_tbl_init(struct enic *enic)
+{
+}
+
+void enic_rfs_flw_tbl_free(struct enic *enic)
+{
+}
+
+#endif /* CONFIG_RFS_ACCEL */
index b6925b368b77ab77a1e4911940f0f76718cf3471..76a85bb0bb73c91d300968b13830749cc3e4c25f 100644 (file)
@@ -4,7 +4,16 @@
 #include "vnic_dev.h"
 #include "enic.h"
 
+#define ENIC_CLSF_EXPIRE_COUNT 128
+
 int enic_addfltr_5t(struct enic *enic, struct flow_keys *keys, u16 rq);
 int enic_delfltr(struct enic *enic, u16 filter_id);
 
+#ifdef CONFIG_RFS_ACCEL
+void enic_rfs_flw_tbl_init(struct enic *enic);
+void enic_rfs_flw_tbl_free(struct enic *enic);
+int enic_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
+                      u16 rxq_index, u32 flow_id);
+#endif /* CONFIG_RFS_ACCEL */
+
 #endif /* _ENIC_CLSF_H_ */
index 151b375337a95ae2fe3d24dd9e13393c9f79e66a..a302f1b3e8ff8d408b481bfad1a8b334d590e3b4 100644 (file)
@@ -52,6 +52,7 @@
 #include "enic.h"
 #include "enic_dev.h"
 #include "enic_pp.h"
+#include "enic_clsf.h"
 
 #define ENIC_NOTIFY_TIMER_PERIOD       (2 * HZ)
 #define WQ_ENET_MAX_DESC_LEN           (1 << WQ_ENET_LEN_BITS)
@@ -1546,6 +1547,7 @@ static int enic_open(struct net_device *netdev)
                vnic_intr_unmask(&enic->intr[i]);
 
        enic_notify_timer_start(enic);
+       enic_rfs_flw_tbl_init(enic);
 
        return 0;
 
@@ -1572,6 +1574,7 @@ static int enic_stop(struct net_device *netdev)
        enic_synchronize_irqs(enic);
 
        del_timer_sync(&enic->notify_timer);
+       enic_rfs_flw_tbl_free(enic);
 
        enic_dev_disable(enic);
 
@@ -2064,6 +2067,9 @@ static const struct net_device_ops enic_netdev_dynamic_ops = {
 #ifdef CONFIG_NET_POLL_CONTROLLER
        .ndo_poll_controller    = enic_poll_controller,
 #endif
+#ifdef CONFIG_RFS_ACCEL
+       .ndo_rx_flow_steer      = enic_rx_flow_steer,
+#endif
 };
 
 static const struct net_device_ops enic_netdev_ops = {
@@ -2084,6 +2090,9 @@ static const struct net_device_ops enic_netdev_ops = {
 #ifdef CONFIG_NET_POLL_CONTROLLER
        .ndo_poll_controller    = enic_poll_controller,
 #endif
+#ifdef CONFIG_RFS_ACCEL
+       .ndo_rx_flow_steer      = enic_rx_flow_steer,
+#endif
 };
 
 static void enic_dev_deinit(struct enic *enic)
@@ -2429,6 +2438,10 @@ static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
        netdev->features |= netdev->hw_features;
 
+#ifdef CONFIG_RFS_ACCEL
+       netdev->hw_features |= NETIF_F_NTUPLE;
+#endif
+
        if (using_dac)
                netdev->features |= NETIF_F_HIGHDMA;
 
index 31d658880c3cfae3cc5a8e8935a68722e8e95ab6..9c96911fb2c8a7dc0bfb2df8e2c9398c15aa4132 100644 (file)
@@ -71,6 +71,7 @@ int enic_get_vnic_config(struct enic *enic)
        GET_CONFIG(intr_mode);
        GET_CONFIG(intr_timer_usec);
        GET_CONFIG(loop_tag);
+       GET_CONFIG(num_arfs);
 
        c->wq_desc_count =
                min_t(u32, ENIC_MAX_WQ_DESCS,
index 609542848e029de6289051ba198e39e7a2e58291..75aced2de86987b6a96205ca6d079f4d5cfd77be 100644 (file)
@@ -32,6 +32,8 @@ struct vnic_enet_config {
        char devname[16];
        u32 intr_timer_usec;
        u16 loop_tag;
+       u16 vf_rq_count;
+       u16 num_arfs;
 };
 
 #define VENETF_TSO             0x1     /* TSO enabled */