net/core/flow_dissector.c

   1 #include <linux/kernel.h>
   2 #include <linux/skbuff.h>
   3 #include <linux/export.h>
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip.h>
   8 #include <net/ipv6.h>
   9 #include <linux/igmp.h>
  10 #include <linux/icmp.h>
  11 #include <linux/sctp.h>
  12 #include <linux/dccp.h>
  13 #include <linux/if_tunnel.h>
  14 #include <linux/if_pppox.h>
  15 #include <linux/ppp_defs.h>
  16 #include <linux/stddef.h>
  17 #include <linux/if_ether.h>
  18 #include <linux/mpls.h>
  19 #include <net/flow_dissector.h>
  20 #include <scsi/fc/fc_fcoe.h>
  21
  22 static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
  23                                         enum flow_dissector_key_id key_id)
  24 {
  25         return flow_dissector->used_keys & (1 << key_id);
  26 }
  27
  28 static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
  29                                        enum flow_dissector_key_id key_id)
  30 {
  31         flow_dissector->used_keys |= (1 << key_id);
  32 }
  33
  34 static void *skb_flow_dissector_target(struct flow_dissector *flow_dissector,
  35                                        enum flow_dissector_key_id key_id,
  36                                        void *target_container)
  37 {
  38         return ((char *) target_container) + flow_dissector->offset[key_id];
  39 }
  40
  41 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
  42                              const struct flow_dissector_key *key,
  43                              unsigned int key_count)
  44 {
  45         unsigned int i;
  46
  47         memset(flow_dissector, 0, sizeof(*flow_dissector));
  48
  49         for (i = 0; i < key_count; i++, key++) {
  50                 /* User should make sure that every key target offset is withing
  51                  * boundaries of unsigned short.
  52                  */
  53                 BUG_ON(key->offset > USHRT_MAX);
  54                 BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
  55                                                    key->key_id));
  56
  57                 skb_flow_dissector_set_key(flow_dissector, key->key_id);
  58                 flow_dissector->offset[key->key_id] = key->offset;
  59         }
  60
  61         /* Ensure that the dissector always includes control and basic key.
  62          * That way we are able to avoid handling lack of these in fast path.
  63          */
  64         BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
  65                                             FLOW_DISSECTOR_KEY_CONTROL));
  66         BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
  67                                             FLOW_DISSECTOR_KEY_BASIC));
  68 }
  69 EXPORT_SYMBOL(skb_flow_dissector_init);
  70
  71 /**
  72  * __skb_flow_get_ports - extract the upper layer ports and return them
  73  * @skb: sk_buff to extract the ports from
  74  * @thoff: transport header offset
  75  * @ip_proto: protocol for which to get port offset
  76  * @data: raw buffer pointer to the packet, if NULL use skb->data
  77  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
  78  *
  79  * The function will try to retrieve the ports at offset thoff + poff where poff
  80  * is the protocol port offset returned from proto_ports_offset
  81  */
  82 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
  83                             void *data, int hlen)
  84 {
  85         int poff = proto_ports_offset(ip_proto);
  86
  87         if (!data) {
  88                 data = skb->data;
  89                 hlen = skb_headlen(skb);
  90         }
  91
  92         if (poff >= 0) {
  93                 __be32 *ports, _ports;
  94
  95                 ports = __skb_header_pointer(skb, thoff + poff,
  96                                              sizeof(_ports), data, hlen, &_ports);
  97                 if (ports)
  98                         return *ports;
  99         }
 100
 101         return 0;
 102 }
 103 EXPORT_SYMBOL(__skb_flow_get_ports);
 104
 105 /**
 106  * __skb_flow_dissect - extract the flow_keys struct and return it
 107  * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 108  * @flow_dissector: list of keys to dissect
 109  * @target_container: target structure to put dissected values into
 110  * @data: raw buffer pointer to the packet, if NULL use skb->data
 111  * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 112  * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 113  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 114  *
 115  * The function will try to retrieve individual keys into target specified
 116  * by flow_dissector from either the skbuff or a raw buffer specified by the
 117  * rest parameters.
 118  *
 119  * Caller must take care of zeroing target container memory.
 120  */
 121 bool __skb_flow_dissect(const struct sk_buff *skb,
 122                         struct flow_dissector *flow_dissector,
 123                         void *target_container,
 124                         void *data, __be16 proto, int nhoff, int hlen)
 125 {
 126         struct flow_dissector_key_control *key_control;
 127         struct flow_dissector_key_basic *key_basic;
 128         struct flow_dissector_key_addrs *key_addrs;
 129         struct flow_dissector_key_ports *key_ports;
 130         struct flow_dissector_key_tags *key_tags;
 131         struct flow_dissector_key_keyid *key_keyid;
 132         u8 ip_proto = 0;
 133
 134         if (!data) {
 135                 data = skb->data;
 136                 proto = skb->protocol;
 137                 nhoff = skb_network_offset(skb);
 138                 hlen = skb_headlen(skb);
 139         }
 140
 141         /* It is ensured by skb_flow_dissector_init() that control key will
 142          * be always present.
 143          */
 144         key_control = skb_flow_dissector_target(flow_dissector,
 145                                                 FLOW_DISSECTOR_KEY_CONTROL,
 146                                                 target_container);
 147
 148         /* It is ensured by skb_flow_dissector_init() that basic key will
 149          * be always present.
 150          */
 151         key_basic = skb_flow_dissector_target(flow_dissector,
 152                                               FLOW_DISSECTOR_KEY_BASIC,
 153                                               target_container);
 154
 155         if (skb_flow_dissector_uses_key(flow_dissector,
 156                                         FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 157                 struct ethhdr *eth = eth_hdr(skb);
 158                 struct flow_dissector_key_eth_addrs *key_eth_addrs;
 159
 160                 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
 161                                                           FLOW_DISSECTOR_KEY_ETH_ADDRS,
 162                                                           target_container);
 163                 memcpy(key_eth_addrs, &eth->h_dest, sizeof(*key_eth_addrs));
 164         }
 165
 166 again:
 167         switch (proto) {
 168         case htons(ETH_P_IP): {
 169                 const struct iphdr *iph;
 170                 struct iphdr _iph;
 171 ip:
 172                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 173                 if (!iph || iph->ihl < 5)
 174                         return false;
 175                 nhoff += iph->ihl * 4;
 176
 177                 ip_proto = iph->protocol;
 178                 if (ip_is_fragment(iph))
 179                         ip_proto = 0;
 180
 181                 if (!skb_flow_dissector_uses_key(flow_dissector,
 182                                                  FLOW_DISSECTOR_KEY_IPV4_ADDRS))
 183                         break;
 184
 185                 key_addrs = skb_flow_dissector_target(flow_dissector,
 186                               FLOW_DISSECTOR_KEY_IPV4_ADDRS, target_container);
 187                 memcpy(&key_addrs->v4addrs, &iph->saddr,
 188                        sizeof(key_addrs->v4addrs));
 189                 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
 190                 break;
 191         }
 192         case htons(ETH_P_IPV6): {
 193                 const struct ipv6hdr *iph;
 194                 struct ipv6hdr _iph;
 195                 __be32 flow_label;
 196
 197 ipv6:
 198                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 199                 if (!iph)
 200                         return false;
 201
 202                 ip_proto = iph->nexthdr;
 203                 nhoff += sizeof(struct ipv6hdr);
 204
 205                 if (skb_flow_dissector_uses_key(flow_dissector,
 206                                                 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 207                         struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;
 208
 209                         key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
 210                                                                    FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 211                                                                    target_container);
 212
 213                         memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
 214                         key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
 215                 }
 216
 217                 flow_label = ip6_flowlabel(iph);
 218                 if (flow_label) {
 219                         if (skb_flow_dissector_uses_key(flow_dissector,
 220                                 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
 221                                 key_tags = skb_flow_dissector_target(flow_dissector,
 222                                                                      FLOW_DISSECTOR_KEY_FLOW_LABEL,
 223                                                                      target_container);
 224                                 key_tags->flow_label = ntohl(flow_label);
 225                         }
 226                 }
 227
 228                 break;
 229         }
 230         case htons(ETH_P_8021AD):
 231         case htons(ETH_P_8021Q): {
 232                 const struct vlan_hdr *vlan;
 233                 struct vlan_hdr _vlan;
 234
 235                 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
 236                 if (!vlan)
 237                         return false;
 238
 239                 if (skb_flow_dissector_uses_key(flow_dissector,
 240                                                 FLOW_DISSECTOR_KEY_VLANID)) {
 241                         key_tags = skb_flow_dissector_target(flow_dissector,
 242                                                              FLOW_DISSECTOR_KEY_VLANID,
 243                                                              target_container);
 244
 245                         key_tags->vlan_id = skb_vlan_tag_get_id(skb);
 246                 }
 247
 248                 proto = vlan->h_vlan_encapsulated_proto;
 249                 nhoff += sizeof(*vlan);
 250                 goto again;
 251         }
 252         case htons(ETH_P_PPP_SES): {
 253                 struct {
 254                         struct pppoe_hdr hdr;
 255                         __be16 proto;
 256                 } *hdr, _hdr;
 257                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 258                 if (!hdr)
 259                         return false;
 260                 proto = hdr->proto;
 261                 nhoff += PPPOE_SES_HLEN;
 262                 switch (proto) {
 263                 case htons(PPP_IP):
 264                         goto ip;
 265                 case htons(PPP_IPV6):
 266                         goto ipv6;
 267                 default:
 268                         return false;
 269                 }
 270         }
 271         case htons(ETH_P_TIPC): {
 272                 struct {
 273                         __be32 pre[3];
 274                         __be32 srcnode;
 275                 } *hdr, _hdr;
 276                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 277                 if (!hdr)
 278                         return false;
 279                 key_basic->n_proto = proto;
 280                 key_control->thoff = (u16)nhoff;
 281
 282                 if (skb_flow_dissector_uses_key(flow_dissector,
 283                                                 FLOW_DISSECTOR_KEY_TIPC_ADDRS)) {
 284                         key_addrs = skb_flow_dissector_target(flow_dissector,
 285                                                               FLOW_DISSECTOR_KEY_TIPC_ADDRS,
 286                                                               target_container);
 287                         key_addrs->tipcaddrs.srcnode = hdr->srcnode;
 288                         key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC_ADDRS;
 289                 }
 290                 return true;
 291         }
 292
 293         case htons(ETH_P_MPLS_UC):
 294         case htons(ETH_P_MPLS_MC): {
 295                 struct mpls_label *hdr, _hdr[2];
 296 mpls:
 297                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
 298                                            hlen, &_hdr);
 299                 if (!hdr)
 300                         return false;
 301
 302                 if ((ntohl(hdr[0].entry) & MPLS_LS_LABEL_MASK) >>
 303                      MPLS_LS_LABEL_SHIFT == MPLS_LABEL_ENTROPY) {
 304                         if (skb_flow_dissector_uses_key(flow_dissector,
 305                                                         FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) {
 306                                 key_keyid = skb_flow_dissector_target(flow_dissector,
 307                                                                       FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
 308                                                                       target_container);
 309                                 key_keyid->keyid = hdr[1].entry &
 310                                         htonl(MPLS_LS_LABEL_MASK);
 311                         }
 312
 313                         key_basic->n_proto = proto;
 314                         key_basic->ip_proto = ip_proto;
 315                         key_control->thoff = (u16)nhoff;
 316
 317                         return true;
 318                 }
 319
 320                 return true;
 321         }
 322
 323         case htons(ETH_P_FCOE):
 324                 key_control->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
 325                 /* fall through */
 326         default:
 327                 return false;
 328         }
 329
 330 ip_proto_again:
 331         switch (ip_proto) {
 332         case IPPROTO_GRE: {
 333                 struct gre_hdr {
 334                         __be16 flags;
 335                         __be16 proto;
 336                 } *hdr, _hdr;
 337
 338                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 339                 if (!hdr)
 340                         return false;
 341                 /*
 342                  * Only look inside GRE if version zero and no
 343                  * routing
 344                  */
 345                 if (hdr->flags & (GRE_VERSION | GRE_ROUTING))
 346                         break;
 347
 348                 proto = hdr->proto;
 349                 nhoff += 4;
 350                 if (hdr->flags & GRE_CSUM)
 351                         nhoff += 4;
 352                 if (hdr->flags & GRE_KEY) {
 353                         const __be32 *keyid;
 354                         __be32 _keyid;
 355
 356                         keyid = __skb_header_pointer(skb, nhoff, sizeof(_keyid),
 357                                                      data, hlen, &_keyid);
 358
 359                         if (!keyid)
 360                                 return false;
 361
 362                         if (skb_flow_dissector_uses_key(flow_dissector,
 363                                                         FLOW_DISSECTOR_KEY_GRE_KEYID)) {
 364                                 key_keyid = skb_flow_dissector_target(flow_dissector,
 365                                                                       FLOW_DISSECTOR_KEY_GRE_KEYID,
 366                                                                       target_container);
 367                                 key_keyid->keyid = *keyid;
 368                         }
 369                         nhoff += 4;
 370                 }
 371                 if (hdr->flags & GRE_SEQ)
 372                         nhoff += 4;
 373                 if (proto == htons(ETH_P_TEB)) {
 374                         const struct ethhdr *eth;
 375                         struct ethhdr _eth;
 376
 377                         eth = __skb_header_pointer(skb, nhoff,
 378                                                    sizeof(_eth),
 379                                                    data, hlen, &_eth);
 380                         if (!eth)
 381                                 return false;
 382                         proto = eth->h_proto;
 383                         nhoff += sizeof(*eth);
 384                 }
 385                 goto again;
 386         }
 387         case NEXTHDR_HOP:
 388         case NEXTHDR_ROUTING:
 389         case NEXTHDR_DEST: {
 390                 u8 _opthdr[2], *opthdr;
 391
 392                 if (proto != htons(ETH_P_IPV6))
 393                         break;
 394
 395                 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr),
 396                                               data, hlen, &_opthdr);
 397                 if (!opthdr)
 398                         return false;
 399
 400                 ip_proto = opthdr[0];
 401                 nhoff += (opthdr[1] + 1) << 3;
 402
 403                 goto ip_proto_again;
 404         }
 405         case IPPROTO_IPIP:
 406                 proto = htons(ETH_P_IP);
 407                 goto ip;
 408         case IPPROTO_IPV6:
 409                 proto = htons(ETH_P_IPV6);
 410                 goto ipv6;
 411         case IPPROTO_MPLS:
 412                 proto = htons(ETH_P_MPLS_UC);
 413                 goto mpls;
 414         default:
 415                 break;
 416         }
 417
 418         key_basic->n_proto = proto;
 419         key_basic->ip_proto = ip_proto;
 420         key_control->thoff = (u16)nhoff;
 421
 422         if (skb_flow_dissector_uses_key(flow_dissector,
 423                                         FLOW_DISSECTOR_KEY_PORTS)) {
 424                 key_ports = skb_flow_dissector_target(flow_dissector,
 425                                                       FLOW_DISSECTOR_KEY_PORTS,
 426                                                       target_container);
 427                 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
 428                                                         data, hlen);
 429         }
 430
 431         return true;
 432 }
 433 EXPORT_SYMBOL(__skb_flow_dissect);
 434
 435 static u32 hashrnd __read_mostly;
 436 static __always_inline void __flow_hash_secret_init(void)
 437 {
 438         net_get_random_once(&hashrnd, sizeof(hashrnd));
 439 }
 440
 441 static __always_inline u32 __flow_hash_words(u32 *words, u32 length, u32 keyval)
 442 {
 443         return jhash2(words, length, keyval);
 444 }
 445
 446 static inline void *flow_keys_hash_start(struct flow_keys *flow)
 447 {
 448         BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32));
 449         return (void *)flow + FLOW_KEYS_HASH_OFFSET;
 450 }
 451
 452 static inline size_t flow_keys_hash_length(struct flow_keys *flow)
 453 {
 454         size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
 455         BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
 456         BUILD_BUG_ON(offsetof(typeof(*flow), addrs) !=
 457                      sizeof(*flow) - sizeof(flow->addrs));
 458
 459         switch (flow->control.addr_type) {
 460         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 461                 diff -= sizeof(flow->addrs.v4addrs);
 462                 break;
 463         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 464                 diff -= sizeof(flow->addrs.v6addrs);
 465                 break;
 466         case FLOW_DISSECTOR_KEY_TIPC_ADDRS:
 467                 diff -= sizeof(flow->addrs.tipcaddrs);
 468                 break;
 469         }
 470         return (sizeof(*flow) - diff) / sizeof(u32);
 471 }
 472
 473 __be32 flow_get_u32_src(const struct flow_keys *flow)
 474 {
 475         switch (flow->control.addr_type) {
 476         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 477                 return flow->addrs.v4addrs.src;
 478         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 479                 return (__force __be32)ipv6_addr_hash(
 480                         &flow->addrs.v6addrs.src);
 481         case FLOW_DISSECTOR_KEY_TIPC_ADDRS:
 482                 return flow->addrs.tipcaddrs.srcnode;
 483         default:
 484                 return 0;
 485         }
 486 }
 487 EXPORT_SYMBOL(flow_get_u32_src);
 488
 489 __be32 flow_get_u32_dst(const struct flow_keys *flow)
 490 {
 491         switch (flow->control.addr_type) {
 492         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 493                 return flow->addrs.v4addrs.dst;
 494         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 495                 return (__force __be32)ipv6_addr_hash(
 496                         &flow->addrs.v6addrs.dst);
 497         default:
 498                 return 0;
 499         }
 500 }
 501 EXPORT_SYMBOL(flow_get_u32_dst);
 502
 503 static inline void __flow_hash_consistentify(struct flow_keys *keys)
 504 {
 505         int addr_diff, i;
 506
 507         switch (keys->control.addr_type) {
 508         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 509                 addr_diff = (__force u32)keys->addrs.v4addrs.dst -
 510                             (__force u32)keys->addrs.v4addrs.src;
 511                 if ((addr_diff < 0) ||
 512                     (addr_diff == 0 &&
 513                      ((__force u16)keys->ports.dst <
 514                       (__force u16)keys->ports.src))) {
 515                         swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
 516                         swap(keys->ports.src, keys->ports.dst);
 517                 }
 518                 break;
 519         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 520                 addr_diff = memcmp(&keys->addrs.v6addrs.dst,
 521                                    &keys->addrs.v6addrs.src,
 522                                    sizeof(keys->addrs.v6addrs.dst));
 523                 if ((addr_diff < 0) ||
 524                     (addr_diff == 0 &&
 525                      ((__force u16)keys->ports.dst <
 526                       (__force u16)keys->ports.src))) {
 527                         for (i = 0; i < 4; i++)
 528                                 swap(keys->addrs.v6addrs.src.s6_addr32[i],
 529                                      keys->addrs.v6addrs.dst.s6_addr32[i]);
 530                         swap(keys->ports.src, keys->ports.dst);
 531                 }
 532                 break;
 533         }
 534 }
 535
 536 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
 537 {
 538         u32 hash;
 539
 540         __flow_hash_consistentify(keys);
 541
 542         hash = __flow_hash_words((u32 *)flow_keys_hash_start(keys),
 543                                  flow_keys_hash_length(keys), keyval);
 544         if (!hash)
 545                 hash = 1;
 546
 547         return hash;
 548 }
 549
 550 u32 flow_hash_from_keys(struct flow_keys *keys)
 551 {
 552         __flow_hash_secret_init();
 553         return __flow_hash_from_keys(keys, hashrnd);
 554 }
 555 EXPORT_SYMBOL(flow_hash_from_keys);
 556
 557 static inline u32 ___skb_get_hash(const struct sk_buff *skb,
 558                                   struct flow_keys *keys, u32 keyval)
 559 {
 560         if (!skb_flow_dissect_flow_keys(skb, keys))
 561                 return 0;
 562
 563         return __flow_hash_from_keys(keys, keyval);
 564 }
 565
 566 struct _flow_keys_digest_data {
 567         __be16  n_proto;
 568         u8      ip_proto;
 569         u8      padding;
 570         __be32  ports;
 571         __be32  src;
 572         __be32  dst;
 573 };
 574
 575 void make_flow_keys_digest(struct flow_keys_digest *digest,
 576                            const struct flow_keys *flow)
 577 {
 578         struct _flow_keys_digest_data *data =
 579             (struct _flow_keys_digest_data *)digest;
 580
 581         BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
 582
 583         memset(digest, 0, sizeof(*digest));
 584
 585         data->n_proto = flow->basic.n_proto;
 586         data->ip_proto = flow->basic.ip_proto;
 587         data->ports = flow->ports.ports;
 588         data->src = flow->addrs.v4addrs.src;
 589         data->dst = flow->addrs.v4addrs.dst;
 590 }
 591 EXPORT_SYMBOL(make_flow_keys_digest);
 592
 593 /**
 594  * __skb_get_hash: calculate a flow hash
 595  * @skb: sk_buff to calculate flow hash from
 596  *
 597  * This function calculates a flow hash based on src/dst addresses
 598  * and src/dst port numbers.  Sets hash in skb to non-zero hash value
 599  * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
 600  * if hash is a canonical 4-tuple hash over transport ports.
 601  */
 602 void __skb_get_hash(struct sk_buff *skb)
 603 {
 604         struct flow_keys keys;
 605         u32 hash;
 606
 607         __flow_hash_secret_init();
 608
 609         hash = ___skb_get_hash(skb, &keys, hashrnd);
 610         if (!hash)
 611                 return;
 612         if (keys.ports.ports)
 613                 skb->l4_hash = 1;
 614         skb->sw_hash = 1;
 615         skb->hash = hash;
 616 }
 617 EXPORT_SYMBOL(__skb_get_hash);
 618
 619 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
 620 {
 621         struct flow_keys keys;
 622
 623         return ___skb_get_hash(skb, &keys, perturb);
 624 }
 625 EXPORT_SYMBOL(skb_get_hash_perturb);
 626
 627 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
 628                    const struct flow_keys *keys, int hlen)
 629 {
 630         u32 poff = keys->control.thoff;
 631
 632         switch (keys->basic.ip_proto) {
 633         case IPPROTO_TCP: {
 634                 /* access doff as u8 to avoid unaligned access */
 635                 const u8 *doff;
 636                 u8 _doff;
 637
 638                 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
 639                                             data, hlen, &_doff);
 640                 if (!doff)
 641                         return poff;
 642
 643                 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
 644                 break;
 645         }
 646         case IPPROTO_UDP:
 647         case IPPROTO_UDPLITE:
 648                 poff += sizeof(struct udphdr);
 649                 break;
 650         /* For the rest, we do not really care about header
 651          * extensions at this point for now.
 652          */
 653         case IPPROTO_ICMP:
 654                 poff += sizeof(struct icmphdr);
 655                 break;
 656         case IPPROTO_ICMPV6:
 657                 poff += sizeof(struct icmp6hdr);
 658                 break;
 659         case IPPROTO_IGMP:
 660                 poff += sizeof(struct igmphdr);
 661                 break;
 662         case IPPROTO_DCCP:
 663                 poff += sizeof(struct dccp_hdr);
 664                 break;
 665         case IPPROTO_SCTP:
 666                 poff += sizeof(struct sctphdr);
 667                 break;
 668         }
 669
 670         return poff;
 671 }
 672
 673 /**
 674  * skb_get_poff - get the offset to the payload
 675  * @skb: sk_buff to get the payload offset from
 676  *
 677  * The function will get the offset to the payload as far as it could
 678  * be dissected.  The main user is currently BPF, so that we can dynamically
 679  * truncate packets without needing to push actual payload to the user
 680  * space and can analyze headers only, instead.
 681  */
 682 u32 skb_get_poff(const struct sk_buff *skb)
 683 {
 684         struct flow_keys keys;
 685
 686         if (!skb_flow_dissect_flow_keys(skb, &keys))
 687                 return 0;
 688
 689         return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
 690 }
 691
 692 static const struct flow_dissector_key flow_keys_dissector_keys[] = {
 693         {
 694                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
 695                 .offset = offsetof(struct flow_keys, control),
 696         },
 697         {
 698                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 699                 .offset = offsetof(struct flow_keys, basic),
 700         },
 701         {
 702                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 703                 .offset = offsetof(struct flow_keys, addrs.v4addrs),
 704         },
 705         {
 706                 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 707                 .offset = offsetof(struct flow_keys, addrs.v6addrs),
 708         },
 709         {
 710                 .key_id = FLOW_DISSECTOR_KEY_TIPC_ADDRS,
 711                 .offset = offsetof(struct flow_keys, addrs.tipcaddrs),
 712         },
 713         {
 714                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
 715                 .offset = offsetof(struct flow_keys, ports),
 716         },
 717         {
 718                 .key_id = FLOW_DISSECTOR_KEY_VLANID,
 719                 .offset = offsetof(struct flow_keys, tags),
 720         },
 721         {
 722                 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL,
 723                 .offset = offsetof(struct flow_keys, tags),
 724         },
 725         {
 726                 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
 727                 .offset = offsetof(struct flow_keys, keyid),
 728         },
 729 };
 730
 731 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
 732         {
 733                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
 734                 .offset = offsetof(struct flow_keys, control),
 735         },
 736         {
 737                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 738                 .offset = offsetof(struct flow_keys, basic),
 739         },
 740 };
 741
 742 struct flow_dissector flow_keys_dissector __read_mostly;
 743 EXPORT_SYMBOL(flow_keys_dissector);
 744
 745 struct flow_dissector flow_keys_buf_dissector __read_mostly;
 746
 747 static int __init init_default_flow_dissectors(void)
 748 {
 749         skb_flow_dissector_init(&flow_keys_dissector,
 750                                 flow_keys_dissector_keys,
 751                                 ARRAY_SIZE(flow_keys_dissector_keys));
 752         skb_flow_dissector_init(&flow_keys_buf_dissector,
 753                                 flow_keys_buf_dissector_keys,
 754                                 ARRAY_SIZE(flow_keys_buf_dissector_keys));
 755         return 0;
 756 }
 757
 758 late_initcall_sync(init_default_flow_dissectors);