2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * PACKET - implements raw packet sockets.
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
35 * Ulises Alonso : Frame number limit removal and
36 * packet_set_ring memory leak.
37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
40 * byte arrays at the end of sockaddr_ll
42 * Johann Baudy : Added TX RING.
43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
55 #include <linux/types.h>
57 #include <linux/capability.h>
58 #include <linux/fcntl.h>
59 #include <linux/socket.h>
61 #include <linux/inet.h>
62 #include <linux/netdevice.h>
63 #include <linux/if_packet.h>
64 #include <linux/wireless.h>
65 #include <linux/kernel.h>
66 #include <linux/kmod.h>
67 #include <linux/slab.h>
68 #include <linux/vmalloc.h>
69 #include <net/net_namespace.h>
71 #include <net/protocol.h>
72 #include <linux/skbuff.h>
74 #include <linux/errno.h>
75 #include <linux/timer.h>
76 #include <asm/uaccess.h>
77 #include <asm/ioctls.h>
79 #include <asm/cacheflush.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83 #include <linux/poll.h>
84 #include <linux/module.h>
85 #include <linux/init.h>
86 #include <linux/mutex.h>
87 #include <linux/if_vlan.h>
88 #include <linux/virtio_net.h>
89 #include <linux/errqueue.h>
90 #include <linux/net_tstamp.h>
91 #include <linux/percpu.h>
93 #include <net/inet_common.h>
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
112 Incoming, dev->hard_header!=NULL
113 mac_header -> ll header
116 Outgoing, dev->hard_header!=NULL
117 mac_header -> ll header
120 Incoming, dev->hard_header==NULL
121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
123 assymetry between rx and tx paths.
126 Outgoing, dev->hard_header==NULL
127 mac_header -> data. ll header is still not built!
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
137 dev->hard_header != NULL
138 mac_header -> ll header
141 dev->hard_header == NULL (ll header is added by device, we cannot control it)
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
149 /* Private packet socket structures. */
151 /* identical to struct packet_mreq except it has
152 * a longer address field.
154 struct packet_mreq_max {
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
168 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
169 int closing, int tx_ring);
171 #define V3_ALIGNMENT (8)
173 #define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
175 #define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178 #define PGV_FROM_VMALLOC 1
180 #define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181 #define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182 #define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183 #define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184 #define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185 #define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186 #define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
189 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
190 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
193 static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
196 static void packet_increment_head(struct packet_ring_buffer *buff);
197 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200 struct packet_sock *);
201 static void prb_retire_current_block(struct tpacket_kbdq_core *,
202 struct packet_sock *, unsigned int status);
203 static int prb_queue_frozen(struct tpacket_kbdq_core *);
204 static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
206 static void prb_retire_rx_blk_timer_expired(unsigned long);
207 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208 static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
216 static void packet_flush_mclist(struct sock *sk);
218 struct packet_skb_cb {
220 struct sockaddr_pkt pkt;
222 /* Trick: alias skb original length with
223 * ll.sll_family and ll.protocol in order
226 unsigned int origlen;
227 struct sockaddr_ll ll;
232 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
234 #define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
235 #define GET_PBLOCK_DESC(x, bid) \
236 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
237 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
238 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
239 #define GET_NEXT_PRB_BLK_NUM(x) \
240 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
241 ((x)->kactive_blk_num+1) : 0)
243 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244 static void __fanout_link(struct sock *sk, struct packet_sock *po);
246 static int packet_direct_xmit(struct sk_buff *skb)
248 struct net_device *dev = skb->dev;
249 netdev_features_t features;
250 struct netdev_queue *txq;
251 int ret = NETDEV_TX_BUSY;
253 if (unlikely(!netif_running(dev) ||
254 !netif_carrier_ok(dev)))
257 features = netif_skb_features(skb);
258 if (skb_needs_linearize(skb, features) &&
259 __skb_linearize(skb))
262 txq = skb_get_tx_queue(dev, skb);
266 HARD_TX_LOCK(dev, txq, smp_processor_id());
267 if (!netif_xmit_frozen_or_drv_stopped(txq))
268 ret = netdev_start_xmit(skb, dev, txq, false);
269 HARD_TX_UNLOCK(dev, txq);
273 if (!dev_xmit_complete(ret))
278 atomic_long_inc(&dev->tx_dropped);
280 return NET_XMIT_DROP;
283 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
285 struct net_device *dev;
288 dev = rcu_dereference(po->cached_dev);
296 static void packet_cached_dev_assign(struct packet_sock *po,
297 struct net_device *dev)
299 rcu_assign_pointer(po->cached_dev, dev);
302 static void packet_cached_dev_reset(struct packet_sock *po)
304 RCU_INIT_POINTER(po->cached_dev, NULL);
307 static bool packet_use_direct_xmit(const struct packet_sock *po)
309 return po->xmit == packet_direct_xmit;
312 static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
314 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
317 static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
319 const struct net_device_ops *ops = dev->netdev_ops;
322 if (ops->ndo_select_queue) {
323 queue_index = ops->ndo_select_queue(dev, skb, NULL,
324 __packet_pick_tx_queue);
325 queue_index = netdev_cap_txqueue(dev, queue_index);
327 queue_index = __packet_pick_tx_queue(dev, skb);
330 skb_set_queue_mapping(skb, queue_index);
333 /* register_prot_hook must be invoked with the po->bind_lock held,
334 * or from a context in which asynchronous accesses to the packet
335 * socket is not possible (packet_create()).
337 static void register_prot_hook(struct sock *sk)
339 struct packet_sock *po = pkt_sk(sk);
343 __fanout_link(sk, po);
345 dev_add_pack(&po->prot_hook);
352 /* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
353 * held. If the sync parameter is true, we will temporarily drop
354 * the po->bind_lock and do a synchronize_net to make sure no
355 * asynchronous packet processing paths still refer to the elements
356 * of po->prot_hook. If the sync parameter is false, it is the
357 * callers responsibility to take care of this.
359 static void __unregister_prot_hook(struct sock *sk, bool sync)
361 struct packet_sock *po = pkt_sk(sk);
366 __fanout_unlink(sk, po);
368 __dev_remove_pack(&po->prot_hook);
373 spin_unlock(&po->bind_lock);
375 spin_lock(&po->bind_lock);
379 static void unregister_prot_hook(struct sock *sk, bool sync)
381 struct packet_sock *po = pkt_sk(sk);
384 __unregister_prot_hook(sk, sync);
387 static inline struct page * __pure pgv_to_page(void *addr)
389 if (is_vmalloc_addr(addr))
390 return vmalloc_to_page(addr);
391 return virt_to_page(addr);
394 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
396 union tpacket_uhdr h;
399 switch (po->tp_version) {
401 h.h1->tp_status = status;
402 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
405 h.h2->tp_status = status;
406 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
410 WARN(1, "TPACKET version not supported.\n");
417 static int __packet_get_status(struct packet_sock *po, void *frame)
419 union tpacket_uhdr h;
424 switch (po->tp_version) {
426 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
427 return h.h1->tp_status;
429 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
430 return h.h2->tp_status;
433 WARN(1, "TPACKET version not supported.\n");
439 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
442 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
445 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
446 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
447 return TP_STATUS_TS_RAW_HARDWARE;
449 if (ktime_to_timespec_cond(skb->tstamp, ts))
450 return TP_STATUS_TS_SOFTWARE;
455 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
458 union tpacket_uhdr h;
462 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
466 switch (po->tp_version) {
468 h.h1->tp_sec = ts.tv_sec;
469 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
472 h.h2->tp_sec = ts.tv_sec;
473 h.h2->tp_nsec = ts.tv_nsec;
477 WARN(1, "TPACKET version not supported.\n");
481 /* one flush is safe, as both fields always lie on the same cacheline */
482 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
488 static void *packet_lookup_frame(struct packet_sock *po,
489 struct packet_ring_buffer *rb,
490 unsigned int position,
493 unsigned int pg_vec_pos, frame_offset;
494 union tpacket_uhdr h;
496 pg_vec_pos = position / rb->frames_per_block;
497 frame_offset = position % rb->frames_per_block;
499 h.raw = rb->pg_vec[pg_vec_pos].buffer +
500 (frame_offset * rb->frame_size);
502 if (status != __packet_get_status(po, h.raw))
508 static void *packet_current_frame(struct packet_sock *po,
509 struct packet_ring_buffer *rb,
512 return packet_lookup_frame(po, rb, rb->head, status);
515 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
517 del_timer_sync(&pkc->retire_blk_timer);
520 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
522 struct sk_buff_head *rb_queue)
524 struct tpacket_kbdq_core *pkc;
526 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
527 GET_PBDQC_FROM_RB(&po->rx_ring);
529 spin_lock_bh(&rb_queue->lock);
530 pkc->delete_blk_timer = 1;
531 spin_unlock_bh(&rb_queue->lock);
533 prb_del_retire_blk_timer(pkc);
536 static void prb_init_blk_timer(struct packet_sock *po,
537 struct tpacket_kbdq_core *pkc,
538 void (*func) (unsigned long))
540 init_timer(&pkc->retire_blk_timer);
541 pkc->retire_blk_timer.data = (long)po;
542 pkc->retire_blk_timer.function = func;
543 pkc->retire_blk_timer.expires = jiffies;
546 static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
548 struct tpacket_kbdq_core *pkc;
553 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
554 GET_PBDQC_FROM_RB(&po->rx_ring);
555 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
558 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
559 int blk_size_in_bytes)
561 struct net_device *dev;
562 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
563 struct ethtool_cmd ecmd;
568 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
569 if (unlikely(!dev)) {
571 return DEFAULT_PRB_RETIRE_TOV;
573 err = __ethtool_get_settings(dev, &ecmd);
574 speed = ethtool_cmd_speed(&ecmd);
578 * If the link speed is so slow you don't really
579 * need to worry about perf anyways
581 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
582 return DEFAULT_PRB_RETIRE_TOV;
589 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
601 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
602 union tpacket_req_u *req_u)
604 p1->feature_req_word = req_u->req3.tp_feature_req_word;
607 static void init_prb_bdqc(struct packet_sock *po,
608 struct packet_ring_buffer *rb,
610 union tpacket_req_u *req_u, int tx_ring)
612 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
613 struct tpacket_block_desc *pbd;
615 memset(p1, 0x0, sizeof(*p1));
617 p1->knxt_seq_num = 1;
619 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
620 p1->pkblk_start = pg_vec[0].buffer;
621 p1->kblk_size = req_u->req3.tp_block_size;
622 p1->knum_blocks = req_u->req3.tp_block_nr;
623 p1->hdrlen = po->tp_hdrlen;
624 p1->version = po->tp_version;
625 p1->last_kactive_blk_num = 0;
626 po->stats.stats3.tp_freeze_q_cnt = 0;
627 if (req_u->req3.tp_retire_blk_tov)
628 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
630 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
631 req_u->req3.tp_block_size);
632 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
633 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
635 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
636 prb_init_ft_ops(p1, req_u);
637 prb_setup_retire_blk_timer(po, tx_ring);
638 prb_open_block(p1, pbd);
641 /* Do NOT update the last_blk_num first.
642 * Assumes sk_buff_head lock is held.
644 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
646 mod_timer(&pkc->retire_blk_timer,
647 jiffies + pkc->tov_in_jiffies);
648 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
653 * 1) We refresh the timer only when we open a block.
654 * By doing this we don't waste cycles refreshing the timer
655 * on packet-by-packet basis.
657 * With a 1MB block-size, on a 1Gbps line, it will take
658 * i) ~8 ms to fill a block + ii) memcpy etc.
659 * In this cut we are not accounting for the memcpy time.
661 * So, if the user sets the 'tmo' to 10ms then the timer
662 * will never fire while the block is still getting filled
663 * (which is what we want). However, the user could choose
664 * to close a block early and that's fine.
666 * But when the timer does fire, we check whether or not to refresh it.
667 * Since the tmo granularity is in msecs, it is not too expensive
668 * to refresh the timer, lets say every '8' msecs.
669 * Either the user can set the 'tmo' or we can derive it based on
670 * a) line-speed and b) block-size.
671 * prb_calc_retire_blk_tmo() calculates the tmo.
674 static void prb_retire_rx_blk_timer_expired(unsigned long data)
676 struct packet_sock *po = (struct packet_sock *)data;
677 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
679 struct tpacket_block_desc *pbd;
681 spin_lock(&po->sk.sk_receive_queue.lock);
683 frozen = prb_queue_frozen(pkc);
684 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
686 if (unlikely(pkc->delete_blk_timer))
689 /* We only need to plug the race when the block is partially filled.
691 * lock(); increment BLOCK_NUM_PKTS; unlock()
692 * copy_bits() is in progress ...
693 * timer fires on other cpu:
694 * we can't retire the current block because copy_bits
698 if (BLOCK_NUM_PKTS(pbd)) {
699 while (atomic_read(&pkc->blk_fill_in_prog)) {
700 /* Waiting for skb_copy_bits to finish... */
705 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
707 if (!BLOCK_NUM_PKTS(pbd)) {
708 /* An empty block. Just refresh the timer. */
711 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
712 if (!prb_dispatch_next_block(pkc, po))
717 /* Case 1. Queue was frozen because user-space was
720 if (prb_curr_blk_in_use(pkc, pbd)) {
722 * Ok, user-space is still behind.
723 * So just refresh the timer.
727 /* Case 2. queue was frozen,user-space caught up,
728 * now the link went idle && the timer fired.
729 * We don't have a block to close.So we open this
730 * block and restart the timer.
731 * opening a block thaws the queue,restarts timer
732 * Thawing/timer-refresh is a side effect.
734 prb_open_block(pkc, pbd);
741 _prb_refresh_rx_retire_blk_timer(pkc);
744 spin_unlock(&po->sk.sk_receive_queue.lock);
747 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
748 struct tpacket_block_desc *pbd1, __u32 status)
750 /* Flush everything minus the block header */
752 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
757 /* Skip the block header(we know header WILL fit in 4K) */
760 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
761 for (; start < end; start += PAGE_SIZE)
762 flush_dcache_page(pgv_to_page(start));
767 /* Now update the block status. */
769 BLOCK_STATUS(pbd1) = status;
771 /* Flush the block header */
773 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
775 flush_dcache_page(pgv_to_page(start));
785 * 2) Increment active_blk_num
787 * Note:We DONT refresh the timer on purpose.
788 * Because almost always the next block will be opened.
790 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
791 struct tpacket_block_desc *pbd1,
792 struct packet_sock *po, unsigned int stat)
794 __u32 status = TP_STATUS_USER | stat;
796 struct tpacket3_hdr *last_pkt;
797 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
798 struct sock *sk = &po->sk;
800 if (po->stats.stats3.tp_drops)
801 status |= TP_STATUS_LOSING;
803 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
804 last_pkt->tp_next_offset = 0;
806 /* Get the ts of the last pkt */
807 if (BLOCK_NUM_PKTS(pbd1)) {
808 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
809 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
811 /* Ok, we tmo'd - so get the current time.
813 * It shouldn't really happen as we don't close empty
814 * blocks. See prb_retire_rx_blk_timer_expired().
818 h1->ts_last_pkt.ts_sec = ts.tv_sec;
819 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
824 /* Flush the block */
825 prb_flush_block(pkc1, pbd1, status);
827 sk->sk_data_ready(sk);
829 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
832 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
834 pkc->reset_pending_on_curr_blk = 0;
838 * Side effect of opening a block:
840 * 1) prb_queue is thawed.
841 * 2) retire_blk_timer is refreshed.
844 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
845 struct tpacket_block_desc *pbd1)
848 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
852 /* We could have just memset this but we will lose the
853 * flexibility of making the priv area sticky
856 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
857 BLOCK_NUM_PKTS(pbd1) = 0;
858 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
862 h1->ts_first_pkt.ts_sec = ts.tv_sec;
863 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
865 pkc1->pkblk_start = (char *)pbd1;
866 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
868 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
869 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
871 pbd1->version = pkc1->version;
872 pkc1->prev = pkc1->nxt_offset;
873 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
875 prb_thaw_queue(pkc1);
876 _prb_refresh_rx_retire_blk_timer(pkc1);
882 * Queue freeze logic:
883 * 1) Assume tp_block_nr = 8 blocks.
884 * 2) At time 't0', user opens Rx ring.
885 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
886 * 4) user-space is either sleeping or processing block '0'.
887 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
888 * it will close block-7,loop around and try to fill block '0'.
890 * __packet_lookup_frame_in_block
891 * prb_retire_current_block()
892 * prb_dispatch_next_block()
893 * |->(BLOCK_STATUS == USER) evaluates to true
894 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
895 * 6) Now there are two cases:
896 * 6.1) Link goes idle right after the queue is frozen.
897 * But remember, the last open_block() refreshed the timer.
898 * When this timer expires,it will refresh itself so that we can
899 * re-open block-0 in near future.
900 * 6.2) Link is busy and keeps on receiving packets. This is a simple
901 * case and __packet_lookup_frame_in_block will check if block-0
902 * is free and can now be re-used.
904 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
905 struct packet_sock *po)
907 pkc->reset_pending_on_curr_blk = 1;
908 po->stats.stats3.tp_freeze_q_cnt++;
911 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
914 * If the next block is free then we will dispatch it
915 * and return a good offset.
916 * Else, we will freeze the queue.
917 * So, caller must check the return value.
919 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
920 struct packet_sock *po)
922 struct tpacket_block_desc *pbd;
926 /* 1. Get current block num */
927 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
929 /* 2. If this block is currently in_use then freeze the queue */
930 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
931 prb_freeze_queue(pkc, po);
937 * open this block and return the offset where the first packet
938 * needs to get stored.
940 prb_open_block(pkc, pbd);
941 return (void *)pkc->nxt_offset;
944 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
945 struct packet_sock *po, unsigned int status)
947 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
949 /* retire/close the current block */
950 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
952 * Plug the case where copy_bits() is in progress on
953 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
954 * have space to copy the pkt in the current block and
955 * called prb_retire_current_block()
957 * We don't need to worry about the TMO case because
958 * the timer-handler already handled this case.
960 if (!(status & TP_STATUS_BLK_TMO)) {
961 while (atomic_read(&pkc->blk_fill_in_prog)) {
962 /* Waiting for skb_copy_bits to finish... */
966 prb_close_block(pkc, pbd, po, status);
971 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
972 struct tpacket_block_desc *pbd)
974 return TP_STATUS_USER & BLOCK_STATUS(pbd);
977 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
979 return pkc->reset_pending_on_curr_blk;
982 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
984 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
985 atomic_dec(&pkc->blk_fill_in_prog);
988 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
989 struct tpacket3_hdr *ppd)
991 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
994 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
995 struct tpacket3_hdr *ppd)
997 ppd->hv1.tp_rxhash = 0;
1000 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
1001 struct tpacket3_hdr *ppd)
1003 if (skb_vlan_tag_present(pkc->skb)) {
1004 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
1005 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1006 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1008 ppd->hv1.tp_vlan_tci = 0;
1009 ppd->hv1.tp_vlan_tpid = 0;
1010 ppd->tp_status = TP_STATUS_AVAILABLE;
1014 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
1015 struct tpacket3_hdr *ppd)
1017 ppd->hv1.tp_padding = 0;
1018 prb_fill_vlan_info(pkc, ppd);
1020 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1021 prb_fill_rxhash(pkc, ppd);
1023 prb_clear_rxhash(pkc, ppd);
1026 static void prb_fill_curr_block(char *curr,
1027 struct tpacket_kbdq_core *pkc,
1028 struct tpacket_block_desc *pbd,
1031 struct tpacket3_hdr *ppd;
1033 ppd = (struct tpacket3_hdr *)curr;
1034 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1036 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1037 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1038 BLOCK_NUM_PKTS(pbd) += 1;
1039 atomic_inc(&pkc->blk_fill_in_prog);
1040 prb_run_all_ft_ops(pkc, ppd);
1043 /* Assumes caller has the sk->rx_queue.lock */
1044 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1045 struct sk_buff *skb,
1050 struct tpacket_kbdq_core *pkc;
1051 struct tpacket_block_desc *pbd;
1054 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1055 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1057 /* Queue is frozen when user space is lagging behind */
1058 if (prb_queue_frozen(pkc)) {
1060 * Check if that last block which caused the queue to freeze,
1061 * is still in_use by user-space.
1063 if (prb_curr_blk_in_use(pkc, pbd)) {
1064 /* Can't record this packet */
1068 * Ok, the block was released by user-space.
1069 * Now let's open that block.
1070 * opening a block also thaws the queue.
1071 * Thawing is a side effect.
1073 prb_open_block(pkc, pbd);
1078 curr = pkc->nxt_offset;
1080 end = (char *)pbd + pkc->kblk_size;
1082 /* first try the current block */
1083 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1084 prb_fill_curr_block(curr, pkc, pbd, len);
1085 return (void *)curr;
1088 /* Ok, close the current block */
1089 prb_retire_current_block(pkc, po, 0);
1091 /* Now, try to dispatch the next block */
1092 curr = (char *)prb_dispatch_next_block(pkc, po);
1094 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1095 prb_fill_curr_block(curr, pkc, pbd, len);
1096 return (void *)curr;
1100 * No free blocks are available.user_space hasn't caught up yet.
1101 * Queue was just frozen and now this packet will get dropped.
1106 static void *packet_current_rx_frame(struct packet_sock *po,
1107 struct sk_buff *skb,
1108 int status, unsigned int len)
1111 switch (po->tp_version) {
1114 curr = packet_lookup_frame(po, &po->rx_ring,
1115 po->rx_ring.head, status);
1118 return __packet_lookup_frame_in_block(po, skb, status, len);
1120 WARN(1, "TPACKET version not supported\n");
1126 static void *prb_lookup_block(struct packet_sock *po,
1127 struct packet_ring_buffer *rb,
1131 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1132 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1134 if (status != BLOCK_STATUS(pbd))
1139 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1142 if (rb->prb_bdqc.kactive_blk_num)
1143 prev = rb->prb_bdqc.kactive_blk_num-1;
1145 prev = rb->prb_bdqc.knum_blocks-1;
1149 /* Assumes caller has held the rx_queue.lock */
1150 static void *__prb_previous_block(struct packet_sock *po,
1151 struct packet_ring_buffer *rb,
1154 unsigned int previous = prb_previous_blk_num(rb);
1155 return prb_lookup_block(po, rb, previous, status);
1158 static void *packet_previous_rx_frame(struct packet_sock *po,
1159 struct packet_ring_buffer *rb,
1162 if (po->tp_version <= TPACKET_V2)
1163 return packet_previous_frame(po, rb, status);
1165 return __prb_previous_block(po, rb, status);
1168 static void packet_increment_rx_head(struct packet_sock *po,
1169 struct packet_ring_buffer *rb)
1171 switch (po->tp_version) {
1174 return packet_increment_head(rb);
1177 WARN(1, "TPACKET version not supported.\n");
1183 static void *packet_previous_frame(struct packet_sock *po,
1184 struct packet_ring_buffer *rb,
1187 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1188 return packet_lookup_frame(po, rb, previous, status);
1191 static void packet_increment_head(struct packet_ring_buffer *buff)
1193 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1196 static void packet_inc_pending(struct packet_ring_buffer *rb)
1198 this_cpu_inc(*rb->pending_refcnt);
1201 static void packet_dec_pending(struct packet_ring_buffer *rb)
1203 this_cpu_dec(*rb->pending_refcnt);
1206 static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1208 unsigned int refcnt = 0;
1211 /* We don't use pending refcount in rx_ring. */
1212 if (rb->pending_refcnt == NULL)
1215 for_each_possible_cpu(cpu)
1216 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1221 static int packet_alloc_pending(struct packet_sock *po)
1223 po->rx_ring.pending_refcnt = NULL;
1225 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1226 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1232 static void packet_free_pending(struct packet_sock *po)
1234 free_percpu(po->tx_ring.pending_refcnt);
1237 #define ROOM_POW_OFF 2
1238 #define ROOM_NONE 0x0
1239 #define ROOM_LOW 0x1
1240 #define ROOM_NORMAL 0x2
1242 static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
1246 len = po->rx_ring.frame_max + 1;
1247 idx = po->rx_ring.head;
1249 idx += len >> pow_off;
1252 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1255 static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1259 len = po->rx_ring.prb_bdqc.knum_blocks;
1260 idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1262 idx += len >> pow_off;
1265 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1268 static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1270 struct sock *sk = &po->sk;
1271 int ret = ROOM_NONE;
1273 if (po->prot_hook.func != tpacket_rcv) {
1274 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
1275 - (skb ? skb->truesize : 0);
1276 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1284 if (po->tp_version == TPACKET_V3) {
1285 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1287 else if (__tpacket_v3_has_room(po, 0))
1290 if (__tpacket_has_room(po, ROOM_POW_OFF))
1292 else if (__tpacket_has_room(po, 0))
1299 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1304 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1305 ret = __packet_rcv_has_room(po, skb);
1306 has_room = ret == ROOM_NORMAL;
1307 if (po->pressure == has_room)
1308 po->pressure = !has_room;
1309 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
1314 static void packet_sock_destruct(struct sock *sk)
1316 skb_queue_purge(&sk->sk_error_queue);
1318 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1319 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1321 if (!sock_flag(sk, SOCK_DEAD)) {
1322 pr_err("Attempt to release alive packet socket: %p\n", sk);
1326 sk_refcnt_debug_dec(sk);
1329 static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1331 int x = atomic_read(&f->rr_cur) + 1;
1339 static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1344 rxhash = skb_get_hash(skb);
1345 for (i = 0; i < ROLLOVER_HLEN; i++)
1346 if (po->rollover->history[i] == rxhash)
1349 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1350 return count > (ROLLOVER_HLEN >> 1);
1353 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1354 struct sk_buff *skb,
1357 return reciprocal_scale(skb_get_hash(skb), num);
1360 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1361 struct sk_buff *skb,
1366 cur = atomic_read(&f->rr_cur);
1367 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1368 fanout_rr_next(f, num))) != cur)
1373 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1374 struct sk_buff *skb,
1377 return smp_processor_id() % num;
1380 static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1381 struct sk_buff *skb,
1384 return prandom_u32_max(num);
1387 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1388 struct sk_buff *skb,
1389 unsigned int idx, bool try_self,
1392 struct packet_sock *po, *po_next, *po_skip = NULL;
1393 unsigned int i, j, room = ROOM_NONE;
1395 po = pkt_sk(f->arr[idx]);
1398 room = packet_rcv_has_room(po, skb);
1399 if (room == ROOM_NORMAL ||
1400 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1405 i = j = min_t(int, po->rollover->sock, num - 1);
1407 po_next = pkt_sk(f->arr[i]);
1408 if (po_next != po_skip && !po_next->pressure &&
1409 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
1411 po->rollover->sock = i;
1412 atomic_long_inc(&po->rollover->num);
1413 if (room == ROOM_LOW)
1414 atomic_long_inc(&po->rollover->num_huge);
1422 atomic_long_inc(&po->rollover->num_failed);
1426 static unsigned int fanout_demux_qm(struct packet_fanout *f,
1427 struct sk_buff *skb,
1430 return skb_get_queue_mapping(skb) % num;
1433 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1435 return f->flags & (flag >> 8);
1438 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1439 struct packet_type *pt, struct net_device *orig_dev)
1441 struct packet_fanout *f = pt->af_packet_priv;
1442 unsigned int num = f->num_members;
1443 struct packet_sock *po;
1446 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1452 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1453 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1458 case PACKET_FANOUT_HASH:
1460 idx = fanout_demux_hash(f, skb, num);
1462 case PACKET_FANOUT_LB:
1463 idx = fanout_demux_lb(f, skb, num);
1465 case PACKET_FANOUT_CPU:
1466 idx = fanout_demux_cpu(f, skb, num);
1468 case PACKET_FANOUT_RND:
1469 idx = fanout_demux_rnd(f, skb, num);
1471 case PACKET_FANOUT_QM:
1472 idx = fanout_demux_qm(f, skb, num);
1474 case PACKET_FANOUT_ROLLOVER:
1475 idx = fanout_demux_rollover(f, skb, 0, false, num);
1479 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1480 idx = fanout_demux_rollover(f, skb, idx, true, num);
1482 po = pkt_sk(f->arr[idx]);
1483 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1486 DEFINE_MUTEX(fanout_mutex);
1487 EXPORT_SYMBOL_GPL(fanout_mutex);
1488 static LIST_HEAD(fanout_list);
1490 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1492 struct packet_fanout *f = po->fanout;
1494 spin_lock(&f->lock);
1495 f->arr[f->num_members] = sk;
1498 spin_unlock(&f->lock);
1501 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1503 struct packet_fanout *f = po->fanout;
1506 spin_lock(&f->lock);
1507 for (i = 0; i < f->num_members; i++) {
1508 if (f->arr[i] == sk)
1511 BUG_ON(i >= f->num_members);
1512 f->arr[i] = f->arr[f->num_members - 1];
1514 spin_unlock(&f->lock);
1517 static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1519 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
1525 static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1527 struct packet_sock *po = pkt_sk(sk);
1528 struct packet_fanout *f, *match;
1529 u8 type = type_flags & 0xff;
1530 u8 flags = type_flags >> 8;
1534 case PACKET_FANOUT_ROLLOVER:
1535 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1537 case PACKET_FANOUT_HASH:
1538 case PACKET_FANOUT_LB:
1539 case PACKET_FANOUT_CPU:
1540 case PACKET_FANOUT_RND:
1541 case PACKET_FANOUT_QM:
1553 if (type == PACKET_FANOUT_ROLLOVER ||
1554 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
1555 po->rollover = kzalloc(sizeof(*po->rollover), GFP_KERNEL);
1558 atomic_long_set(&po->rollover->num, 0);
1559 atomic_long_set(&po->rollover->num_huge, 0);
1560 atomic_long_set(&po->rollover->num_failed, 0);
1563 mutex_lock(&fanout_mutex);
1565 list_for_each_entry(f, &fanout_list, list) {
1567 read_pnet(&f->net) == sock_net(sk)) {
1573 if (match && match->flags != flags)
1577 match = kzalloc(sizeof(*match), GFP_KERNEL);
1580 write_pnet(&match->net, sock_net(sk));
1583 match->flags = flags;
1584 atomic_set(&match->rr_cur, 0);
1585 INIT_LIST_HEAD(&match->list);
1586 spin_lock_init(&match->lock);
1587 atomic_set(&match->sk_ref, 0);
1588 match->prot_hook.type = po->prot_hook.type;
1589 match->prot_hook.dev = po->prot_hook.dev;
1590 match->prot_hook.func = packet_rcv_fanout;
1591 match->prot_hook.af_packet_priv = match;
1592 match->prot_hook.id_match = match_fanout_group;
1593 dev_add_pack(&match->prot_hook);
1594 list_add(&match->list, &fanout_list);
1597 if (match->type == type &&
1598 match->prot_hook.type == po->prot_hook.type &&
1599 match->prot_hook.dev == po->prot_hook.dev) {
1601 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1602 __dev_remove_pack(&po->prot_hook);
1604 atomic_inc(&match->sk_ref);
1605 __fanout_link(sk, po);
1610 mutex_unlock(&fanout_mutex);
1612 kfree(po->rollover);
1613 po->rollover = NULL;
1618 static void fanout_release(struct sock *sk)
1620 struct packet_sock *po = pkt_sk(sk);
1621 struct packet_fanout *f;
1627 mutex_lock(&fanout_mutex);
1630 if (atomic_dec_and_test(&f->sk_ref)) {
1632 dev_remove_pack(&f->prot_hook);
1635 mutex_unlock(&fanout_mutex);
1638 kfree_rcu(po->rollover, rcu);
1641 static const struct proto_ops packet_ops;
1643 static const struct proto_ops packet_ops_spkt;
1645 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1646 struct packet_type *pt, struct net_device *orig_dev)
1649 struct sockaddr_pkt *spkt;
1652 * When we registered the protocol we saved the socket in the data
1653 * field for just this event.
1656 sk = pt->af_packet_priv;
1659 * Yank back the headers [hope the device set this
1660 * right or kerboom...]
1662 * Incoming packets have ll header pulled,
1665 * For outgoing ones skb->data == skb_mac_header(skb)
1666 * so that this procedure is noop.
1669 if (skb->pkt_type == PACKET_LOOPBACK)
1672 if (!net_eq(dev_net(dev), sock_net(sk)))
1675 skb = skb_share_check(skb, GFP_ATOMIC);
1679 /* drop any routing info */
1682 /* drop conntrack reference */
1685 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1687 skb_push(skb, skb->data - skb_mac_header(skb));
1690 * The SOCK_PACKET socket receives _all_ frames.
1693 spkt->spkt_family = dev->type;
1694 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1695 spkt->spkt_protocol = skb->protocol;
1698 * Charge the memory to the socket. This is done specifically
1699 * to prevent sockets using all the memory up.
1702 if (sock_queue_rcv_skb(sk, skb) == 0)
1713 * Output a raw packet to a device layer. This bypasses all the other
1714 * protocol layers and you must therefore supply it with a complete frame
1717 static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1720 struct sock *sk = sock->sk;
1721 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1722 struct sk_buff *skb = NULL;
1723 struct net_device *dev;
1729 * Get and verify the address.
1733 if (msg->msg_namelen < sizeof(struct sockaddr))
1735 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1736 proto = saddr->spkt_protocol;
1738 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1741 * Find the device first to size check it
1744 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1747 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1753 if (!(dev->flags & IFF_UP))
1757 * You may not queue a frame bigger than the mtu. This is the lowest level
1758 * raw protocol and you must do your own fragmentation at this level.
1761 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1762 if (!netif_supports_nofcs(dev)) {
1763 err = -EPROTONOSUPPORT;
1766 extra_len = 4; /* We're doing our own CRC */
1770 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1774 size_t reserved = LL_RESERVED_SPACE(dev);
1775 int tlen = dev->needed_tailroom;
1776 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1779 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1782 /* FIXME: Save some space for broken drivers that write a hard
1783 * header at transmission time by themselves. PPP is the notable
1784 * one here. This should really be fixed at the driver level.
1786 skb_reserve(skb, reserved);
1787 skb_reset_network_header(skb);
1789 /* Try to align data part correctly */
1794 skb_reset_network_header(skb);
1796 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1802 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
1803 /* Earlier code assumed this would be a VLAN pkt,
1804 * double-check this now that we have the actual
1807 struct ethhdr *ehdr;
1808 skb_reset_mac_header(skb);
1809 ehdr = eth_hdr(skb);
1810 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1816 skb->protocol = proto;
1818 skb->priority = sk->sk_priority;
1819 skb->mark = sk->sk_mark;
1821 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1823 if (unlikely(extra_len == 4))
1826 skb_probe_transport_header(skb, 0);
1828 dev_queue_xmit(skb);
1839 static unsigned int run_filter(const struct sk_buff *skb,
1840 const struct sock *sk,
1843 struct sk_filter *filter;
1846 filter = rcu_dereference(sk->sk_filter);
1848 res = SK_RUN_FILTER(filter, skb);
1855 * This function makes lazy skb cloning in hope that most of packets
1856 * are discarded by BPF.
1858 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1859 * and skb->cb are mangled. It works because (and until) packets
1860 * falling here are owned by current CPU. Output packets are cloned
1861 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1862 * sequencially, so that if we return skb to original state on exit,
1863 * we will not harm anyone.
1866 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1867 struct packet_type *pt, struct net_device *orig_dev)
1870 struct sockaddr_ll *sll;
1871 struct packet_sock *po;
1872 u8 *skb_head = skb->data;
1873 int skb_len = skb->len;
1874 unsigned int snaplen, res;
1876 if (skb->pkt_type == PACKET_LOOPBACK)
1879 sk = pt->af_packet_priv;
1882 if (!net_eq(dev_net(dev), sock_net(sk)))
1887 if (dev->header_ops) {
1888 /* The device has an explicit notion of ll header,
1889 * exported to higher levels.
1891 * Otherwise, the device hides details of its frame
1892 * structure, so that corresponding packet head is
1893 * never delivered to user.
1895 if (sk->sk_type != SOCK_DGRAM)
1896 skb_push(skb, skb->data - skb_mac_header(skb));
1897 else if (skb->pkt_type == PACKET_OUTGOING) {
1898 /* Special case: outgoing packets have ll header at head */
1899 skb_pull(skb, skb_network_offset(skb));
1905 res = run_filter(skb, sk, snaplen);
1907 goto drop_n_restore;
1911 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1914 if (skb_shared(skb)) {
1915 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1919 if (skb_head != skb->data) {
1920 skb->data = skb_head;
1927 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
1929 sll = &PACKET_SKB_CB(skb)->sa.ll;
1930 sll->sll_hatype = dev->type;
1931 sll->sll_pkttype = skb->pkt_type;
1932 if (unlikely(po->origdev))
1933 sll->sll_ifindex = orig_dev->ifindex;
1935 sll->sll_ifindex = dev->ifindex;
1937 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1939 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
1940 * Use their space for storing the original skb length.
1942 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
1944 if (pskb_trim(skb, snaplen))
1947 skb_set_owner_r(skb, sk);
1951 /* drop conntrack reference */
1954 spin_lock(&sk->sk_receive_queue.lock);
1955 po->stats.stats1.tp_packets++;
1956 sock_skb_set_dropcount(sk, skb);
1957 __skb_queue_tail(&sk->sk_receive_queue, skb);
1958 spin_unlock(&sk->sk_receive_queue.lock);
1959 sk->sk_data_ready(sk);
1963 spin_lock(&sk->sk_receive_queue.lock);
1964 po->stats.stats1.tp_drops++;
1965 atomic_inc(&sk->sk_drops);
1966 spin_unlock(&sk->sk_receive_queue.lock);
1969 if (skb_head != skb->data && skb_shared(skb)) {
1970 skb->data = skb_head;
1978 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1979 struct packet_type *pt, struct net_device *orig_dev)
1982 struct packet_sock *po;
1983 struct sockaddr_ll *sll;
1984 union tpacket_uhdr h;
1985 u8 *skb_head = skb->data;
1986 int skb_len = skb->len;
1987 unsigned int snaplen, res;
1988 unsigned long status = TP_STATUS_USER;
1989 unsigned short macoff, netoff, hdrlen;
1990 struct sk_buff *copy_skb = NULL;
1994 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1995 * We may add members to them until current aligned size without forcing
1996 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1998 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1999 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2001 if (skb->pkt_type == PACKET_LOOPBACK)
2004 sk = pt->af_packet_priv;
2007 if (!net_eq(dev_net(dev), sock_net(sk)))
2010 if (dev->header_ops) {
2011 if (sk->sk_type != SOCK_DGRAM)
2012 skb_push(skb, skb->data - skb_mac_header(skb));
2013 else if (skb->pkt_type == PACKET_OUTGOING) {
2014 /* Special case: outgoing packets have ll header at head */
2015 skb_pull(skb, skb_network_offset(skb));
2021 res = run_filter(skb, sk, snaplen);
2023 goto drop_n_restore;
2025 if (skb->ip_summed == CHECKSUM_PARTIAL)
2026 status |= TP_STATUS_CSUMNOTREADY;
2027 else if (skb->pkt_type != PACKET_OUTGOING &&
2028 (skb->ip_summed == CHECKSUM_COMPLETE ||
2029 skb_csum_unnecessary(skb)))
2030 status |= TP_STATUS_CSUM_VALID;
2035 if (sk->sk_type == SOCK_DGRAM) {
2036 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2039 unsigned int maclen = skb_network_offset(skb);
2040 netoff = TPACKET_ALIGN(po->tp_hdrlen +
2041 (maclen < 16 ? 16 : maclen)) +
2043 macoff = netoff - maclen;
2045 if (po->tp_version <= TPACKET_V2) {
2046 if (macoff + snaplen > po->rx_ring.frame_size) {
2047 if (po->copy_thresh &&
2048 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
2049 if (skb_shared(skb)) {
2050 copy_skb = skb_clone(skb, GFP_ATOMIC);
2052 copy_skb = skb_get(skb);
2053 skb_head = skb->data;
2056 skb_set_owner_r(copy_skb, sk);
2058 snaplen = po->rx_ring.frame_size - macoff;
2059 if ((int)snaplen < 0)
2062 } else if (unlikely(macoff + snaplen >
2063 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2066 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2067 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2068 snaplen, nval, macoff);
2070 if (unlikely((int)snaplen < 0)) {
2072 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2075 spin_lock(&sk->sk_receive_queue.lock);
2076 h.raw = packet_current_rx_frame(po, skb,
2077 TP_STATUS_KERNEL, (macoff+snaplen));
2080 if (po->tp_version <= TPACKET_V2) {
2081 packet_increment_rx_head(po, &po->rx_ring);
2083 * LOSING will be reported till you read the stats,
2084 * because it's COR - Clear On Read.
2085 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2088 if (po->stats.stats1.tp_drops)
2089 status |= TP_STATUS_LOSING;
2091 po->stats.stats1.tp_packets++;
2093 status |= TP_STATUS_COPY;
2094 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2096 spin_unlock(&sk->sk_receive_queue.lock);
2098 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
2100 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
2101 getnstimeofday(&ts);
2103 status |= ts_status;
2105 switch (po->tp_version) {
2107 h.h1->tp_len = skb->len;
2108 h.h1->tp_snaplen = snaplen;
2109 h.h1->tp_mac = macoff;
2110 h.h1->tp_net = netoff;
2111 h.h1->tp_sec = ts.tv_sec;
2112 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
2113 hdrlen = sizeof(*h.h1);
2116 h.h2->tp_len = skb->len;
2117 h.h2->tp_snaplen = snaplen;
2118 h.h2->tp_mac = macoff;
2119 h.h2->tp_net = netoff;
2120 h.h2->tp_sec = ts.tv_sec;
2121 h.h2->tp_nsec = ts.tv_nsec;
2122 if (skb_vlan_tag_present(skb)) {
2123 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
2124 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2125 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2127 h.h2->tp_vlan_tci = 0;
2128 h.h2->tp_vlan_tpid = 0;
2130 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
2131 hdrlen = sizeof(*h.h2);
2134 /* tp_nxt_offset,vlan are already populated above.
2135 * So DONT clear those fields here
2137 h.h3->tp_status |= status;
2138 h.h3->tp_len = skb->len;
2139 h.h3->tp_snaplen = snaplen;
2140 h.h3->tp_mac = macoff;
2141 h.h3->tp_net = netoff;
2142 h.h3->tp_sec = ts.tv_sec;
2143 h.h3->tp_nsec = ts.tv_nsec;
2144 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
2145 hdrlen = sizeof(*h.h3);
2151 sll = h.raw + TPACKET_ALIGN(hdrlen);
2152 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
2153 sll->sll_family = AF_PACKET;
2154 sll->sll_hatype = dev->type;
2155 sll->sll_protocol = skb->protocol;
2156 sll->sll_pkttype = skb->pkt_type;
2157 if (unlikely(po->origdev))
2158 sll->sll_ifindex = orig_dev->ifindex;
2160 sll->sll_ifindex = dev->ifindex;
2164 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
2165 if (po->tp_version <= TPACKET_V2) {
2168 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2171 for (start = h.raw; start < end; start += PAGE_SIZE)
2172 flush_dcache_page(pgv_to_page(start));
2177 if (po->tp_version <= TPACKET_V2) {
2178 __packet_set_status(po, h.raw, status);
2179 sk->sk_data_ready(sk);
2181 prb_clear_blk_fill_status(&po->rx_ring);
2185 if (skb_head != skb->data && skb_shared(skb)) {
2186 skb->data = skb_head;
2194 po->stats.stats1.tp_drops++;
2195 spin_unlock(&sk->sk_receive_queue.lock);
2197 sk->sk_data_ready(sk);
2198 kfree_skb(copy_skb);
2199 goto drop_n_restore;
2202 static void tpacket_destruct_skb(struct sk_buff *skb)
2204 struct packet_sock *po = pkt_sk(skb->sk);
2206 if (likely(po->tx_ring.pg_vec)) {
2210 ph = skb_shinfo(skb)->destructor_arg;
2211 packet_dec_pending(&po->tx_ring);
2213 ts = __packet_set_timestamp(po, ph, skb);
2214 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2220 static bool ll_header_truncated(const struct net_device *dev, int len)
2222 /* net device doesn't like empty head */
2223 if (unlikely(len <= dev->hard_header_len)) {
2224 net_warn_ratelimited("%s: packet size is too short (%d <= %d)\n",
2225 current->comm, len, dev->hard_header_len);
2232 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2233 void *frame, struct net_device *dev, int size_max,
2234 __be16 proto, unsigned char *addr, int hlen)
2236 union tpacket_uhdr ph;
2237 int to_write, offset, len, tp_len, nr_frags, len_max;
2238 struct socket *sock = po->sk.sk_socket;
2245 skb->protocol = proto;
2247 skb->priority = po->sk.sk_priority;
2248 skb->mark = po->sk.sk_mark;
2249 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
2250 skb_shinfo(skb)->destructor_arg = ph.raw;
2252 switch (po->tp_version) {
2254 tp_len = ph.h2->tp_len;
2257 tp_len = ph.h1->tp_len;
2260 if (unlikely(tp_len > size_max)) {
2261 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2265 skb_reserve(skb, hlen);
2266 skb_reset_network_header(skb);
2268 if (!packet_use_direct_xmit(po))
2269 skb_probe_transport_header(skb, 0);
2270 if (unlikely(po->tp_tx_has_off)) {
2271 int off_min, off_max, off;
2272 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2273 off_max = po->tx_ring.frame_size - tp_len;
2274 if (sock->type == SOCK_DGRAM) {
2275 switch (po->tp_version) {
2277 off = ph.h2->tp_net;
2280 off = ph.h1->tp_net;
2284 switch (po->tp_version) {
2286 off = ph.h2->tp_mac;
2289 off = ph.h1->tp_mac;
2293 if (unlikely((off < off_min) || (off_max < off)))
2295 data = ph.raw + off;
2297 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2301 if (sock->type == SOCK_DGRAM) {
2302 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2304 if (unlikely(err < 0))
2306 } else if (dev->hard_header_len) {
2307 if (ll_header_truncated(dev, tp_len))
2310 skb_push(skb, dev->hard_header_len);
2311 err = skb_store_bits(skb, 0, data,
2312 dev->hard_header_len);
2316 data += dev->hard_header_len;
2317 to_write -= dev->hard_header_len;
2320 offset = offset_in_page(data);
2321 len_max = PAGE_SIZE - offset;
2322 len = ((to_write > len_max) ? len_max : to_write);
2324 skb->data_len = to_write;
2325 skb->len += to_write;
2326 skb->truesize += to_write;
2327 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2329 while (likely(to_write)) {
2330 nr_frags = skb_shinfo(skb)->nr_frags;
2332 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2333 pr_err("Packet exceed the number of skb frags(%lu)\n",
2338 page = pgv_to_page(data);
2340 flush_dcache_page(page);
2342 skb_fill_page_desc(skb, nr_frags, page, offset, len);
2345 len_max = PAGE_SIZE;
2346 len = ((to_write > len_max) ? len_max : to_write);
2352 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2354 struct sk_buff *skb;
2355 struct net_device *dev;
2357 int err, reserve = 0;
2359 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2360 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2361 int tp_len, size_max;
2362 unsigned char *addr;
2364 int status = TP_STATUS_AVAILABLE;
2367 mutex_lock(&po->pg_vec_lock);
2369 if (likely(saddr == NULL)) {
2370 dev = packet_cached_dev_get(po);
2375 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2377 if (msg->msg_namelen < (saddr->sll_halen
2378 + offsetof(struct sockaddr_ll,
2381 proto = saddr->sll_protocol;
2382 addr = saddr->sll_addr;
2383 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2387 if (unlikely(dev == NULL))
2390 if (unlikely(!(dev->flags & IFF_UP)))
2393 reserve = dev->hard_header_len + VLAN_HLEN;
2394 size_max = po->tx_ring.frame_size
2395 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2397 if (size_max > dev->mtu + reserve)
2398 size_max = dev->mtu + reserve;
2401 ph = packet_current_frame(po, &po->tx_ring,
2402 TP_STATUS_SEND_REQUEST);
2403 if (unlikely(ph == NULL)) {
2404 if (need_wait && need_resched())
2409 status = TP_STATUS_SEND_REQUEST;
2410 hlen = LL_RESERVED_SPACE(dev);
2411 tlen = dev->needed_tailroom;
2412 skb = sock_alloc_send_skb(&po->sk,
2413 hlen + tlen + sizeof(struct sockaddr_ll),
2416 if (unlikely(skb == NULL)) {
2417 /* we assume the socket was initially writeable ... */
2418 if (likely(len_sum > 0))
2422 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2424 if (tp_len > dev->mtu + dev->hard_header_len) {
2425 struct ethhdr *ehdr;
2426 /* Earlier code assumed this would be a VLAN pkt,
2427 * double-check this now that we have the actual
2431 skb_reset_mac_header(skb);
2432 ehdr = eth_hdr(skb);
2433 if (ehdr->h_proto != htons(ETH_P_8021Q))
2436 if (unlikely(tp_len < 0)) {
2438 __packet_set_status(po, ph,
2439 TP_STATUS_AVAILABLE);
2440 packet_increment_head(&po->tx_ring);
2444 status = TP_STATUS_WRONG_FORMAT;
2450 packet_pick_tx_queue(dev, skb);
2452 skb->destructor = tpacket_destruct_skb;
2453 __packet_set_status(po, ph, TP_STATUS_SENDING);
2454 packet_inc_pending(&po->tx_ring);
2456 status = TP_STATUS_SEND_REQUEST;
2457 err = po->xmit(skb);
2458 if (unlikely(err > 0)) {
2459 err = net_xmit_errno(err);
2460 if (err && __packet_get_status(po, ph) ==
2461 TP_STATUS_AVAILABLE) {
2462 /* skb was destructed already */
2467 * skb was dropped but not destructed yet;
2468 * let's treat it like congestion or err < 0
2472 packet_increment_head(&po->tx_ring);
2474 } while (likely((ph != NULL) ||
2475 /* Note: packet_read_pending() might be slow if we have
2476 * to call it as it's per_cpu variable, but in fast-path
2477 * we already short-circuit the loop with the first
2478 * condition, and luckily don't have to go that path
2481 (need_wait && packet_read_pending(&po->tx_ring))));
2487 __packet_set_status(po, ph, status);
2492 mutex_unlock(&po->pg_vec_lock);
2496 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2497 size_t reserve, size_t len,
2498 size_t linear, int noblock,
2501 struct sk_buff *skb;
2503 /* Under a page? Don't bother with paged skb. */
2504 if (prepad + len < PAGE_SIZE || !linear)
2507 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2512 skb_reserve(skb, reserve);
2513 skb_put(skb, linear);
2514 skb->data_len = len - linear;
2515 skb->len += len - linear;
2520 static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2522 struct sock *sk = sock->sk;
2523 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2524 struct sk_buff *skb;
2525 struct net_device *dev;
2527 unsigned char *addr;
2528 int err, reserve = 0;
2529 struct virtio_net_hdr vnet_hdr = { 0 };
2532 struct packet_sock *po = pkt_sk(sk);
2533 unsigned short gso_type = 0;
2539 * Get and verify the address.
2542 if (likely(saddr == NULL)) {
2543 dev = packet_cached_dev_get(po);
2548 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2550 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2552 proto = saddr->sll_protocol;
2553 addr = saddr->sll_addr;
2554 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2558 if (unlikely(dev == NULL))
2561 if (unlikely(!(dev->flags & IFF_UP)))
2564 if (sock->type == SOCK_RAW)
2565 reserve = dev->hard_header_len;
2566 if (po->has_vnet_hdr) {
2567 vnet_hdr_len = sizeof(vnet_hdr);
2570 if (len < vnet_hdr_len)
2573 len -= vnet_hdr_len;
2576 n = copy_from_iter(&vnet_hdr, vnet_hdr_len, &msg->msg_iter);
2577 if (n != vnet_hdr_len)
2580 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2581 (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
2582 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
2583 __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
2584 vnet_hdr.hdr_len = __cpu_to_virtio16(false,
2585 __virtio16_to_cpu(false, vnet_hdr.csum_start) +
2586 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
2589 if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
2592 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2593 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2594 case VIRTIO_NET_HDR_GSO_TCPV4:
2595 gso_type = SKB_GSO_TCPV4;
2597 case VIRTIO_NET_HDR_GSO_TCPV6:
2598 gso_type = SKB_GSO_TCPV6;
2600 case VIRTIO_NET_HDR_GSO_UDP:
2601 gso_type = SKB_GSO_UDP;
2607 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2608 gso_type |= SKB_GSO_TCP_ECN;
2610 if (vnet_hdr.gso_size == 0)
2616 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2617 if (!netif_supports_nofcs(dev)) {
2618 err = -EPROTONOSUPPORT;
2621 extra_len = 4; /* We're doing our own CRC */
2625 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2629 hlen = LL_RESERVED_SPACE(dev);
2630 tlen = dev->needed_tailroom;
2631 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2632 __virtio16_to_cpu(false, vnet_hdr.hdr_len),
2633 msg->msg_flags & MSG_DONTWAIT, &err);
2637 skb_set_network_header(skb, reserve);
2640 if (sock->type == SOCK_DGRAM) {
2641 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
2642 if (unlikely(offset < 0))
2645 if (ll_header_truncated(dev, len))
2649 /* Returns -EFAULT on error */
2650 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
2654 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
2656 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
2657 /* Earlier code assumed this would be a VLAN pkt,
2658 * double-check this now that we have the actual
2661 struct ethhdr *ehdr;
2662 skb_reset_mac_header(skb);
2663 ehdr = eth_hdr(skb);
2664 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2670 skb->protocol = proto;
2672 skb->priority = sk->sk_priority;
2673 skb->mark = sk->sk_mark;
2675 packet_pick_tx_queue(dev, skb);
2677 if (po->has_vnet_hdr) {
2678 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2679 u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
2680 u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
2681 if (!skb_partial_csum_set(skb, s, o)) {
2687 skb_shinfo(skb)->gso_size =
2688 __virtio16_to_cpu(false, vnet_hdr.gso_size);
2689 skb_shinfo(skb)->gso_type = gso_type;
2691 /* Header must be checked, and gso_segs computed. */
2692 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2693 skb_shinfo(skb)->gso_segs = 0;
2695 len += vnet_hdr_len;
2698 if (!packet_use_direct_xmit(po))
2699 skb_probe_transport_header(skb, reserve);
2700 if (unlikely(extra_len == 4))
2703 err = po->xmit(skb);
2704 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2720 static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2722 struct sock *sk = sock->sk;
2723 struct packet_sock *po = pkt_sk(sk);
2725 if (po->tx_ring.pg_vec)
2726 return tpacket_snd(po, msg);
2728 return packet_snd(sock, msg, len);
2732 * Close a PACKET socket. This is fairly simple. We immediately go
2733 * to 'closed' state and remove our protocol entry in the device list.
2736 static int packet_release(struct socket *sock)
2738 struct sock *sk = sock->sk;
2739 struct packet_sock *po;
2741 union tpacket_req_u req_u;
2749 mutex_lock(&net->packet.sklist_lock);
2750 sk_del_node_init_rcu(sk);
2751 mutex_unlock(&net->packet.sklist_lock);
2754 sock_prot_inuse_add(net, sk->sk_prot, -1);
2757 spin_lock(&po->bind_lock);
2758 unregister_prot_hook(sk, false);
2759 packet_cached_dev_reset(po);
2761 if (po->prot_hook.dev) {
2762 dev_put(po->prot_hook.dev);
2763 po->prot_hook.dev = NULL;
2765 spin_unlock(&po->bind_lock);
2767 packet_flush_mclist(sk);
2769 if (po->rx_ring.pg_vec) {
2770 memset(&req_u, 0, sizeof(req_u));
2771 packet_set_ring(sk, &req_u, 1, 0);
2774 if (po->tx_ring.pg_vec) {
2775 memset(&req_u, 0, sizeof(req_u));
2776 packet_set_ring(sk, &req_u, 1, 1);
2783 * Now the socket is dead. No more input will appear.
2790 skb_queue_purge(&sk->sk_receive_queue);
2791 packet_free_pending(po);
2792 sk_refcnt_debug_release(sk);
2799 * Attach a packet hook.
2802 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
2804 struct packet_sock *po = pkt_sk(sk);
2805 const struct net_device *dev_curr;
2817 spin_lock(&po->bind_lock);
2819 proto_curr = po->prot_hook.type;
2820 dev_curr = po->prot_hook.dev;
2822 need_rehook = proto_curr != proto || dev_curr != dev;
2825 unregister_prot_hook(sk, true);
2828 po->prot_hook.type = proto;
2830 if (po->prot_hook.dev)
2831 dev_put(po->prot_hook.dev);
2833 po->prot_hook.dev = dev;
2835 po->ifindex = dev ? dev->ifindex : 0;
2836 packet_cached_dev_assign(po, dev);
2839 if (proto == 0 || !need_rehook)
2842 if (!dev || (dev->flags & IFF_UP)) {
2843 register_prot_hook(sk);
2845 sk->sk_err = ENETDOWN;
2846 if (!sock_flag(sk, SOCK_DEAD))
2847 sk->sk_error_report(sk);
2851 spin_unlock(&po->bind_lock);
2857 * Bind a packet socket to a device
2860 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2863 struct sock *sk = sock->sk;
2865 struct net_device *dev;
2872 if (addr_len != sizeof(struct sockaddr))
2874 strlcpy(name, uaddr->sa_data, sizeof(name));
2876 dev = dev_get_by_name(sock_net(sk), name);
2878 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
2882 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2884 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2885 struct sock *sk = sock->sk;
2886 struct net_device *dev = NULL;
2894 if (addr_len < sizeof(struct sockaddr_ll))
2896 if (sll->sll_family != AF_PACKET)
2899 if (sll->sll_ifindex) {
2901 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
2905 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
2911 static struct proto packet_proto = {
2913 .owner = THIS_MODULE,
2914 .obj_size = sizeof(struct packet_sock),
2918 * Create a packet of type SOCK_PACKET.
2921 static int packet_create(struct net *net, struct socket *sock, int protocol,
2925 struct packet_sock *po;
2926 __be16 proto = (__force __be16)protocol; /* weird, but documented */
2929 if (!ns_capable(net->user_ns, CAP_NET_RAW))
2931 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2932 sock->type != SOCK_PACKET)
2933 return -ESOCKTNOSUPPORT;
2935 sock->state = SS_UNCONNECTED;
2938 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
2942 sock->ops = &packet_ops;
2943 if (sock->type == SOCK_PACKET)
2944 sock->ops = &packet_ops_spkt;
2946 sock_init_data(sock, sk);
2949 sk->sk_family = PF_PACKET;
2951 po->xmit = dev_queue_xmit;
2953 err = packet_alloc_pending(po);
2957 packet_cached_dev_reset(po);
2959 sk->sk_destruct = packet_sock_destruct;
2960 sk_refcnt_debug_inc(sk);
2963 * Attach a protocol block
2966 spin_lock_init(&po->bind_lock);
2967 mutex_init(&po->pg_vec_lock);
2968 po->rollover = NULL;
2969 po->prot_hook.func = packet_rcv;
2971 if (sock->type == SOCK_PACKET)
2972 po->prot_hook.func = packet_rcv_spkt;
2974 po->prot_hook.af_packet_priv = sk;
2977 po->prot_hook.type = proto;
2978 register_prot_hook(sk);
2981 mutex_lock(&net->packet.sklist_lock);
2982 sk_add_node_rcu(sk, &net->packet.sklist);
2983 mutex_unlock(&net->packet.sklist_lock);
2986 sock_prot_inuse_add(net, &packet_proto, 1);
2997 * Pull a packet from our receive queue and hand it to the user.
2998 * If necessary we block.
3001 static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3004 struct sock *sk = sock->sk;
3005 struct sk_buff *skb;
3007 int vnet_hdr_len = 0;
3008 unsigned int origlen = 0;
3011 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
3015 /* What error should we return now? EUNATTACH? */
3016 if (pkt_sk(sk)->ifindex < 0)
3020 if (flags & MSG_ERRQUEUE) {
3021 err = sock_recv_errqueue(sk, msg, len,
3022 SOL_PACKET, PACKET_TX_TIMESTAMP);
3027 * Call the generic datagram receiver. This handles all sorts
3028 * of horrible races and re-entrancy so we can forget about it
3029 * in the protocol layers.
3031 * Now it will return ENETDOWN, if device have just gone down,
3032 * but then it will block.
3035 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
3038 * An error occurred so return it. Because skb_recv_datagram()
3039 * handles the blocking we don't see and worry about blocking
3046 if (pkt_sk(sk)->pressure)
3047 packet_rcv_has_room(pkt_sk(sk), NULL);
3049 if (pkt_sk(sk)->has_vnet_hdr) {
3050 struct virtio_net_hdr vnet_hdr = { 0 };
3053 vnet_hdr_len = sizeof(vnet_hdr);
3054 if (len < vnet_hdr_len)
3057 len -= vnet_hdr_len;
3059 if (skb_is_gso(skb)) {
3060 struct skb_shared_info *sinfo = skb_shinfo(skb);
3062 /* This is a hint as to how much should be linear. */
3064 __cpu_to_virtio16(false, skb_headlen(skb));
3066 __cpu_to_virtio16(false, sinfo->gso_size);
3067 if (sinfo->gso_type & SKB_GSO_TCPV4)
3068 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
3069 else if (sinfo->gso_type & SKB_GSO_TCPV6)
3070 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
3071 else if (sinfo->gso_type & SKB_GSO_UDP)
3072 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
3073 else if (sinfo->gso_type & SKB_GSO_FCOE)
3077 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
3078 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
3080 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
3082 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3083 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
3084 vnet_hdr.csum_start = __cpu_to_virtio16(false,
3085 skb_checksum_start_offset(skb));
3086 vnet_hdr.csum_offset = __cpu_to_virtio16(false,
3088 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3089 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
3090 } /* else everything is zero */
3092 err = memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_len);
3097 /* You lose any data beyond the buffer you gave. If it worries
3098 * a user program they can ask the device for its MTU
3104 msg->msg_flags |= MSG_TRUNC;
3107 err = skb_copy_datagram_msg(skb, 0, msg, copied);
3111 if (sock->type != SOCK_PACKET) {
3112 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3114 /* Original length was stored in sockaddr_ll fields */
3115 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3116 sll->sll_family = AF_PACKET;
3117 sll->sll_protocol = skb->protocol;
3120 sock_recv_ts_and_drops(msg, sk, skb);
3122 if (msg->msg_name) {
3123 /* If the address length field is there to be filled
3124 * in, we fill it in now.
3126 if (sock->type == SOCK_PACKET) {
3127 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
3128 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3130 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3132 msg->msg_namelen = sll->sll_halen +
3133 offsetof(struct sockaddr_ll, sll_addr);
3135 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3139 if (pkt_sk(sk)->auxdata) {
3140 struct tpacket_auxdata aux;
3142 aux.tp_status = TP_STATUS_USER;
3143 if (skb->ip_summed == CHECKSUM_PARTIAL)
3144 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
3145 else if (skb->pkt_type != PACKET_OUTGOING &&
3146 (skb->ip_summed == CHECKSUM_COMPLETE ||
3147 skb_csum_unnecessary(skb)))
3148 aux.tp_status |= TP_STATUS_CSUM_VALID;
3150 aux.tp_len = origlen;
3151 aux.tp_snaplen = skb->len;
3153 aux.tp_net = skb_network_offset(skb);
3154 if (skb_vlan_tag_present(skb)) {
3155 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
3156 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3157 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3159 aux.tp_vlan_tci = 0;
3160 aux.tp_vlan_tpid = 0;
3162 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
3166 * Free or return the buffer as appropriate. Again this
3167 * hides all the races and re-entrancy issues from us.
3169 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
3172 skb_free_datagram(sk, skb);
3177 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3178 int *uaddr_len, int peer)
3180 struct net_device *dev;
3181 struct sock *sk = sock->sk;
3186 uaddr->sa_family = AF_PACKET;
3187 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
3189 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3191 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
3193 *uaddr_len = sizeof(*uaddr);
3198 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3199 int *uaddr_len, int peer)
3201 struct net_device *dev;
3202 struct sock *sk = sock->sk;
3203 struct packet_sock *po = pkt_sk(sk);
3204 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
3209 sll->sll_family = AF_PACKET;
3210 sll->sll_ifindex = po->ifindex;
3211 sll->sll_protocol = po->num;
3212 sll->sll_pkttype = 0;
3214 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
3216 sll->sll_hatype = dev->type;
3217 sll->sll_halen = dev->addr_len;
3218 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
3220 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3224 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
3229 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3233 case PACKET_MR_MULTICAST:
3234 if (i->alen != dev->addr_len)
3237 return dev_mc_add(dev, i->addr);
3239 return dev_mc_del(dev, i->addr);
3241 case PACKET_MR_PROMISC:
3242 return dev_set_promiscuity(dev, what);
3243 case PACKET_MR_ALLMULTI:
3244 return dev_set_allmulti(dev, what);
3245 case PACKET_MR_UNICAST:
3246 if (i->alen != dev->addr_len)
3249 return dev_uc_add(dev, i->addr);
3251 return dev_uc_del(dev, i->addr);
3259 static void packet_dev_mclist_delete(struct net_device *dev,
3260 struct packet_mclist **mlp)
3262 struct packet_mclist *ml;
3264 while ((ml = *mlp) != NULL) {
3265 if (ml->ifindex == dev->ifindex) {
3266 packet_dev_mc(dev, ml, -1);
3274 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3276 struct packet_sock *po = pkt_sk(sk);
3277 struct packet_mclist *ml, *i;
3278 struct net_device *dev;
3284 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3289 if (mreq->mr_alen > dev->addr_len)
3293 i = kmalloc(sizeof(*i), GFP_KERNEL);
3298 for (ml = po->mclist; ml; ml = ml->next) {
3299 if (ml->ifindex == mreq->mr_ifindex &&
3300 ml->type == mreq->mr_type &&
3301 ml->alen == mreq->mr_alen &&
3302 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3304 /* Free the new element ... */
3310 i->type = mreq->mr_type;
3311 i->ifindex = mreq->mr_ifindex;
3312 i->alen = mreq->mr_alen;
3313 memcpy(i->addr, mreq->mr_address, i->alen);
3315 i->next = po->mclist;
3317 err = packet_dev_mc(dev, i, 1);
3319 po->mclist = i->next;
3328 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3330 struct packet_mclist *ml, **mlp;
3334 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3335 if (ml->ifindex == mreq->mr_ifindex &&
3336 ml->type == mreq->mr_type &&
3337 ml->alen == mreq->mr_alen &&
3338 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3339 if (--ml->count == 0) {
3340 struct net_device *dev;
3342 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3344 packet_dev_mc(dev, ml, -1);
3354 static void packet_flush_mclist(struct sock *sk)
3356 struct packet_sock *po = pkt_sk(sk);
3357 struct packet_mclist *ml;
3363 while ((ml = po->mclist) != NULL) {
3364 struct net_device *dev;
3366 po->mclist = ml->next;
3367 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3369 packet_dev_mc(dev, ml, -1);
3376 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3378 struct sock *sk = sock->sk;
3379 struct packet_sock *po = pkt_sk(sk);
3382 if (level != SOL_PACKET)
3383 return -ENOPROTOOPT;
3386 case PACKET_ADD_MEMBERSHIP:
3387 case PACKET_DROP_MEMBERSHIP:
3389 struct packet_mreq_max mreq;
3391 memset(&mreq, 0, sizeof(mreq));
3392 if (len < sizeof(struct packet_mreq))
3394 if (len > sizeof(mreq))
3396 if (copy_from_user(&mreq, optval, len))
3398 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3400 if (optname == PACKET_ADD_MEMBERSHIP)
3401 ret = packet_mc_add(sk, &mreq);
3403 ret = packet_mc_drop(sk, &mreq);
3407 case PACKET_RX_RING:
3408 case PACKET_TX_RING:
3410 union tpacket_req_u req_u;
3413 switch (po->tp_version) {
3416 len = sizeof(req_u.req);
3420 len = sizeof(req_u.req3);
3425 if (pkt_sk(sk)->has_vnet_hdr)
3427 if (copy_from_user(&req_u.req, optval, len))
3429 return packet_set_ring(sk, &req_u, 0,
3430 optname == PACKET_TX_RING);
3432 case PACKET_COPY_THRESH:
3436 if (optlen != sizeof(val))
3438 if (copy_from_user(&val, optval, sizeof(val)))
3441 pkt_sk(sk)->copy_thresh = val;
3444 case PACKET_VERSION:
3448 if (optlen != sizeof(val))
3450 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3452 if (copy_from_user(&val, optval, sizeof(val)))
3458 po->tp_version = val;
3464 case PACKET_RESERVE:
3468 if (optlen != sizeof(val))
3470 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3472 if (copy_from_user(&val, optval, sizeof(val)))
3474 po->tp_reserve = val;
3481 if (optlen != sizeof(val))
3483 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3485 if (copy_from_user(&val, optval, sizeof(val)))
3487 po->tp_loss = !!val;
3490 case PACKET_AUXDATA:
3494 if (optlen < sizeof(val))
3496 if (copy_from_user(&val, optval, sizeof(val)))
3499 po->auxdata = !!val;
3502 case PACKET_ORIGDEV:
3506 if (optlen < sizeof(val))
3508 if (copy_from_user(&val, optval, sizeof(val)))
3511 po->origdev = !!val;
3514 case PACKET_VNET_HDR:
3518 if (sock->type != SOCK_RAW)
3520 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3522 if (optlen < sizeof(val))
3524 if (copy_from_user(&val, optval, sizeof(val)))
3527 po->has_vnet_hdr = !!val;
3530 case PACKET_TIMESTAMP:
3534 if (optlen != sizeof(val))
3536 if (copy_from_user(&val, optval, sizeof(val)))
3539 po->tp_tstamp = val;
3546 if (optlen != sizeof(val))
3548 if (copy_from_user(&val, optval, sizeof(val)))
3551 return fanout_add(sk, val & 0xffff, val >> 16);
3553 case PACKET_TX_HAS_OFF:
3557 if (optlen != sizeof(val))
3559 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3561 if (copy_from_user(&val, optval, sizeof(val)))
3563 po->tp_tx_has_off = !!val;
3566 case PACKET_QDISC_BYPASS:
3570 if (optlen != sizeof(val))
3572 if (copy_from_user(&val, optval, sizeof(val)))
3575 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3579 return -ENOPROTOOPT;
3583 static int packet_getsockopt(struct socket *sock, int level, int optname,
3584 char __user *optval, int __user *optlen)
3587 int val, lv = sizeof(val);
3588 struct sock *sk = sock->sk;
3589 struct packet_sock *po = pkt_sk(sk);
3591 union tpacket_stats_u st;
3592 struct tpacket_rollover_stats rstats;
3594 if (level != SOL_PACKET)
3595 return -ENOPROTOOPT;
3597 if (get_user(len, optlen))
3604 case PACKET_STATISTICS:
3605 spin_lock_bh(&sk->sk_receive_queue.lock);
3606 memcpy(&st, &po->stats, sizeof(st));
3607 memset(&po->stats, 0, sizeof(po->stats));
3608 spin_unlock_bh(&sk->sk_receive_queue.lock);
3610 if (po->tp_version == TPACKET_V3) {
3611 lv = sizeof(struct tpacket_stats_v3);
3612 st.stats3.tp_packets += st.stats3.tp_drops;
3615 lv = sizeof(struct tpacket_stats);
3616 st.stats1.tp_packets += st.stats1.tp_drops;
3621 case PACKET_AUXDATA:
3624 case PACKET_ORIGDEV:
3627 case PACKET_VNET_HDR:
3628 val = po->has_vnet_hdr;
3630 case PACKET_VERSION:
3631 val = po->tp_version;
3634 if (len > sizeof(int))
3636 if (copy_from_user(&val, optval, len))
3640 val = sizeof(struct tpacket_hdr);
3643 val = sizeof(struct tpacket2_hdr);
3646 val = sizeof(struct tpacket3_hdr);
3652 case PACKET_RESERVE:
3653 val = po->tp_reserve;
3658 case PACKET_TIMESTAMP:
3659 val = po->tp_tstamp;
3663 ((u32)po->fanout->id |
3664 ((u32)po->fanout->type << 16) |
3665 ((u32)po->fanout->flags << 24)) :
3668 case PACKET_ROLLOVER_STATS:
3671 rstats.tp_all = atomic_long_read(&po->rollover->num);
3672 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3673 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3675 lv = sizeof(rstats);
3677 case PACKET_TX_HAS_OFF:
3678 val = po->tp_tx_has_off;
3680 case PACKET_QDISC_BYPASS:
3681 val = packet_use_direct_xmit(po);
3684 return -ENOPROTOOPT;
3689 if (put_user(len, optlen))
3691 if (copy_to_user(optval, data, len))
3697 static int packet_notifier(struct notifier_block *this,
3698 unsigned long msg, void *ptr)
3701 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3702 struct net *net = dev_net(dev);
3705 sk_for_each_rcu(sk, &net->packet.sklist) {
3706 struct packet_sock *po = pkt_sk(sk);
3709 case NETDEV_UNREGISTER:
3711 packet_dev_mclist_delete(dev, &po->mclist);
3715 if (dev->ifindex == po->ifindex) {
3716 spin_lock(&po->bind_lock);
3718 __unregister_prot_hook(sk, false);
3719 sk->sk_err = ENETDOWN;
3720 if (!sock_flag(sk, SOCK_DEAD))
3721 sk->sk_error_report(sk);
3723 if (msg == NETDEV_UNREGISTER) {
3724 packet_cached_dev_reset(po);
3726 if (po->prot_hook.dev)
3727 dev_put(po->prot_hook.dev);
3728 po->prot_hook.dev = NULL;
3730 spin_unlock(&po->bind_lock);
3734 if (dev->ifindex == po->ifindex) {
3735 spin_lock(&po->bind_lock);
3737 register_prot_hook(sk);
3738 spin_unlock(&po->bind_lock);
3748 static int packet_ioctl(struct socket *sock, unsigned int cmd,
3751 struct sock *sk = sock->sk;
3756 int amount = sk_wmem_alloc_get(sk);
3758 return put_user(amount, (int __user *)arg);
3762 struct sk_buff *skb;
3765 spin_lock_bh(&sk->sk_receive_queue.lock);
3766 skb = skb_peek(&sk->sk_receive_queue);
3769 spin_unlock_bh(&sk->sk_receive_queue.lock);
3770 return put_user(amount, (int __user *)arg);
3773 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3775 return sock_get_timestampns(sk, (struct timespec __user *)arg);
3785 case SIOCGIFBRDADDR:
3786 case SIOCSIFBRDADDR:
3787 case SIOCGIFNETMASK:
3788 case SIOCSIFNETMASK:
3789 case SIOCGIFDSTADDR:
3790 case SIOCSIFDSTADDR:
3792 return inet_dgram_ops.ioctl(sock, cmd, arg);
3796 return -ENOIOCTLCMD;
3801 static unsigned int packet_poll(struct file *file, struct socket *sock,
3804 struct sock *sk = sock->sk;
3805 struct packet_sock *po = pkt_sk(sk);
3806 unsigned int mask = datagram_poll(file, sock, wait);
3808 spin_lock_bh(&sk->sk_receive_queue.lock);
3809 if (po->rx_ring.pg_vec) {
3810 if (!packet_previous_rx_frame(po, &po->rx_ring,
3812 mask |= POLLIN | POLLRDNORM;
3814 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
3816 spin_unlock_bh(&sk->sk_receive_queue.lock);
3817 spin_lock_bh(&sk->sk_write_queue.lock);
3818 if (po->tx_ring.pg_vec) {
3819 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3820 mask |= POLLOUT | POLLWRNORM;
3822 spin_unlock_bh(&sk->sk_write_queue.lock);
3827 /* Dirty? Well, I still did not learn better way to account
3831 static void packet_mm_open(struct vm_area_struct *vma)
3833 struct file *file = vma->vm_file;
3834 struct socket *sock = file->private_data;
3835 struct sock *sk = sock->sk;
3838 atomic_inc(&pkt_sk(sk)->mapped);
3841 static void packet_mm_close(struct vm_area_struct *vma)
3843 struct file *file = vma->vm_file;
3844 struct socket *sock = file->private_data;
3845 struct sock *sk = sock->sk;
3848 atomic_dec(&pkt_sk(sk)->mapped);
3851 static const struct vm_operations_struct packet_mmap_ops = {
3852 .open = packet_mm_open,
3853 .close = packet_mm_close,
3856 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3861 for (i = 0; i < len; i++) {
3862 if (likely(pg_vec[i].buffer)) {
3863 if (is_vmalloc_addr(pg_vec[i].buffer))
3864 vfree(pg_vec[i].buffer);
3866 free_pages((unsigned long)pg_vec[i].buffer,
3868 pg_vec[i].buffer = NULL;
3874 static char *alloc_one_pg_vec_page(unsigned long order)
3877 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3878 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3880 buffer = (char *) __get_free_pages(gfp_flags, order);
3884 /* __get_free_pages failed, fall back to vmalloc */
3885 buffer = vzalloc((1 << order) * PAGE_SIZE);
3889 /* vmalloc failed, lets dig into swap here */
3890 gfp_flags &= ~__GFP_NORETRY;
3891 buffer = (char *) __get_free_pages(gfp_flags, order);
3895 /* complete and utter failure */
3899 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
3901 unsigned int block_nr = req->tp_block_nr;
3905 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
3906 if (unlikely(!pg_vec))
3909 for (i = 0; i < block_nr; i++) {
3910 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
3911 if (unlikely(!pg_vec[i].buffer))
3912 goto out_free_pgvec;
3919 free_pg_vec(pg_vec, order, block_nr);
3924 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
3925 int closing, int tx_ring)
3927 struct pgv *pg_vec = NULL;
3928 struct packet_sock *po = pkt_sk(sk);
3929 int was_running, order = 0;
3930 struct packet_ring_buffer *rb;
3931 struct sk_buff_head *rb_queue;
3934 /* Added to avoid minimal code churn */
3935 struct tpacket_req *req = &req_u->req;
3937 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3938 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3939 WARN(1, "Tx-ring is not supported.\n");
3943 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3944 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3948 if (atomic_read(&po->mapped))
3950 if (packet_read_pending(rb))
3954 if (req->tp_block_nr) {
3955 /* Sanity tests and some calculations */
3957 if (unlikely(rb->pg_vec))
3960 switch (po->tp_version) {
3962 po->tp_hdrlen = TPACKET_HDRLEN;
3965 po->tp_hdrlen = TPACKET2_HDRLEN;
3968 po->tp_hdrlen = TPACKET3_HDRLEN;
3973 if (unlikely((int)req->tp_block_size <= 0))
3975 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
3977 if (po->tp_version >= TPACKET_V3 &&
3978 (int)(req->tp_block_size -
3979 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3981 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
3984 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
3987 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3988 if (unlikely(rb->frames_per_block <= 0))
3990 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3995 order = get_order(req->tp_block_size);
3996 pg_vec = alloc_pg_vec(req, order);
3997 if (unlikely(!pg_vec))
3999 switch (po->tp_version) {
4001 /* Transmit path is not supported. We checked
4002 * it above but just being paranoid
4005 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
4014 if (unlikely(req->tp_frame_nr))
4020 /* Detach socket from network */
4021 spin_lock(&po->bind_lock);
4022 was_running = po->running;
4026 __unregister_prot_hook(sk, false);
4028 spin_unlock(&po->bind_lock);
4033 mutex_lock(&po->pg_vec_lock);
4034 if (closing || atomic_read(&po->mapped) == 0) {
4036 spin_lock_bh(&rb_queue->lock);
4037 swap(rb->pg_vec, pg_vec);
4038 rb->frame_max = (req->tp_frame_nr - 1);
4040 rb->frame_size = req->tp_frame_size;
4041 spin_unlock_bh(&rb_queue->lock);
4043 swap(rb->pg_vec_order, order);
4044 swap(rb->pg_vec_len, req->tp_block_nr);
4046 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4047 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4048 tpacket_rcv : packet_rcv;
4049 skb_queue_purge(rb_queue);
4050 if (atomic_read(&po->mapped))
4051 pr_err("packet_mmap: vma is busy: %d\n",
4052 atomic_read(&po->mapped));
4054 mutex_unlock(&po->pg_vec_lock);
4056 spin_lock(&po->bind_lock);
4059 register_prot_hook(sk);
4061 spin_unlock(&po->bind_lock);
4062 if (closing && (po->tp_version > TPACKET_V2)) {
4063 /* Because we don't support block-based V3 on tx-ring */
4065 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
4070 free_pg_vec(pg_vec, order, req->tp_block_nr);
4075 static int packet_mmap(struct file *file, struct socket *sock,
4076 struct vm_area_struct *vma)
4078 struct sock *sk = sock->sk;
4079 struct packet_sock *po = pkt_sk(sk);
4080 unsigned long size, expected_size;
4081 struct packet_ring_buffer *rb;
4082 unsigned long start;
4089 mutex_lock(&po->pg_vec_lock);
4092 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4094 expected_size += rb->pg_vec_len
4100 if (expected_size == 0)
4103 size = vma->vm_end - vma->vm_start;
4104 if (size != expected_size)
4107 start = vma->vm_start;
4108 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4109 if (rb->pg_vec == NULL)
4112 for (i = 0; i < rb->pg_vec_len; i++) {
4114 void *kaddr = rb->pg_vec[i].buffer;
4117 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4118 page = pgv_to_page(kaddr);
4119 err = vm_insert_page(vma, start, page);
4128 atomic_inc(&po->mapped);
4129 vma->vm_ops = &packet_mmap_ops;
4133 mutex_unlock(&po->pg_vec_lock);
4137 static const struct proto_ops packet_ops_spkt = {
4138 .family = PF_PACKET,
4139 .owner = THIS_MODULE,
4140 .release = packet_release,
4141 .bind = packet_bind_spkt,
4142 .connect = sock_no_connect,
4143 .socketpair = sock_no_socketpair,
4144 .accept = sock_no_accept,
4145 .getname = packet_getname_spkt,
4146 .poll = datagram_poll,
4147 .ioctl = packet_ioctl,
4148 .listen = sock_no_listen,
4149 .shutdown = sock_no_shutdown,
4150 .setsockopt = sock_no_setsockopt,
4151 .getsockopt = sock_no_getsockopt,
4152 .sendmsg = packet_sendmsg_spkt,
4153 .recvmsg = packet_recvmsg,
4154 .mmap = sock_no_mmap,
4155 .sendpage = sock_no_sendpage,
4158 static const struct proto_ops packet_ops = {
4159 .family = PF_PACKET,
4160 .owner = THIS_MODULE,
4161 .release = packet_release,
4162 .bind = packet_bind,
4163 .connect = sock_no_connect,
4164 .socketpair = sock_no_socketpair,
4165 .accept = sock_no_accept,
4166 .getname = packet_getname,
4167 .poll = packet_poll,
4168 .ioctl = packet_ioctl,
4169 .listen = sock_no_listen,
4170 .shutdown = sock_no_shutdown,
4171 .setsockopt = packet_setsockopt,
4172 .getsockopt = packet_getsockopt,
4173 .sendmsg = packet_sendmsg,
4174 .recvmsg = packet_recvmsg,
4175 .mmap = packet_mmap,
4176 .sendpage = sock_no_sendpage,
4179 static const struct net_proto_family packet_family_ops = {
4180 .family = PF_PACKET,
4181 .create = packet_create,
4182 .owner = THIS_MODULE,
4185 static struct notifier_block packet_netdev_notifier = {
4186 .notifier_call = packet_notifier,
4189 #ifdef CONFIG_PROC_FS
4191 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
4194 struct net *net = seq_file_net(seq);
4197 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
4200 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4202 struct net *net = seq_file_net(seq);
4203 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
4206 static void packet_seq_stop(struct seq_file *seq, void *v)
4212 static int packet_seq_show(struct seq_file *seq, void *v)
4214 if (v == SEQ_START_TOKEN)
4215 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4217 struct sock *s = sk_entry(v);
4218 const struct packet_sock *po = pkt_sk(s);
4221 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
4223 atomic_read(&s->sk_refcnt),
4228 atomic_read(&s->sk_rmem_alloc),
4229 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
4236 static const struct seq_operations packet_seq_ops = {
4237 .start = packet_seq_start,
4238 .next = packet_seq_next,
4239 .stop = packet_seq_stop,
4240 .show = packet_seq_show,
4243 static int packet_seq_open(struct inode *inode, struct file *file)
4245 return seq_open_net(inode, file, &packet_seq_ops,
4246 sizeof(struct seq_net_private));
4249 static const struct file_operations packet_seq_fops = {
4250 .owner = THIS_MODULE,
4251 .open = packet_seq_open,
4253 .llseek = seq_lseek,
4254 .release = seq_release_net,
4259 static int __net_init packet_net_init(struct net *net)
4261 mutex_init(&net->packet.sklist_lock);
4262 INIT_HLIST_HEAD(&net->packet.sklist);
4264 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
4270 static void __net_exit packet_net_exit(struct net *net)
4272 remove_proc_entry("packet", net->proc_net);
4275 static struct pernet_operations packet_net_ops = {
4276 .init = packet_net_init,
4277 .exit = packet_net_exit,
4281 static void __exit packet_exit(void)
4283 unregister_netdevice_notifier(&packet_netdev_notifier);
4284 unregister_pernet_subsys(&packet_net_ops);
4285 sock_unregister(PF_PACKET);
4286 proto_unregister(&packet_proto);
4289 static int __init packet_init(void)
4291 int rc = proto_register(&packet_proto, 0);
4296 sock_register(&packet_family_ops);
4297 register_pernet_subsys(&packet_net_ops);
4298 register_netdevice_notifier(&packet_netdev_notifier);
4303 module_init(packet_init);
4304 module_exit(packet_exit);
4305 MODULE_LICENSE("GPL");
4306 MODULE_ALIAS_NETPROTO(PF_PACKET);