Linux 收包和发包流程

/**
 * ixgbe_alloc_q_vector - Allocate memory for a single interrupt vector
 * @adapter: board private structure to initialize
 * @v_count: q_vectors allocated on adapter, used for ring interleaving
 * @v_idx: index of vector in adapter struct
 * @txr_count: total number of Tx rings to allocate
 * @txr_idx: index of first Tx ring to allocate
 * @xdp_count: total number of XDP rings to allocate
 * @xdp_idx: index of first XDP ring to allocate
 * @rxr_count: total number of Rx rings to allocate
 * @rxr_idx: index of first Rx ring to allocate
 *
 * We allocate one q_vector.  If allocation fails we return -ENOMEM.
 **/
static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
                int v_count, int v_idx,
                int txr_count, int txr_idx,
                int xdp_count, int xdp_idx,
                int rxr_count, int rxr_idx)
{
	/* ... */

    /* initialize NAPI */
    netif_napi_add(adapter->netdev, &q_vector->napi,
               ixgbe_poll, 64);
}

static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
{
    struct ixgbe_q_vector *q_vector = data;

    /* EIAM disabled interrupts (on this vector) for us */

    if (q_vector->rx.ring || q_vector->tx.ring)
        napi_schedule_irqoff(&q_vector->napi);

    return IRQ_HANDLED;
}

static inline void ____napi_schedule(struct softnet_data *sd,
                     struct napi_struct *napi)
{
	/* ... */

    list_add_tail(&napi->poll_list, &sd->poll_list);
    __raise_softirq_irqoff(NET_RX_SOFTIRQ);
}

static __latent_entropy void net_rx_action(struct softirq_action *h)
{
    struct softnet_data *sd = this_cpu_ptr(&softnet_data);
    unsigned long time_limit = jiffies +
        usecs_to_jiffies(netdev_budget_usecs);
    int budget = netdev_budget;
    LIST_HEAD(list);
    LIST_HEAD(repoll);

    local_irq_disable();
    list_splice_init(&sd->poll_list, &list);
    local_irq_enable();

    for (;;) {
        struct napi_struct *n;

        if (list_empty(&list)) {
            if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
                return;
            break;
        }

        n = list_first_entry(&list, struct napi_struct, poll_list);
        budget -= napi_poll(n, &repoll);

        /* If softirq window is exhausted then punt.
         * Allow this to run for 2 jiffies since which will allow
         * an average latency of 1.5/HZ.
         */
        if (unlikely(budget <= 0 ||
                 time_after_eq(jiffies, time_limit))) {
            sd->time_squeeze++;
            break;
        }
    }

    local_irq_disable();

    list_splice_tail_init(&sd->poll_list, &list);
    list_splice_tail(&repoll, &list);
    list_splice(&list, &sd->poll_list);
    if (!list_empty(&sd->poll_list))
        __raise_softirq_irqoff(NET_RX_SOFTIRQ);

    net_rps_action_and_irq_enable(sd);
}

/**
 * ixgbe_poll - NAPI Rx polling callback
 * @napi: structure for representing this polling device
 * @budget: how many packets driver is allowed to clean
 *
 * This function is used for legacy and MSI, NAPI mode
 **/
int ixgbe_poll(struct napi_struct *napi, int budget)
{

  /* ... */

  // 为什么会在RX软中断中清理tx？
  ixgbe_for_each_ring(ring, q_vector->tx) {
    bool wd = ring->xsk_pool ?
        ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) :
        ixgbe_clean_tx_irq(q_vector, ring, budget);

    if (!wd)
      clean_complete = false;
  }

  /* ... */

  // rx ring buffer处理
  ixgbe_for_each_ring(ring, q_vector->rx) {
    int cleaned = ring->xsk_pool ?
            ixgbe_clean_rx_irq_zc(q_vector, ring,
                per_ring_budget) :
            ixgbe_clean_rx_irq(q_vector, ring,
             per_ring_budget);

    work_done += cleaned;
    if (cleaned >= per_ring_budget)
      clean_complete = false;
  }

  /* ... */

  return min(work_done, budget - 1);
}

// drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c:ixgbe_clean_rx_irq_zc
xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp);

// drivers/net/ethernet/intel/ixgbe/ixgbe_main.c:ixgbe_clean_rx_irq
skb = ixgbe_run_xdp(adapter, rx_ring, &xdp);

// drivers/net/ethernet/intel/ixgbe/ixgbe_main.c:ixgbe_clean_rx_irq

	rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt);

    /* retrieve a buffer from the ring */
    if (!skb) {
      unsigned char *hard_start;

      hard_start = page_address(rx_buffer->page) +
             rx_buffer->page_offset - offset;
      xdp_prepare_buff(&xdp, hard_start, offset, size, true);
#if (PAGE_SIZE > 4096)
      /* At larger PAGE_SIZE, frame_sz depend on len size */
      xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size);
#endif
      skb = ixgbe_run_xdp(adapter, rx_ring, &xdp);
    }

// drivers/net/ethernet/intel/ixgbe/ixgbe.h

struct ixgbe_rx_buffer {
  union {
    struct {
      struct sk_buff *skb;
      dma_addr_t dma;
      struct page *page;
      __u32 page_offset;
      __u16 pagecnt_bias;
    };
    struct {
      bool discard;
      struct xdp_buff *xdp;
    };
  };
};

// net/core/dev.c:__netif_receive_skb_core 抓包hook逻辑
  list_for_each_entry_rcu(ptype, &ptype_all, list) {
    if (pt_prev)
      ret = deliver_skb(skb, pt_prev, orig_dev);
    pt_prev = ptype;
  }

  list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
    if (pt_prev)
      ret = deliver_skb(skb, pt_prev, orig_dev);
    pt_prev = ptype;
  }

// net/core/dev.c:__netif_receive_skb_core 协议栈回调
  type = skb->protocol;

  /* deliver only exact match when indicated */
  if (likely(!deliver_exact)) {
    deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
               &ptype_base[ntohs(type) &
               PTYPE_HASH_MASK]);
  }

  deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
             &orig_dev->ptype_specific);

  if (unlikely(skb->dev != orig_dev)) {
    deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
               &skb->dev->ptype_specific);
  }

static inline int deliver_skb(struct sk_buff *skb,
            struct packet_type *pt_prev,
            struct net_device *orig_dev)
{
  if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
    return -ENOMEM;
  refcount_inc(&skb->users);
  return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
}

static inline void deliver_ptype_list_skb(struct sk_buff *skb,
            struct packet_type **pt,
            struct net_device *orig_dev,
            __be16 type,
            struct list_head *ptype_list)
{
  struct packet_type *ptype, *pt_prev = *pt;

  list_for_each_entry_rcu(ptype, ptype_list, list) {
    if (ptype->type != type)
      continue;
    if (pt_prev)
      deliver_skb(skb, pt_prev, orig_dev);
    pt_prev = ptype;
  }
  *pt = pt_prev;
}

// include/linux/netdevice.h
struct packet_type {
  __be16      type; /* This is really htons(ether_type). */
  bool      ignore_outgoing;
  struct net_device *dev; /* NULL is wildcarded here       */
  int     (*func) (struct sk_buff *,
           struct net_device *,
           struct packet_type *,
           struct net_device *);
  void      (*list_func) (struct list_head *,
                struct packet_type *,
                struct net_device *);
  bool      (*id_match)(struct packet_type *ptype,
              struct sock *sk);
  void      *af_packet_priv;
  struct list_head  list;
};

// net/core/dev.c

static inline struct list_head *ptype_head(const struct packet_type *pt)
{
  if (pt->type == htons(ETH_P_ALL))
    return pt->dev ? &pt->dev->ptype_all : &ptype_all;
  else
    return pt->dev ? &pt->dev->ptype_specific :
         &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
}

/**
 *  dev_add_pack - add packet handler
 *  @pt: packet type declaration
 *
 *  Add a protocol handler to the networking stack. The passed &packet_type
 *  is linked into kernel lists and may not be freed until it has been
 *  removed from the kernel lists.
 *
 *  This call does not sleep therefore it can not
 *  guarantee all CPU's that are in middle of receiving packets
 *  will see the new packet type (until the next received packet).
 */

void dev_add_pack(struct packet_type *pt)
{
  struct list_head *head = ptype_head(pt);

  spin_lock(&ptype_lock);
  list_add_rcu(&pt->list, head);
  spin_unlock(&ptype_lock);
}
EXPORT_SYMBOL(dev_add_pack);

// net/ipv4/af_inet.c
static struct packet_type ip_packet_type __read_mostly = {
  .type = cpu_to_be16(ETH_P_IP),
  .func = ip_rcv,
  .list_func = ip_list_rcv,
};

    if (q->enqueue) {
        rc = __dev_xmit_skb(skb, q, dev, txq);
        goto out;
    }

// net/core/dev.c
int dev_tx_weight __read_mostly = 64;

// net/sched/sch_generic.c
static inline bool qdisc_restart(struct Qdisc *q, int *packets)
{
    spinlock_t *root_lock = NULL;
    struct netdev_queue *txq;
    struct net_device *dev;
    struct sk_buff *skb;
    bool validate;

    /* Dequeue packet */
    skb = dequeue_skb(q, &validate, packets);
    if (unlikely(!skb))
        return false;

    if (!(q->flags & TCQ_F_NOLOCK))
        root_lock = qdisc_lock(q);

    dev = qdisc_dev(q);
    txq = skb_get_tx_queue(dev, skb);

    return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
}

void __qdisc_run(struct Qdisc *q)
{
    int quota = dev_tx_weight;
    int packets;

    while (qdisc_restart(q, &packets)) {
        quota -= packets;
        if (quota <= 0) {
            if (q->flags & TCQ_F_NOLOCK)
                set_bit(__QDISC_STATE_MISSED, &q->state);
            else
                __netif_schedule(q);

            break;
        }
    }
}

// net/core/dev.c
static int xmit_one(struct sk_buff *skb, struct net_device *dev,
            struct netdev_queue *txq, bool more)
{
    unsigned int len;
    int rc;

    if (dev_nit_active(dev))
        dev_queue_xmit_nit(skb, dev);

    len = skb->len;
    PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
    trace_net_dev_start_xmit(skb, dev);
    rc = netdev_start_xmit(skb, dev, txq, more);
    trace_net_dev_xmit(skb, rc, dev, len);

    return rc;
}

struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
                    struct netdev_queue *txq, int *ret)
{
    struct sk_buff *skb = first;
    int rc = NETDEV_TX_OK;

    while (skb) {
        struct sk_buff *next = skb->next;

        skb_mark_not_on_list(skb);
        rc = xmit_one(skb, dev, txq, next != NULL);
        if (unlikely(!dev_xmit_complete(rc))) {
            skb->next = next;
            goto out;
        }

        skb = next;
        if (netif_tx_queue_stopped(txq) && skb) {
            rc = NETDEV_TX_BUSY;
            break;
        }
    }

out:
    *ret = rc;
    return skb;
}

// net/core/dev.c
void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
{
    unsigned long flags;

    if (unlikely(!skb))
        return;

    if (likely(refcount_read(&skb->users) == 1)) {
        smp_rmb();
        refcount_set(&skb->users, 0);
    } else if (likely(!refcount_dec_and_test(&skb->users))) {
        return;
    }
    get_kfree_skb_cb(skb)->reason = reason;
    local_irq_save(flags);
    skb->next = __this_cpu_read(softnet_data.completion_queue);
    __this_cpu_write(softnet_data.completion_queue, skb);
    raise_softirq_irqoff(NET_TX_SOFTIRQ);
    local_irq_restore(flags);
}

void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
{
    if (in_irq() || irqs_disabled())
        __dev_kfree_skb_irq(skb, reason);
    else
        dev_kfree_skb(skb);
}
EXPORT_SYMBOL(__dev_kfree_skb_any);

// net/core/dev.c:net_tx_action
	if (sd->completion_queue) {
        struct sk_buff *clist;

        local_irq_disable();
        clist = sd->completion_queue;
        sd->completion_queue = NULL;
        local_irq_enable();

        while (clist) {
            struct sk_buff *skb = clist;

            clist = clist->next;

            WARN_ON(refcount_read(&skb->users));
            if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
                trace_consume_skb(skb);
            else
                trace_kfree_skb(skb, net_tx_action);

            if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
                __kfree_skb(skb);
            else
                __kfree_skb_defer(skb);
        }
    }

// net/core/dev.c:xmit_one
    if (dev_nit_active(dev))
        dev_queue_xmit_nit(skb, dev);

// net/core/dev.c:__dev_queue_xmit
# ifdef CONFIG_NET_EGRESS
    if (static_branch_unlikely(&egress_needed_key)) {
        skb = sch_handle_egress(skb, &rc, dev);
        if (!skb)
            goto out;
    }
# endif

static struct tcf_proto_ops cls_bpf_ops __read_mostly = {
    .kind       =   "bpf",
    .owner      =   THIS_MODULE,
    .classify   =   cls_bpf_classify,
    .init       =   cls_bpf_init,
    .destroy    =   cls_bpf_destroy,
    .get        =   cls_bpf_get,
    .change     =   cls_bpf_change,
    .delete     =   cls_bpf_delete,
    .walk       =   cls_bpf_walk,
    .reoffload  =   cls_bpf_reoffload,
    .dump       =   cls_bpf_dump,
    .bind_class =   cls_bpf_bind_class,
};