Chinaunix首页 | 论坛 | 博客

随风去的ChinaUnix博客

首页　| 　博文目录　| 　关于我

随风去

  • 博客访问： 85788
  • 博文数量： 15
  • 博客积分： 0
  • 博客等级： 民兵
  • 技术积分： 210
  • 用 户 组： 普通用户
  • 注册时间： 2014-01-05 15:27
文章分类

全部博文（15）

文章存档

2014年（15）

我的朋友
最近访客
推荐博文
相关博文
ip分片以及网卡的卸载功能

分类： LINUX

2014-03-01 16:30:48

IP头限制一个包最大为64K，但是实际上一般没法一次发送这么大的包，以太网一般有MTU限制一次发包的大小
MTU一般为1500字节。因此发包过程涉及packet  fragmentation，该流程在L3层完成
TCP把类似的操作提到了L4层，引入了MSS的概念，这个一般是根据MTU生成的。
随着网卡速度的提升，网络收发包会占用大量cpu，为了尽量减少cpu的使用，网卡集成了很多之前软件完成的功能，即卸载功能
首先看不开启所有卸载功能，发包大于1500字节的情况。

例子：负载为25000字节
__ip_append_data函数中涉及skb包的分配，length为25000+L4=25008
  1. if (!skb)
  2.         goto alloc_new_skb;

  4.     while (length > 0) {
  5.         /* Check if the remaining data fits into current packet. */
  6.         copy = mtu - skb->len;
  7.         if (copy < length)
  8.             copy = maxfraglen - skb->len;
  9.         if (copy <= 0) {
  10.             char *data;
  11.             unsigned int datalen;
  12.             unsigned int fraglen;
  13.             unsigned int fraggap;
  14.             unsigned int alloclen;
  15.             struct sk_buff *skb_prev;
  16. alloc_new_skb:
  17.             skb_prev = skb;
  18.             if (skb_prev)
  19.                 fraggap = skb_prev->len - maxfraglen;
  20.             else
  21.                 fraggap = 0;

  23.             /*
  24.              * If remaining data exceeds the mtu,
  25.              * we know we need more fragment(s).
  26.              */
  27.             datalen = length + fraggap;
  28.             if (datalen > mtu - fragheaderlen)         
  29.                 datalen = maxfraglen - fragheaderlen; //maxfraglen=1500，fragheaderlen=20，datalen=1480
  30.             fraglen = datalen + fragheaderlen;

  32.             if ((flags & MSG_MORE) &&
  33.              !(rt->dst.dev->features&NETIF_F_SG))
  34.                 alloclen = mtu;
  35.             else
  36.                 alloclen = fraglen;  //alloclen=1500

  38.             alloclen += exthdrlen;

  40.             /* The last fragment gets additional space at tail.
  41.              * Note, with MSG_MORE we overallocate on fragments,
  42.              * because we have no idea what fragment will be
  43.              * the last.
  44.              */
  45.             if (datalen == length + fraggap)
  46.                 alloclen += rt->dst.trailer_len;

  48.             if (transhdrlen) {
  49.                 skb = sock_alloc_send_skb(sk,
  50.                         alloclen + hh_len + 15,   //第一个skb分配1500+16+15，在__alloc_skb,该大小会和L1 cacheline对齐（64），即变成1536
  51.                         (flags & MSG_DONTWAIT), &err);
  52.             } else {
  53.                 skb = NULL;
  54.                 if (atomic_read(&sk->sk_wmem_alloc) <=
  55.                  2 * sk->sk_sndbuf)
  56.                     skb = sock_wmalloc(sk,  
  57.                              alloclen + hh_len + 15, 1,
  58.                              sk->sk_allocation);
  59.                 if (unlikely(skb == NULL))
  60.                     err = -ENOBUFS;
  61.                 else
  62.                     /* only the initial fragment is
  63.                      time stamped */
  64.                     cork->tx_flags = 0;
  65.             }
  66.             if (skb == NULL)
  67.                 goto error;

  69.             /*
  70.              *    Fill in the control structures
  71.              */
  72.             skb->ip_summed = csummode;
  73.             skb->csum = 0;
  74.             skb_reserve(skb, hh_len);
  75.             skb_shinfo(skb)->tx_flags = cork->tx_flags;

  77.             /*
  78.              *    Find where to start putting bytes.
  79.              */
  80.             data = skb_put(skb, fraglen + exthdrlen); //skb->len+=1500
  81.             skb_set_network_header(skb, exthdrlen);
  82.             skb->transport_header = (skb->network_header +
  83.                          fragheaderlen);
  84.             data += fragheaderlen + exthdrlen;

  86.             if (fraggap) {
  87.                 skb->csum = skb_copy_and_csum_bits(
  88.                     skb_prev, maxfraglen,
  89.                     data + transhdrlen, fraggap, 0);
  90.                 skb_prev->csum = csum_sub(skb_prev->csum,
  91.                              skb->csum);
  92.                 data += fraggap;
  93.                 pskb_trim_unique(skb_prev, maxfraglen);
  94.             }

  96.             copy = datalen - transhdrlen - fraggap; //copy=MTU-L3-L4=1472
  97.                                                     //第二个以后的分片不再区别L4,每次拷贝1480字节
  98.             if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
  99.                 err = -EFAULT;
  100.                 kfree_skb(skb);
  101.                 goto error;
  102.             }

  104.             offset += copy;
  105.             length -= datalen - fraggap;
  106.             transhdrlen = 0;
  107.             exthdrlen = 0;
  108.             csummode = CHECKSUM_NONE;

  110.             /*
  111.              * Put the packet on the pending queue.
  112.              */
  113.             __skb_queue_tail(queue, skb);
  114.             continue;
  115.         }
分配多个skb，拷贝数据，注意length包含L4的头，第二个开始的skb不再区别L4头。


其中涉及SG卸载功能的地方有:

  1. if ((flags & MSG_MORE) &&
  2.      !(rt->dst.dev->features&NETIF_F_SG))
  3.         alloclen = mtu;
  4. else
  5.         alloclen = fraglen
即有SG功能的话,每次都是按需分配,如果没有SG的功能,同时又MORE的标志的话,按最大分段大小进行分配,这么做的原因在于SG可以支持数据包对应的内存不连续

  1.         if (!(rt->dst.dev->features&NETIF_F_SG)) {
  2.             unsigned int off;

  4.             off = skb->len;
  5.             if (getfrag(from, skb_put(skb, copy),
  6.                     offset, copy, off, skb) < 0) {
  7.                 __skb_trim(skb, off);
  8.                 err = -EFAULT;
  9.                 goto error;
  10.             }
  11.         } else {
  12.             int i = skb_shinfo(skb)->nr_frags;
  13.             skb_frag_t *frag = &skb_shinfo(skb)->frags[i-1];
  14.             struct page *page = cork->page;
  15.             int off = cork->off;
  16.             unsigned int left;

  18.             if (page && (left = PAGE_SIZE - off) > 0) {
  19.                 if (copy >= left)
  20.                     copy = left;
  21.                 if (page != frag->page) {
  22.                     if (i == MAX_SKB_FRAGS) {
  23.                         err = -EMSGSIZE;
  24.                         goto error;
  25.                     }
  26.                     get_page(page);
  27.                     skb_fill_page_desc(skb, i, page, off, 0);
  28.                     frag = &skb_shinfo(skb)->frags[i];
  29.                 }
  30.             } else if (i < MAX_SKB_FRAGS) {
  31.                 if (copy > PAGE_SIZE)
  32.                     copy = PAGE_SIZE;
  33.                 page = alloc_pages(sk->sk_allocation, 0);
  34.                 if (page == NULL) {
  35.                     err = -ENOMEM;
  36.                     goto error;
  37.                 }
  38.                 cork->page = page;
  39.                 cork->off = 0;

  41.                 skb_fill_page_desc(skb, i, page, 0, 0);
  42.                 frag = &skb_shinfo(skb)->frags[i];
  43.             } else {
  44.                 err = -EMSGSIZE;
  45.                 goto error;
  46.             }
  47.             if (getfrag(from, page_address(frag->page)+frag->page_offset+frag->size, offset, copy, skb->len, skb) < 0) {
  48.                 err = -EFAULT;
  49.                 goto error;
  50.             }
  51.             cork->off += copy;
  52.             frag->size += copy;
  53.             skb->len += copy;
  54.             skb->data_len += copy;
  55.             skb->truesize += copy;
  56.             atomic_add(copy, &sk->sk_wmem_alloc);
  57.         }
  58.         offset += copy;
  59.         length -= copy;
  60.     }
可以看到不管支不支持SG,都涉及内存的拷贝,只不过拷贝的目的地不一样而已。
理论上SG使得每个SKB支持16个页，大小为64K，但是实际上__ip_append_data分片的规格是MTU，大于MTU就会重新分配一个SKB
函数__ip_make_skb除了填充L3头外，还对skb的组织进行了修改：


所有的大小都会在第一个skb中体现，即本例中skb->len=25028，即25000+L3+L4
后续调用ip_finish_output进行发送，这里涉及另外一个卸载功能：GSO

  1. static int ip_finish_output(struct sk_buff *skb)
  2. {
  3. #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
  4.     /* Policy lookup after SNAT yielded a new policy */
  5.     if (skb_dst(skb)->xfrm != NULL) {
  6.         IPCB(skb)->flags |= IPSKB_REROUTED;
  7.         return dst_output(skb);
  8.     }
  9. #endif
  10.     if (skb->len > ip_skb_dst_mtu(skb) && !skb_is_gso(skb))
  11.         return ip_fragment(skb, ip_finish_output2);
  12.     else
  13.         return ip_finish_output2(skb);
  14. }
如果包的长度大于MTU，并且不支持GSO的话，先调用ip_fragment，否则直接调用ip_finish_output2

  1. int ip_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
  2. {
  3.     struct iphdr *iph;
  4.     int ptr;
  5.     struct net_device *dev;
  6.     struct sk_buff *skb2;
  7.     unsigned int mtu, hlen, left, len, ll_rs;
  8.     int offset;
  9.     __be16 not_last_frag;
  10.     struct rtable *rt = skb_rtable(skb);
  11.     int err = 0;

  13.     dev = rt->dst.dev;

  15.     /*
  16.      *    Point into the IP datagram header.
  17.      */

  19.     iph = ip_hdr(skb);

  21.     if (unlikely((iph->frag_off & htons(IP_DF)) && !skb->local_df)) {
  22.         IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
  23.         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
  24.              htonl(ip_skb_dst_mtu(skb)));
  25.         kfree_skb(skb);
  26.         return -EMSGSIZE;
  27.     }

  29.     /*
  30.      *    Setup starting values.
  31.      */

  33.     hlen = iph->ihl * 4;
  34.     mtu = dst_mtu(&rt->dst) - hlen;    /* Size of data space */
  35. #ifdef CONFIG_BRIDGE_NETFILTER
  36.     if (skb->nf_bridge)
  37.         mtu -= nf_bridge_mtu_reduction(skb);
  38. #endif
  39.     IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;

  41.     /* When frag_list is given, use it. First, check its validity:
  42.      * some transformers could create wrong frag_list or break existing
  43.      * one, it is not prohibited. In this case fall back to copying.
  44.      *
  45.      * LATER: this step can be merged to real generation of fragments,
  46.      * we can switch to copy when see the first bad fragment.
  47.      */
  48.     if (skb_has_frag_list(skb)) {
  49.         struct sk_buff *frag, *frag2;
  50.         int first_len = skb_pagelen(skb);  //pagelen包含主buffer以及fragments的大小，不包含frag_list

  52.         if (first_len - hlen > mtu ||  // 本例中skb不包含fragments，只有frag_list,first_len=mtu+hlen
  53.          ((first_len - hlen) & 7) ||  //条件头不满足，走fast流程 
  54.          ip_is_fragment(iph) ||
  55.          skb_cloned(skb))
  56.             goto slow_path;

  58.         skb_walk_frags(skb, frag) {  //对每个frag进行检查
  59.             /* Correct geometry. */
  60.             if (frag->len > mtu ||
  61.              ((frag->len & 7) && frag->next) ||
  62.              skb_headroom(frag) < hlen)
  63.                 goto slow_path_clean;

  65.             /* Partially cloned skb? */
  66.             if (skb_shared(frag))
  67.                 goto slow_path_clean;

  69.             BUG_ON(frag->sk);
  70.             if (skb->sk) {
  71.                 frag->sk = skb->sk;
  72.                 frag->destructor = sock_wfree;
  73.             }
  74.             skb->truesize -= frag->truesize;
  75.         }

  77.         /* Everything is OK. */

  79.         err = 0;
  80.         offset = 0;
  81.         frag = skb_shinfo(skb)->frag_list;
  82.         skb_frag_list_init(skb);
  83.         skb->data_len = first_len - skb_headlen(skb);  //data_len=0
  84.         skb->len = first_len;            //len=1500
  85.         iph->tot_len = htons(first_len);
  86.         iph->frag_off = htons(IP_MF);
  87.         ip_send_check(iph);

  89.         for (;;) {
  90.             /* Prepare header of the next frame,
  91.              * before previous one went down. */
  92.             if (frag) {
  93.                 frag->ip_summed = CHECKSUM_NONE;
  94.                 skb_reset_transport_header(frag);
  95.                 __skb_push(frag, hlen);
  96.                 skb_reset_network_header(frag);
  97.                 memcpy(skb_network_header(frag), iph, hlen);  //赋值IP头
  98.                 iph = ip_hdr(frag);
  99.                 iph->tot_len = htons(frag->len);
  100.                 ip_copy_metadata(frag, skb);
  101.                 if (offset == 0)
  102.                     ip_options_fragment(frag);
  103.                 offset += skb->len - hlen;
  104.                 iph->frag_off = htons(offset>>3);
  105.                 if (frag->next != NULL)
  106.                     iph->frag_off |= htons(IP_MF);
  107.                 /* Ready, complete checksum */
  108.                 ip_send_check(iph);
  109.             }

  111.             err = output(skb);  //调用ip_finish_output2，除了第一个skb有L4头外，其他的skb没有

  113.             if (!err)
  114.                 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
  115.             if (err || !frag)
  116.                 break;

  118.             skb = frag;
  119.             frag = skb->next;
  120.             skb->next = NULL;
  121.         }

  123.         if (err == 0) {
  124.             IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
  125.             return 0;
  126.         }

  128.         while (frag) {
  129.             skb = frag->next;
  130.             kfree_skb(frag);
  131.             frag = skb;
  132.         }
  133.         IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
  134.         return err;

  136. slow_path_clean:
  137.         skb_walk_frags(skb, frag2) {
  138.             if (frag2 == frag)
  139.                 break;
  140.             frag2->sk = NULL;
  141.             frag2->destructor = NULL;
  142.             skb->truesize += frag2->truesize;
  143.         }
  144.     }

  146. slow_path:
  147.     left = skb->len - hlen;        /* Space per frame */
  148.     ptr = hlen;        /* Where to start from */

  150.     /* for bridged IP traffic encapsulated inside f.e. a vlan header,
  151.      * we need to make room for the encapsulating header
  152.      */
  153.     ll_rs = LL_RESERVED_SPACE_EXTRA(rt->dst.dev, nf_bridge_pad(skb));

  155.     /*
  156.      *    Fragment the datagram.
  157.      */

  159.     offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
  160.     not_last_frag = iph->frag_off & htons(IP_MF);

  162.     /*
  163.      *    Keep copying data until we run out.
  164.      */

  166.     while (left > 0) {
  167.         len = left;
  168.         /* IF: it doesn't fit, use 'mtu' - the data space left */
  169.         if (len > mtu)
  170.             len = mtu;
  171.         /* IF: we are not sending up to and including the packet end
  172.          then align the next start on an eight byte boundary */
  173.         if (len < left)    {
  174.             len &= ~7;
  175.         }
  176.         /*
  177.          *    Allocate buffer.
  178.          */

  180.         if ((skb2 = alloc_skb(len+hlen+ll_rs, GFP_ATOMIC)) == NULL) {
  181.             NETDEBUG(KERN_INFO "IP: frag: no memory for new fragment!\n");
  182.             err = -ENOMEM;
  183.             goto fail;
  184.         }

  186.         /*
  187.          *    Set up data on packet
  188.          */

  190.         ip_copy_metadata(skb2, skb);
  191.         skb_reserve(skb2, ll_rs);
  192.         skb_put(skb2, len + hlen);
  193.         skb_reset_network_header(skb2);
  194.         skb2->transport_header = skb2->network_header + hlen;

  196.         /*
  197.          *    Charge the memory for the fragment to any owner
  198.          *    it might possess
  199.          */

  201.         if (skb->sk)
  202.             skb_set_owner_w(skb2, skb->sk);

  204.         /*
  205.          *    Copy the packet header into the new buffer.
  206.          */

  208.         skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);

  210.         /*
  211.          *    Copy a block of the IP datagram.
  212.          */
  213.         if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
  214.             BUG();
  215.         left -= len;

  217.         /*
  218.          *    Fill in the new header fields.
  219.          */
  220.         iph = ip_hdr(skb2);
  221.         iph->frag_off = htons((offset >> 3));

  223.         /* ANK: dirty, but effective trick. Upgrade options only if
  224.          * the segment to be fragmented was THE FIRST (otherwise,
  225.          * options are already fixed) and make it ONCE
  226.          * on the initial skb, so that all the following fragments
  227.          * will inherit fixed options.
  228.          */
  229.         if (offset == 0)
  230.             ip_options_fragment(skb);

  232.         /*
  233.          *    Added AC : If we are fragmenting a fragment that's not the
  234.          *         last fragment then keep MF on each bit
  235.          */
  236.         if (left > 0 || not_last_frag)
  237.             iph->frag_off |= htons(IP_MF);
  238.         ptr += len;
  239.         offset += len;

  241.         /*
  242.          *    Put this fragment into the sending queue.
  243.          */
  244.         iph->tot_len = htons(len + hlen);

  246.         ip_send_check(iph);

  248.         err = output(skb2);
  249.         if (err)
  250.             goto fail;

  252.         IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
  253.     }
  254.     kfree_skb(skb);
  255.     IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
  256.     return err;

  258. fail:
  259.     kfree_skb(skb);
  260.     IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGFAILS);
  261.     return err;
  262. }
可以看到fast流程就是对没有skb增加L3头，然后调用ip_finish_output2发送。



阅读(2998) | 评论(0) | 转发(0) |
0

上一篇：socket的初始化

下一篇：网卡驱动的收发包流程

给主人留下些什么吧！~~
关于我们 | 关于IT168 | 联系方式 | 广告合作 | 法律声明 | 免费注册

Copyright 2001-2010 ChinaUnix.net All Rights Reserved 北京皓辰网域网络信息技术有限公司. 版权所有

感谢所有关心和支持过ChinaUnix的朋友们

16024965号-6