flannel网络环境验证

安装brctl命令
yum install -y bridge-utils

    cni0只有在pod运行时才会出现。

tcpdump -i ens192 -nn icmp

可以看到veth这些接口都是桥接到cni0上的。

yum install -y tcpdump

[root@localhost /]#
[root@localhost /]# tcpdump -i ens3 -nn host 192.168.27.35


    brctl show表示查看已有网桥。

1 2 3 4 5 6 7 8 9 10

[root@node1 ~]#  tcpdump -i cni0 -nn icmp tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on cni0, link-type EN10MB (Ethernet), capture size 262144 bytes 23:40:11.370754 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 4864, seq 96, length 64 23:40:11.370988 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 4864, seq 96, length 64 23:40:12.370888 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 4864, seq 97, length 64 23:40:12.371090 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 4864, seq 97, length 64 ^X23:40:13.371015 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 4864, seq 98, length 64 23:40:13.371239 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 4864, seq 98, length 64 23:40:14.371128 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 4864, seq 99, length 64

    可以看到，在node节点，可以在cni0端口上抓到容器里面的Ping时的包。

    其实，上面ping时的数据流是先从cni0进来，然后从flannel.1出去，最后借助物理网卡ens32发出去。所以，我们在flannel.1上也能抓到包：

1 2 3 4 5 6 7

[root@node1 ~]#  tcpdump -i flannel.1 -nn icmp tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on flannel.1, link-type EN10MB (Ethernet), capture size 262144 bytes 03:12:36.823315 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 4864, seq 12840, length 64 03:12:36.823496 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 4864, seq 12840, length 64 03:12:37.823490 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 4864, seq 12841, length 64 03:12:37.823634 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 4864, seq 12841, length 64

[root@node1 /]#
[root@node1 /]# ip route show
default via 192.168.27.1 dev ens3
10.244.0.0/24 via 10.244.0.0 dev flannel.1 onlink
10.244.1.0/24 dev cni0 proto kernel scope link src 10.244.1.1
10.244.2.0/24 via 10.244.2.0 dev flannel.1 onlink
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
192.168.27.0/24 dev ens3 proto kernel scope link src 192.168.27.34
[root@node1 /]#
都有指向对端pod的路由从本地的flannel.1端口转出去
[root@node2 ~]#
[root@node2 ~]# ip route show
default via 192.168.27.1 dev ens3
10.244.0.0/24 via 10.244.0.0 dev flannel.1 onlink
10.244.1.0/24 via 10.244.1.0 dev flannel.1 onlink
10.244.2.0/24 dev cni0 proto kernel scope link src 10.244.2.1
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
192.168.27.0/24 dev ens3 proto kernel scope link src 192.168.27.35
[root@node2 ~]#

报文区别：
17:52:44.426638 IP (tos 0x0, ttl 64, id 27637, offset 0, flags [none], proto UDP (17), length 134)
    192.168.27.35.54674 > 192.168.27.34.8472: [no cksum] OTV, flags [I] (0x08), overlay 0, instance 1
IP (tos 0x0, ttl 63, id 42775, offset 0, flags [DF], proto ICMP (1), length 84)
    10.244.2.4 > 10.244.1.2: ICMP echo request, id 4096, seq 0, length 64
17:52:44.426788 IP (tos 0x0, ttl 64, id 9501, offset 0, flags [none], proto UDP (17), length 134)
    192.168.27.34.41553 > 192.168.27.35.8472: [no cksum] OTV, flags [I] (0x08), overlay 0, instance 1
IP (tos 0x0, ttl 63, id 64851, offset 0, flags [none], proto ICMP (1), length 84)
    10.244.1.2 > 10.244.2.4: ICMP echo reply, id 4096, seq 0, length 64
17:56:36.693508 IP (tos 0x0, ttl 64, id 16481, offset 0, flags [DF], proto ICMP (1), length 84)
    192.168.27.35 > 192.168.27.34: ICMP echo request, id 2048, seq 6, length 64
17:56:36.693558 IP (tos 0x0, ttl 64, id 29100, offset 0, flags [none], proto ICMP (1), length 84)
    192.168.27.34 > 192.168.27.35: ICMP echo reply, id 2048, seq 6, length 64

  同样，在ens192物理网卡上也能抓到包： 

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17

[root@node1 ~]# tcpdump -i ens192 -nn host 172.16.1.102  #172.16.1.102是node2的物理ip tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on ens192, link-type EN10MB (Ethernet), capture size 262144 bytes 10:59:24.234174 IP 172.16.1.101.60617 > 172.16.1.102.8472: OTV, flags [I] (0x08), overlay 0, instance 1 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 7168, seq 0, length 64 10:59:24.234434 IP 172.16.1.102.54894 > 172.16.1.101.8472: OTV, flags [I] (0x08), overlay 0, instance 1 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 7168, seq 0, length 64 10:59:25.234301 IP 172.16.1.101.60617 > 172.16.1.102.8472: OTV, flags [I] (0x08), overlay 0, instance 1 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 7168, seq 1, length 64 10:59:25.234469 IP 172.16.1.102.54894 > 172.16.1.101.8472: OTV, flags [I] (0x08), overlay 0, instance 1 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 7168, seq 1, length 64 10:59:26.234415 IP 172.16.1.101.60617 > 172.16.1.102.8472: OTV, flags [I] (0x08), overlay 0, instance 1 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 7168, seq 2, length 64 10:59:26.234592 IP 172.16.1.102.54894 > 172.16.1.101.8472: OTV, flags [I] (0x08), overlay 0, instance 1 IP 10.244.2.88 > 10.244.1.97: ICMP echo reply, id 7168, seq 2, length 64 10:59:27.234528 IP 172.16.1.101.60617 > 172.16.1.102.8472: OTV, flags [I] (0x08), overlay 0, instance 1 IP 10.244.1.97 > 10.244.2.88: ICMP echo request, id 7168, seq 3, length 64