LVS 内核实现分析（2）

3 重要数据结构

  这里重点介绍其中用到的三个数据结构，

3.1 ip_vs_conn 

/*
 *    IP_VS structure allocated for each dynamically scheduled connection每一个动态的连接会
 有一个ip_vs_conn结构，它包含客户端，IPVS，RealServer的地址和端口信息*/
struct ip_vs_conn {
    struct list_head        c_list; /* hashed list heads */

    /* Protocol, addresses and port numbers */
    u16                      af;        /* address family */
    union nf_inet_addr       caddr; /* client address */
    union nf_inet_addr       vaddr; /* virtual address */
    union nf_inet_addr       daddr; /* destination address */
    __be16                   cport;             /*客户端的端口*/
    __be16                   vport;            /*IPVS的端口*/
    __be16                   dport;            /*RS的端口*/
    __u16                   protocol; /* Which protocol (TCP/UDP)协议号 */

    /* counter and timer ip_vs_conn对象的使用计数。其初值为1，__ip_vs_conn_in_get/__ip_vs_conn_put成对调用 */
    atomic_t        refcnt;        /* reference count */
    struct timer_list    timer;        /* Expiration timerip_vs_conn对象的生存期，当timer到期时，对象被销毁 */
    volatile unsigned long    timeout;    /* timeoutip_vs_conn对象动态的超时时间，每当对象操作完毕，timeout值用来更新timer，以延长对象的生存期。timeout受连接状态等的影响 */

    /* Flags and state transition */
    spinlock_t              lock; /* lock for state transition */
    volatile __u16          flags; /* status flags */
    volatile __u16          state; /* state info */
    volatile __u16          old_state; /* old state, to be used for
                         * state transition triggerd
                         * synchronization
                         */

    /* Control members */
    struct ip_vs_conn *control; /* Master control connection */
    atomic_t                n_control; /* Number of controlled ones */
    struct ip_vs_dest *dest; /* real server 指向此连接对象对应的ip_vs_dest对象*/
    atomic_t                in_pkts; /* incoming packet counter */

    /* packet transmitter for different forwarding methods. If it
       mangles the packet, it must return NF_DROP or better NF_STOLEN,
       otherwise this must be changed to a sk_buff **.
     */
    int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp,
               struct ip_vs_protocol *pp); /*不同的发包函数，三种模式对应三个不同的发包函数*/

    /* Note: we can group the following members into a structure,
     in order to save more space, and the following members are
       only used in VS/NAT anyway */
    struct ip_vs_app *app; /* bound ip_vs_app object */
    void *app_data; /* Application private data */
    struct ip_vs_seq        in_seq; /* incoming seq. struct */
    struct ip_vs_seq        out_seq; /* outgoing seq. struct */
};

3.2  ip_vs_dest

/*
 *    The real server destination forwarding entry
 *    with ip address, port number, and so on.该结构体主要描述的是real server的相关信息，也是ipvsdm配置的到内核的相关信息
 */
struct ip_vs_dest {
    struct list_head    n_list; /* for the dests in the service */
    struct list_head    d_list; /* for table with all the dests t是全局hash链表ip_vs_rtable的一个节点*/

    u16            af;        /* address family 地址的协议族 AF_INET/AF_INET6 */
    union nf_inet_addr    addr;        /* IP address of the server Real Server 的地址 */
    __be16            port;        /* port number of the server Real Server 端口号 */
/*ip_vs_dest对象的状态标志位，IP_VS_DEST_F_AVAILABLE表示此真实服务器可用，IP_VS_DEST_F_OVERLOAD表示此真实服务器超负荷。*/
    volatile unsigned    flags;        /* dest status flags */
/*ip_vs_dest对象的连接标志位。这些标志位本身不是用来标示 ip_vs_dest对象的，而是由ip_vs_dest对象创建ip_vs_conn对象时，赋给后者的。IP_VS_CONN_F_MASQ、 IP_VS_CONN_F_TUNNEL和IP_VS_CONN_F_DROUTE，分别代表NAT、TUN和DR三种模式*/
    atomic_t        conn_flags;    /* flags to copy to conn */
    atomic_t        weight;        /* server weight  权重用于调度*/
/*ip_vs_dest对象的引用计数，初值为0，当对象被加入链表或从链表删除，或者被ip_vs_conn对象引用时，refcnt相应地增或减1*/
    atomic_t        refcnt;        /* reference counter */
    struct ip_vs_stats      stats; /* statistics */

    /* connection counters and thresholds 连接的统计和阈值 */
    atomic_t        activeconns;    /* active connections */
    atomic_t        inactconns;    /* inactive connections */
    atomic_t        persistconns;    /* persistent connections */
    __u32            u_threshold;    /* upper threshold */
    __u32            l_threshold;    /* lower threshold */

    /* for destination cache */
    spinlock_t        dst_lock;    /* lock of dst_cache */
    struct dst_entry    *dst_cache;    /* destination cache entry */
    u32            dst_rtos;    /* RT_TOS(tos) for dst */

    /* for virtual service  LVS的相关信息*/
    struct ip_vs_service    *svc;        /* service it belongs to */
    __u16            protocol;    /* which protocol (TCP/UDP) */
    union nf_inet_addr    vaddr;        /* virtual IP address */
    __be16            vport;        /* virtual port number */
    __u32            vfwmark;    /* firewall mark of service 防火墙标志*/
};

3. 3  ip_vs_service_user_kern

下面这个结构体主要是用户空间向内核空间发送IPVS Server的相关配置。

struct ip_vs_service_user_kern {
    /* virtual service addresses */
    u16            af;
    u16            protocol;
    union nf_inet_addr    addr;        /* virtual ip address */
    u16            port;
    u32            fwmark;        /* firwall mark of service */

    /* virtual service options */
    char            *sched_name;
    unsigned        flags;        /* virtual service flags */
    unsigned        timeout;    /* persistent timeout in sec */
    u32            netmask;    /* persistent netmask */
};

3.4  ip_vs_dest_user_kern

下面这个结构主要是用户空间向内核空间发送Real Server的相关数据使用。

struct ip_vs_dest_user_kern {
    /* destination server address */
    union nf_inet_addr    addr;
    u16            port;

    /* real server options */
    unsigned        conn_flags;    /* connection flags */
    int            weight;        /* destination weight */

    /* thresholds for active connections */
    u32            u_threshold;    /* upper threshold */
    u32            l_threshold;    /* lower threshold */
};

3.5  ip_vs_service

/*
 *    The information about the virtual service offered to the net
 *    and the forwarding entries
 */
struct ip_vs_service {
/*s_list是全局hash链表ip_vs_svc_table的一个节点 static struct list_head ip_vs_svc_table[IP_VS_SVC_TAB_SIZE]; 它是一个数组，每个成员是一个链表头。将ip_vs_service对象的协议类型、地址和端口进行hash，hash值作为数组下标，然后将此对象置入数组成员对应的链表中。*/
    struct list_head    s_list; /* for normal service table */
    struct list_head    f_list; /* for fwmark-based service table */
/*refcnt和usecnt分别是ip_vs_service对象的引用计数和使用计数。它们是atomic_t类型的变量。refcnt在对象新建时为0，当ip_vs_service对象被加入链表或从链表删除，或者被 ip_vs_dest对象引用时，refcnt相应地增或减1。usecnt初始化为1*/
    atomic_t        refcnt; /* reference counter */
    atomic_t        usecnt; /* use counter */

    u16            af; /* address family  地址族*/
    __u16            protocol; /* which protocol (TCP/UDP) */
    union nf_inet_addr    addr;     /* IP address for virtual service 虚拟服务器的地址 */
    __be16            port;     /* port number for the service  虚拟服务器的端口号*/
    __u32                   fwmark; /* firewall mark of the service */
    /*ip_vs_service对象的状态标志位，可以取IP_VS_SVC_F_PERSISTENT和 IP_VS_SVC_F_HASHED。前者表示IPVS服务使用了基于IP地址的会话保持，即同一IP地址发起的连接将被负载到同一台真实服务器上。后 者表示ip_vs_service对象已被加入到ip_vs_svc_table链表中*/
unsigned        flags;     /* service status flags */
/*timeout和netmask只有在IP_VS_SVC_F_PERSISTENT标志位被设置时才有效, timeout是会话的超时时间，超过 此时间后，会话将不再有效。同一IP地址发起的两个连接，如果间隔超过此时间，则未必会被负载到同一台真实服务器上*/
    unsigned        timeout; /* persistent timeout in ticks */
/*netmask可以将会话保持设置成 基于IP网段的，即同一网段发起的连接将被负载到同一台真实服务器上。*/
    __be32            netmask; /* grouping granularity */
/*destinations是ip_vs_dest对象链表，它代指了IPVS服务对应的真实服务器列表。num_dests是服务器个数*/
    struct list_head    destinations; /* real server d-linked list */
    __u32            num_dests; /* number of servers */
    struct ip_vs_stats      stats; /* statistics for the service */
    struct ip_vs_app    *inc;     /* bind conns to this app inc */

    /* for scheduling 指向了一个ip_vs_scheduler对象，它代指一种调度算法*/
    struct ip_vs_scheduler    *scheduler; /* bound scheduler object */
    rwlock_t        sched_lock; /* lock sched_data */
    void            *sched_data; /* scheduler application data */
};

3.6  ip_vs_dest_user_kern

用户空间真实服务器信息

struct ip_vs_dest_user_kern {
    /* destination server address */
    union nf_inet_addr    addr;
    __be16            port;

    /* real server options */
    unsigned int        conn_flags;    /* connection flags */
    int            weight;        /* destination weight */

    /* thresholds for active connections */
    u32            u_threshold;    /* upper threshold */
    u32            l_threshold;    /* lower threshold */

    /* Address family of addr */
    u16            af;
};

3.7  ip_vs_service_user_kern

用户空间虚拟服务器信息

struct ip_vs_service_user_kern {
    /* virtual service addresses */
    u16            af;
    u16            protocol;
    union nf_inet_addr    addr;        /* virtual ip address */
    __be16            port;
    u32            fwmark;        /* firwall mark of service */

    /* virtual service options */
    char            *sched_name;
    char            *pe_name;
    unsigned int        flags;        /* virtual service flags */
    unsigned int        timeout;    /* persistent timeout in sec */
    __be32            netmask;    /* persistent netmask or plen */
};

3.8  ip_vs_protocol

struct ip_vs_protocol {
    struct ip_vs_protocol    *next;
 /*协议的名字*/
    char            *name;
/*协议号*/
    u16            protocol;
    u16            num_states;
    int            dont_defrag;

    void (*init)(struct ip_vs_protocol *pp);

    void (*exit)(struct ip_vs_protocol *pp);

    int (*init_netns)(struct net *net, struct ip_vs_proto_data *pd);

    void (*exit_netns)(struct net *net, struct ip_vs_proto_data *pd);

/*协议调度函数*/
int (*conn_schedule)(int af, struct sk_buff *skb,
                 struct ip_vs_proto_data *pd,
             int *verdict, struct ip_vs_conn **cpp,
                 struct ip_vs_iphdr *iph);
 /*查in方向的IPVS*/
    struct ip_vs_conn *
    (*conn_in_get)(int af,
         const struct sk_buff *skb,
         const struct ip_vs_iphdr *iph,
         int inverse);
 /*查out方向的IPVS*/
    struct ip_vs_conn *
    (*conn_out_get)(int af,
            const struct sk_buff *skb,
            const struct ip_vs_iphdr *iph,
            int inverse);
 /*SNAT处理函数*/
    int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp,
                struct ip_vs_conn *cp, struct ip_vs_iphdr *iph);
 /*DNAT处理函数*/
    int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp,
                struct ip_vs_conn *cp, struct ip_vs_iphdr *iph);
 /*校验和处理函数*/
    int (*csum_check)(int af, struct sk_buff *skb,
              struct ip_vs_protocol *pp);

    const char *(*state_name)(int state);
 /*状态转换函数*/
    void (*state_transition)(struct ip_vs_conn *cp, int direction,
                 const struct sk_buff *skb,
                 struct ip_vs_proto_data *pd);

    int (*register_app)(struct net *net, struct ip_vs_app *inc);

    void (*unregister_app)(struct net *net, struct ip_vs_app *inc);
 /*多连接的应用绑定函数*/
    int (*app_conn_bind)(struct ip_vs_conn *cp);

    void (*debug_packet)(int af, struct ip_vs_protocol *pp,
             const struct sk_buff *skb,
             int offset,
             const char *msg);

    void (*timeout_change)(struct ip_vs_proto_data *pd, int flags);
};