How conntrack works in Netfilter

Part 1 - Initialization

1. Initialization
   1. nf_conntrack_standalone
      1. Big picture(Assume that no more networking namespace here, only a global one. This means CONFIG_NET_NS is not defined.)

2. Some details
   1. Initialize some global hashtable variables(net/netfilter/nf_conntrack_core.c)

static int nf_conntrack_init_init_net(void)
{
    int max_factor = 8;
    int ret;

    /* Idea from tcp.c: use 1/16384 of memory.  On i386: 32MB
     * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
    if (!nf_conntrack_htable_size) {
        nf_conntrack_htable_size
            = (((num_physpages << PAGE_SHIFT) / 16384)
               / sizeof(struct hlist_head));  // Hashtable size is related to memory size and maximum size is 16384
        if (num_physpages > (1024 * 1024 * 1024 / PAGE_SIZE))
            nf_conntrack_htable_size = 16384;
        if (nf_conntrack_htable_size < 32)
            nf_conntrack_htable_size = 32;

        /* Use a max. factor of four by default to get the same max as
         * with the old struct list_heads. When a table size is given
         * we use the old value of 8 to avoid reducing the max.
         * entries. */
        max_factor = 4;
    }
    nf_conntrack_max = max_factor * nf_conntrack_htable_size; // maximum tracking connections

    printk("nf_conntrack version %s (%u buckets, %d max)\n",
           NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
           nf_conntrack_max);

    nf_conntrack_cachep = kmem_cache_create("nf_conntrack",  //
                        sizeof(struct nf_conn),
                        0, SLAB_DESTROY_BY_RCU, NULL);
    if (!nf_conntrack_cachep) {
        printk(KERN_ERR "Unable to create nf_conn slab cache\n");
        ret = -ENOMEM;
        goto err_cache;
    }

    ret = nf_conntrack_proto_init();
    if (ret < 0)
        goto err_proto;

    ret = nf_conntrack_helper_init();
    if (ret < 0)
        goto err_helper;

    return 0;

err_helper:
    nf_conntrack_proto_fini();
err_proto:
    kmem_cache_destroy(nf_conntrack_cachep);
err_cache:
    return ret;
}

2. nf_conntrack_proto_init(net/netfilter/nf_conntrack_proto.c)

int nf_conntrack_proto_init(void)
{
    unsigned int i;
    int err;

    err = nf_ct_l4proto_register_sysctl(&nf_conntrack_l4proto_generic);
    if (err < 0)
        return err;

    for (i = 0; i < AF_MAX; i++)
        rcu_assign_pointer(nf_ct_l3protos[i],   // Global pointer array which point to struct nf_conntrack_l3proto
                   &nf_conntrack_l3proto_generic); // Initialize all l3 protocols of conntrack
    return 0;
}

3. nf_conntrack_helper_init(net/netfilter/nf_conntrack_helper.c)

int nf_conntrack_helper_init(void)
{
    int err;

    nf_ct_helper_hsize = 1; /* gets rounded up to use one page */
    nf_ct_helper_hash = nf_ct_alloc_hashtable(&nf_ct_helper_hsize,
                          &nf_ct_helper_vmalloc, 0);
    if (!nf_ct_helper_hash)
        return -ENOMEM;

    err = nf_ct_extend_register(&helper_extend);
    if (err < 0)
        goto err1;

    return 0;

err1:
    nf_ct_free_hashtable(nf_ct_helper_hash, nf_ct_helper_vmalloc,
                 nf_ct_helper_hsize);
    return err;
}

4. nf_ct_extend_register(net/netfilter/nf_conntrack_extend.c)

int nf_ct_extend_register(struct nf_ct_ext_type *type)
{
    int ret = 0;

    mutex_lock(&nf_ct_ext_type_mutex);
    if (nf_ct_ext_types[type->id]) {
        ret = -EBUSY;
        goto out;
    }

    /* This ensures that nf_ct_ext_create() can allocate enough area
       before updating alloc_size */
    type->alloc_size = ALIGN(sizeof(struct nf_ct_ext), type->align)
               + type->len;
    rcu_assign_pointer(nf_ct_ext_types[type->id], type); // Global list for struct nf_ct_ext_types
    update_alloc_size(type);
out:
    mutex_unlock(&nf_ct_ext_type_mutex);
    return ret;
}
EXPORT_SYMBOL_GPL(nf_ct_extend_register);

5. nf_conntrack_expect_init(net/netfilter/nf_conntrack_expect.c)

int nf_conntrack_expect_init(struct net *net)
{
    int err = -ENOMEM;

    if (net_eq(net, &init_net)) {
        if (!nf_ct_expect_hsize) {
            nf_ct_expect_hsize = nf_conntrack_htable_size / 256;
            if (!nf_ct_expect_hsize)
                nf_ct_expect_hsize = 1;
        }
        nf_ct_expect_max = nf_ct_expect_hsize * 4;
    }

    net->ct.expect_count = 0;
    net->ct.expect_hash = nf_ct_alloc_hashtable(&nf_ct_expect_hsize,
                          &net->ct.expect_vmalloc, 0); // Global hash list for expectations
    if (net->ct.expect_hash == NULL)
        goto err1;

    if (net_eq(net, &init_net)) {
        nf_ct_expect_cachep = kmem_cache_create("nf_conntrack_expect",
                    sizeof(struct nf_conntrack_expect),
                    0, 0, NULL);
        if (!nf_ct_expect_cachep)
            goto err2;
    }

    err = exp_proc_init(net);
    if (err < 0)
        goto err3;

    return 0;

err3:
    if (net_eq(net, &init_net))
        kmem_cache_destroy(nf_ct_expect_cachep);
err2:
    nf_ct_free_hashtable(net->ct.expect_hash, net->ct.expect_vmalloc,
                 nf_ct_expect_hsize);
err1:
    return err;
}

2. How ipv4 initialize connection tracking module?
   1. Big picture

2. More details
   
   1. nf_conntrack_l4proto_register(net/netfilter/nf_conntrack_proto.c)
      

int nf_conntrack_l4proto_register(struct nf_conntrack_l4proto *l4proto)
{
    int ret = 0;

    if (l4proto->l3proto >= PF_MAX)
        return -EBUSY;

    if ((l4proto->to_nlattr && !l4proto->nlattr_size)
        || (l4proto->tuple_to_nlattr && !l4proto->nlattr_tuple_size))
        return -EINVAL;

    mutex_lock(&nf_ct_proto_mutex);
    if (!nf_ct_protos[l4proto->l3proto]) {
        /* l3proto may be loaded latter. */
        struct nf_conntrack_l4proto **proto_array;
        int i;

        proto_array = kmalloc(MAX_NF_CT_PROTO *
                      sizeof(struct nf_conntrack_l4proto *),
                      GFP_KERNEL);
        if (proto_array == NULL) {
            ret = -ENOMEM;
            goto out_unlock;
        }

        for (i = 0; i < MAX_NF_CT_PROTO; i++)
            proto_array[i] = &nf_conntrack_l4proto_generic;
        nf_ct_protos[l4proto->l3proto] = proto_array;
    } else if (nf_ct_protos[l4proto->l3proto][l4proto->l4proto] !=
                    &nf_conntrack_l4proto_generic) {
        ret = -EBUSY;
        goto out_unlock;
    }

    ret = nf_ct_l4proto_register_sysctl(l4proto);
    if (ret < 0)
        goto out_unlock;

    l4proto->nla_size = 0;
    if (l4proto->nlattr_size)
        l4proto->nla_size += l4proto->nlattr_size();
    if (l4proto->nlattr_tuple_size)
        l4proto->nla_size += 3 * l4proto->nlattr_tuple_size();

    rcu_assign_pointer(nf_ct_protos[l4proto->l3proto][l4proto->l4proto],
               l4proto);

out_unlock:
    mutex_unlock(&nf_ct_proto_mutex);
    return ret;
}
EXPORT_SYMBOL_GPL(nf_conntrack_l4proto_register);

2. nf_conntrack_l3proto_register(net/ipv4/netfilter/nf_conntrack_l3proto_ipv4.c)

int nf_conntrack_l3proto_register(struct nf_conntrack_l3proto *proto)
{
    int ret = 0;

    if (proto->l3proto >= AF_MAX)
        return -EBUSY;

    if (proto->tuple_to_nlattr && !proto->nlattr_tuple_size)
        return -EINVAL;

    mutex_lock(&nf_ct_proto_mutex);
    if (nf_ct_l3protos[proto->l3proto] != &nf_conntrack_l3proto_generic) {
        ret = -EBUSY;
        goto out_unlock;
    }

    ret = nf_ct_l3proto_register_sysctl(proto);
    if (ret < 0)
        goto out_unlock;

    if (proto->nlattr_tuple_size)
        proto->nla_size = 3 * proto->nlattr_tuple_size();

    rcu_assign_pointer(nf_ct_l3protos[proto->l3proto], proto);

out_unlock:
    mutex_unlock(&nf_ct_proto_mutex);
    return ret;
}
EXPORT_SYMBOL_GPL(nf_conntrack_l3proto_register);

3. What we got after ipv4 connection tracking initialization

 

Part 2 - How to track FTP connections

1. Initialization
   1. nf_conntrack_ftp_init(net/netfilter/nf_conntrack_ftp.c)
      

static int __init nf_conntrack_ftp_init(void)
{
    int i, j = -1, ret = 0;
    char *tmpname;

    ftp_buffer = kmalloc(65536, GFP_KERNEL);
    if (!ftp_buffer)
        return -ENOMEM;

    if (ports_c == 0) // static unsigned int ports_c;
        ports[ports_c++] = FTP_PORT; //static u_int16_t ports[MAX_PORTS];

    /* FIXME should be configurable whether IPv4 and IPv6 FTP connections
         are tracked or not - YK */
    for (i = 0; i < ports_c; i++) {
        ftp[i][0].tuple.src.l3num = PF_INET;// Will be used when hash tuple
        ftp[i][1].tuple.src.l3num = PF_INET6;
        for (j = 0; j < 2; j++) {
            ftp[i][j].tuple.src.u.tcp.port = htons(ports[i]); // Will be used when hash tuple
            ftp[i][j].tuple.dst.protonum = IPPROTO_TCP; // Will be used when hash tuple
            ftp[i][j].expect_policy = &ftp_exp_policy;
            ftp[i][j].me = THIS_MODULE;
            ftp[i][j].help = help;
            tmpname = &ftp_names[i][j][0];
            if (ports[i] == FTP_PORT)
                sprintf(tmpname, "ftp");
            else
                sprintf(tmpname, "ftp-%d", ports[i]);
            ftp[i][j].name = tmpname;

            pr_debug("nf_ct_ftp: registering helper for pf: %d "
                 "port: %d\n",
                 ftp[i][j].tuple.src.l3num, ports[i]);
            ret = nf_conntrack_helper_register(&ftp[i][j]);
            if (ret) {
                printk("nf_ct_ftp: failed to register helper "
                       " for pf: %d port: %d\n",
                    ftp[i][j].tuple.src.l3num, ports[i]);
                nf_conntrack_ftp_fini();
                return ret;
            }
        }
    }

    return 0;
}

2. nf_conntrack_helper_register(net/netfilter/nf_conntrack_helper.c)
   

int nf_conntrack_helper_register(struct nf_conntrack_helper *me)
{
    unsigned int h = helper_hash(&me->tuple);

    BUG_ON(me->expect_policy == NULL);
    BUG_ON(me->expect_class_max >= NF_CT_MAX_EXPECT_CLASSES);
    BUG_ON(strlen(me->name) > NF_CT_HELPER_NAME_LEN - 1);

    mutex_lock(&nf_ct_helper_mutex);
    hlist_add_head_rcu(&me->hnode, &nf_ct_helper_hash[h]); // Global helper list
    nf_ct_helper_count++;
    mutex_unlock(&nf_ct_helper_mutex);

    return 0;
}

3. helper_hash(net/netfilter/nf_conntrack_helper.c)

/* Stupid hash, but collision free for the default registrations of the
 * helpers currently in the kernel. */
static unsigned int helper_hash(const struct nf_conntrack_tuple *tuple)
{
    return (((tuple->src.l3num << 8) | tuple->dst.protonum) ^
        (__force __u16)tuple->src.u.all) % nf_ct_helper_hsize;
}

4. How ftp helper registered

2. How ftp helper works
   1. Big picture

2. Main helper function(net/netfilter/nf_conntrack_ftp.c)

static int help(struct sk_buff *skb,
        unsigned int protoff,
        struct nf_conn *ct,
        enum ip_conntrack_info ctinfo)
{
    unsigned int dataoff, datalen;
    const struct tcphdr *th;
    struct tcphdr _tcph;
    const char *fb_ptr;
    int ret;
    u32 seq;
    int dir = CTINFO2DIR(ctinfo);
    unsigned int uninitialized_var(matchlen), uninitialized_var(matchoff);
    struct nf_ct_ftp_master *ct_ftp_info = &nfct_help(ct)->help.ct_ftp_info;
    struct nf_conntrack_expect *exp;
    union nf_inet_addr *daddr;
    struct nf_conntrack_man cmd = {};
    unsigned int i;
    int found = 0, ends_in_nl;
    typeof(nf_nat_ftp_hook) nf_nat_ftp;

    /* Until there's been traffic both ways, don't look in packets. */
    if (ctinfo != IP_CT_ESTABLISHED
        && ctinfo != IP_CT_ESTABLISHED+IP_CT_IS_REPLY) {
        pr_debug("ftp: Conntrackinfo = %u\n", ctinfo);
        return NF_ACCEPT;
    }

    th = skb_header_pointer(skb, protoff, sizeof(_tcph), &_tcph);
    if (th == NULL)
        return NF_ACCEPT;

    dataoff = protoff + th->doff * 4;
    /* No data? */
    if (dataoff >= skb->len) {
        pr_debug("ftp: dataoff(%u) >= skblen(%u)\n", dataoff,
             skb->len);
        return NF_ACCEPT;
    }
    datalen = skb->len - dataoff;

    spin_lock_bh(&nf_ftp_lock);
    fb_ptr = skb_header_pointer(skb, dataoff, datalen, ftp_buffer);
    BUG_ON(fb_ptr == NULL);

    ends_in_nl = (fb_ptr[datalen - 1] == '\n');
    seq = ntohl(th->seq) + datalen;

    /* Look up to see if we're just after a \n. */
    if (!find_nl_seq(ntohl(th->seq), ct_ftp_info, dir)) {
        /* Now if this ends in \n, update ftp info. */
        pr_debug("nf_conntrack_ftp: wrong seq pos %s(%u) or %s(%u)\n",
             ct_ftp_info->seq_aft_nl_num[dir] > 0 ? "" : "(UNSET)",
             ct_ftp_info->seq_aft_nl[dir][0],
             ct_ftp_info->seq_aft_nl_num[dir] > 1 ? "" : "(UNSET)",
             ct_ftp_info->seq_aft_nl[dir][1]);
        ret = NF_ACCEPT;
        goto out_update_nl;
    }

    /* Initialize IP/IPv6 addr to expected address (it's not mentioned
       in EPSV responses) */
    cmd.l3num = nf_ct_l3num(ct);
    memcpy(cmd.u3.all, &ct->tuplehash[dir].tuple.src.u3.all,
           sizeof(cmd.u3.all));

    for (i = 0; i < ARRAY_SIZE(search[dir]); i++) {
        found = find_pattern(fb_ptr, datalen,    // Try to find packet which trigger a ftp connection tracking
                     search[dir][i].pattern,
                     search[dir][i].plen,
                     search[dir][i].skip,
                     search[dir][i].term,
                     &matchoff, &matchlen,
                     &cmd,
                     search[dir][i].getnum);
        if (found) break;
    }
    if (found == -1) {
        /* We don't usually drop packets.  After all, this is
           connection tracking, not packet filtering.
           However, it is necessary for accurate tracking in
           this case. */
        pr_debug("conntrack_ftp: partial %s %u+%u\n",
             search[dir][i].pattern,  ntohl(th->seq), datalen);
        ret = NF_DROP;
        goto out;
    } else if (found == 0) { /* No match */
        ret = NF_ACCEPT;
        goto out_update_nl;
    }

    pr_debug("conntrack_ftp: match `%.*s' (%u bytes at %u)\n",
         matchlen, fb_ptr + matchoff,
         matchlen, ntohl(th->seq) + matchoff);

    exp = nf_ct_expect_alloc(ct);
    if (exp == NULL) {
        ret = NF_DROP;
        goto out;
    }

    /* We refer to the reverse direction ("!dir") tuples here,
     * because we're expecting something in the other direction.
     * Doesn't matter unless NAT is happening.  */
    daddr = &ct->tuplehash[!dir].tuple.dst.u3;

    /* Update the ftp info */
    if ((cmd.l3num == nf_ct_l3num(ct)) &&
        memcmp(&cmd.u3.all, &ct->tuplehash[dir].tuple.src.u3.all,
             sizeof(cmd.u3.all))) {
        /* Enrico Scholz's passive FTP to partially RNAT'd ftp
           server: it really wants us to connect to a
           different IP address.  Simply don't record it for
           NAT. */
        if (cmd.l3num == PF_INET) {
            pr_debug("conntrack_ftp: NOT RECORDING: %pI4 != %pI4\n",
                 &cmd.u3.ip,
                 &ct->tuplehash[dir].tuple.src.u3.ip);
        } else {
            pr_debug("conntrack_ftp: NOT RECORDING: %pI6 != %pI6\n",
                 cmd.u3.ip6,
                 ct->tuplehash[dir].tuple.src.u3.ip6);
        }

        /* Thanks to Cristiano Lincoln Mattos
           for reporting this potential
           problem (DMZ machines opening holes to internal
           networks, or the packet filter itself). */
        if (!loose) {
            ret = NF_ACCEPT;
            goto out_put_expect;
        }
        daddr = &cmd.u3;
    }

    nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, cmd.l3num, // Setup expectation for this ftp connection
              &ct->tuplehash[!dir].tuple.src.u3, daddr,
              IPPROTO_TCP, NULL, &cmd.u.tcp.port);

    /* Now, NAT might want to mangle the packet, and register the
     * (possibly changed) expectation itself. */
    nf_nat_ftp = rcu_dereference(nf_nat_ftp_hook);
    if (nf_nat_ftp && ct->status & IPS_NAT_MASK)
        ret = nf_nat_ftp(skb, ctinfo, search[dir][i].ftptype,
                 matchoff, matchlen, exp);
    else {
        /* Can't expect this?  Best to drop packet now. */
        if (nf_ct_expect_related(exp) != 0)
            ret = NF_DROP;
        else
            ret = NF_ACCEPT;
    }

out_put_expect:
    nf_ct_expect_put(exp);

out_update_nl:
    /* Now if this ends in \n, update ftp info.  Seq may have been
     * adjusted by NAT code. */
    if (ends_in_nl)
        update_nl_seq(ct, seq, ct_ftp_info, dir, skb);
 out:
    spin_unlock_bh(&nf_ftp_lock);
    return ret;
}

3. Pattern search(net/netfilter/nf_conntrack_ftp.c)
   1. Structure
      

static struct ftp_search {
    const char *pattern;
    size_t plen;
    char skip;
    char term;
    enum nf_ct_ftp_type ftptype;
    int (*getnum)(const char *, size_t, struct nf_conntrack_man *, char);
} search[IP_CT_DIR_MAX][2] = { // 2 directions and 2 modes
    [IP_CT_DIR_ORIGINAL] = {
        {
            .pattern    = "PORT",
            .plen        = sizeof("PORT") - 1,
            .skip        = ' ',
            .term        = '\r',
            .ftptype    = NF_CT_FTP_PORT,
            .getnum        = try_rfc959,
        },
        {
            .pattern    = "EPRT",
            .plen        = sizeof("EPRT") - 1,
            .skip        = ' ',
            .term        = '\r',
            .ftptype    = NF_CT_FTP_EPRT,
            .getnum        = try_eprt,
        },
    },
    [IP_CT_DIR_REPLY] = {
        {
            .pattern    = "227 ",
            .plen        = sizeof("227 ") - 1,
            .skip        = '(',
            .term        = ')',
            .ftptype    = NF_CT_FTP_PASV,
            .getnum        = try_rfc959,
        },
        {
            .pattern    = "229 ",
            .plen        = sizeof("229 ") - 1,
            .skip        = '(',
            .term        = ')',
            .ftptype    = NF_CT_FTP_EPSV,
            .getnum        = try_epsv_response,
        },
    },
};

2. try_rfc959(net/netfilter/nf_conntrack_ftp.c)

/* Returns 0, or length of numbers: 192,168,1,1,5,6 */
static int try_rfc959(const char *data, size_t dlen,
              struct nf_conntrack_man *cmd, char term)
{
    int length;
    u_int32_t array[6];

    length = try_number(data, dlen, array, 6, ',', term);
    if (length == 0)
        return 0;

    cmd->u3.ip =  htonl((array[0] << 24) | (array[1] << 16) |
                    (array[2] << 8) | array[3]);
    cmd->u.tcp.port = htons((array[4] << 8) | array[5]);
    return length;
}

3. try_eprt(net/netfilter/nf_conntrack_ftp.c)

/* Returns 0, or length of numbers: |1|132.235.1.2|6275| or |2|3ffe::1|6275| */
static int try_eprt(const char *data, size_t dlen, struct nf_conntrack_man *cmd,
            char term)
{
    char delim;
    int length;

    /* First character is delimiter, then "1" for IPv4 or "2" for IPv6,
       then delimiter again. */
    if (dlen <= 3) {
        pr_debug("EPRT: too short\n");
        return 0;
    }
    delim = data[0];
    if (isdigit(delim) || delim < 33 || delim > 126 || data[2] != delim) {
        pr_debug("try_eprt: invalid delimitter.\n");
        return 0;
    }

    if ((cmd->l3num == PF_INET && data[1] != '1') ||
        (cmd->l3num == PF_INET6 && data[1] != '2')) {
        pr_debug("EPRT: invalid protocol number.\n");
        return 0;
    }

    pr_debug("EPRT: Got %c%c%c\n", delim, data[1], delim);

    if (data[1] == '1') {
        u_int32_t array[4];

        /* Now we have IP address. */
        length = try_number(data + 3, dlen - 3, array, 4, '.', delim);
        if (length != 0)
            cmd->u3.ip = htonl((array[0] << 24) | (array[1] << 16)
                       | (array[2] << 8) | array[3]);
    } else {
        /* Now we have IPv6 address. */
        length = get_ipv6_addr(data + 3, dlen - 3,
                       (struct in6_addr *)cmd->u3.ip6, delim);
    }

    if (length == 0)
        return 0;
    pr_debug("EPRT: Got IP address!\n");
    /* Start offset includes initial "|1|", and trailing delimiter */
    return get_port(data, 3 + length + 1, dlen, delim, &cmd->u.tcp.port);
}

4. try_epsv_response(net/netfilter/nf_conntrack_ftp.c)
   

/* Returns 0, or length of numbers: |||6446| */
static int try_epsv_response(const char *data, size_t dlen,
                 struct nf_conntrack_man *cmd, char term)
{
    char delim;

    /* Three delimiters. */
    if (dlen <= 3) return 0;
    delim = data[0];
    if (isdigit(delim) || delim < 33 || delim > 126
        || data[1] != delim || data[2] != delim)
        return 0;

    return get_port(data, 3, dlen, delim, &cmd->u.tcp.port);
}

5. nf_ct_expect_init(net/netfilter/nf_conntrack_expect.c)
   

void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class,
               u_int8_t family,
               const union nf_inet_addr *saddr,
               const union nf_inet_addr *daddr,
               u_int8_t proto, const __be16 *src, const __be16 *dst)
{
    int len;

    if (family == AF_INET)
        len = 4;
    else
        len = 16;

    exp->flags = 0;
    exp->class = class;
    exp->expectfn = NULL;
    exp->helper = NULL;
    exp->tuple.src.l3num = family;
    exp->tuple.dst.protonum = proto;

    if (saddr) {
        memcpy(&exp->tuple.src.u3, saddr, len);
        if (sizeof(exp->tuple.src.u3) > len)
            /* address needs to be cleared for nf_ct_tuple_equal */
            memset((void *)&exp->tuple.src.u3 + len, 0x00,
                   sizeof(exp->tuple.src.u3) - len);
        memset(&exp->mask.src.u3, 0xFF, len);
        if (sizeof(exp->mask.src.u3) > len)
            memset((void *)&exp->mask.src.u3 + len, 0x00,
                   sizeof(exp->mask.src.u3) - len);
    } else {
        memset(&exp->tuple.src.u3, 0x00, sizeof(exp->tuple.src.u3));
        memset(&exp->mask.src.u3, 0x00, sizeof(exp->mask.src.u3));
    }

    if (src) {
        exp->tuple.src.u.all = *src;
        exp->mask.src.u.all = htons(0xFFFF);
    } else {
        exp->tuple.src.u.all = 0;
        exp->mask.src.u.all = 0;
    }

    memcpy(&exp->tuple.dst.u3, daddr, len);
    if (sizeof(exp->tuple.dst.u3) > len)
        /* address needs to be cleared for nf_ct_tuple_equal */
        memset((void *)&exp->tuple.dst.u3 + len, 0x00,
               sizeof(exp->tuple.dst.u3) - len);

    exp->tuple.dst.u.all = *dst;
}
EXPORT_SYMBOL_GPL(nf_ct_expect_init);

2. Let us take a real ftp traffic as an example
   1. The partial real ftp traffic

No.     Time        Source                Destination           Protocol Info
      1 0.000000    172.20.9.84           172.20.9.80           TCP      eportcomm > ftp [SYN] Seq=0 Win=65535 Len=0 MSS=1460

Frame 1 (62 bytes on wire, 62 bytes captured)
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: eportcomm (4666), Dst Port: ftp (21), Seq: 0, Len: 0

No.     Time        Source                Destination           Protocol Info
      2 0.000135    172.20.9.80           172.20.9.84           TCP      ftp > eportcomm [SYN, ACK] Seq=0 Ack=1 Win=5840 Len=0 MSS=1460

Frame 2 (62 bytes on wire, 62 bytes captured)
Internet Protocol, Src: 172.20.9.80 (172.20.9.80), Dst: 172.20.9.84 (172.20.9.84)
Transmission Control Protocol, Src Port: ftp (21), Dst Port: eportcomm (4666), Seq: 0, Ack: 1, Len: 0

No.     Time        Source                Destination           Protocol Info
      3 0.000152    172.20.9.84           172.20.9.80           TCP      eportcomm > ftp [ACK] Seq=1 Ack=1 Win=65535 Len=0

Frame 3 (54 bytes on wire, 54 bytes captured)
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: eportcomm (4666), Dst Port: ftp (21), Seq: 1, Ack: 1, Len: 0

No.     Time        Source                Destination           Protocol Info
      4 0.006313    172.20.9.80           172.20.9.84           FTP      Response: 220 Welcome to Test's FTP service.

Frame 4 (91 bytes on wire, 91 bytes captured)
Internet Protocol, Src: 172.20.9.80 (172.20.9.80), Dst: 172.20.9.84 (172.20.9.84)
Transmission Control Protocol, Src Port: ftp (21), Dst Port: eportcomm (4666), Seq: 1, Ack: 1, Len: 37
File Transfer Protocol (FTP)

...........

No.     Time        Source                Destination           Protocol Info
     12 8.471443    172.20.9.84           172.20.9.80           TCP      eportcomm > ftp [ACK] Seq=32 Ack=95 Win=65441 Len=0

Frame 12 (54 bytes on wire, 54 bytes captured)
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: eportcomm (4666), Dst Port: ftp (21), Seq: 32, Ack: 95, Len: 0

No.     Time        Source                Destination           Protocol Info
     13 10.124938   172.20.9.84           172.20.9.80           FTP      Request: PORT 172,20,9,84,18,67

Frame 13 (78 bytes on wire, 78 bytes captured)
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: eportcomm (4666), Dst Port: ftp (21), Seq: 32, Ack: 95, Len: 24
File Transfer Protocol (FTP)

No.     Time        Source                Destination           Protocol Info
     14 10.125258   172.20.9.80           172.20.9.84           FTP      Response: 200 PORT command successful. Consider using PASV.

Frame 14 (105 bytes on wire, 105 bytes captured)
Internet Protocol, Src: 172.20.9.80 (172.20.9.80), Dst: 172.20.9.84 (172.20.9.84)
Transmission Control Protocol, Src Port: ftp (21), Dst Port: eportcomm (4666), Seq: 95, Ack: 56, Len: 51
File Transfer Protocol (FTP)

No.     Time        Source                Destination           Protocol Info
     15 10.126652   172.20.9.84           172.20.9.80           FTP      Request: NLST

Frame 15 (60 bytes on wire, 60 bytes captured)
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: eportcomm (4666), Dst Port: ftp (21), Seq: 56, Ack: 146, Len: 6
File Transfer Protocol (FTP)

No.     Time        Source                Destination           Protocol Info
     16 10.127158   172.20.9.80           172.20.9.84           TCP      ftp-data > dhct-status [SYN] Seq=0 Win=5840 Len=0 MSS=1460 TSV=21742713 TSER=0 WS=5

Frame 16 (74 bytes on wire, 74 bytes captured)
Internet Protocol, Src: 172.20.9.80 (172.20.9.80), Dst: 172.20.9.84 (172.20.9.84)
Transmission Control Protocol, Src Port: ftp-data (20), Dst Port: dhct-status (4675), Seq: 0, Len: 0

No.     Time        Source                Destination           Protocol Info
     17 10.127192   172.20.9.84           172.20.9.80           TCP      dhct-status > ftp-data [SYN, ACK] Seq=0 Ack=1 Win=65535 Len=0 MSS=1460 WS=0 TSV=0 TSER=0

Frame 17 (78 bytes on wire, 78 bytes captured)
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: dhct-status (4675), Dst Port: ftp-data (20), Seq: 0, Ack: 1, Len: 0

No.     Time        Source                Destination           Protocol Info
     18 10.127311   172.20.9.80           172.20.9.84           TCP      ftp-data > dhct-status [ACK] Seq=1 Ack=1 Win=5856 Len=0 TSV=21742713 TSER=0

Frame 18 (66 bytes on wire, 66 bytes captured)
Internet Protocol, Src: 172.20.9.80 (172.20.9.80), Dst: 172.20.9.84 (172.20.9.84)
Transmission Control Protocol, Src Port: ftp-data (20), Dst Port: dhct-status (4675), Seq: 1, Ack: 1, Len: 0

No.     Time        Source                Destination           Protocol Info
     19 10.127412   172.20.9.80           172.20.9.84           FTP      Response: 150 Here comes the directory listing.

2. Assume we are on 172.20.9.80(ftp server side), so hooks on point NF_INET_PRE_ROUTING and NF_INET_LOCAL_IN will be invoked.
   1. Big Picture
      

2. What we got from Frame 1

No.     Time        Source                Destination           Protocol Info
      1 0.000000    172.20.9.84           172.20.9.80           TCP      eportcomm > ftp [SYN] Seq=0 Win=65535 Len=0 MSS=1460
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: eportcomm (4666), Dst Port: ftp (21), Seq: 0, Len: 0

3. What we got from Frame 2
   

No.     Time        Source                Destination           Protocol Info
     2 0.000135    172.20.9.80           172.20.9.84           TCP      ftp> eportcomm [SYN, ACK] Seq=0 Ack=1 Win=5840 Len=0 MSS=1460

Frame 2 (62 bytes on wire, 62 bytes captured)
Internet Protocol, Src: 172.20.9.80 (172.20.9.80), Dst: 172.20.9.84 (172.20.9.84)
Transmission Control Protocol, Src Port: ftp (21), Dst Port: eportcomm (4666), Seq: 0, Ack: 1, Len: 0

4. What we got from Frame 13

No.     Time        Source                Destination           Protocol Info
     13 10.124938   172.20.9.84           172.20.9.80           FTP      Request: PORT 172,20,9,84,18,67

Frame 13 (78 bytes on wire, 78 bytes captured)
Internet Protocol, Src: 172.20.9.84 (172.20.9.84), Dst: 172.20.9.80 (172.20.9.80)
Transmission Control Protocol, Src Port: eportcomm (4666), Dst Port: ftp (21), Seq: 32, Ack: 95, Len: 24
File Transfer Protocol (FTP)

5. What we got from Frame 16

No.     Time        Source                Destination           Protocol Info
    16 10.127158   172.20.9.80           172.20.9.84           TCP     ftp-data > dhct-status [SYN] Seq=0 Win=5840 Len=0 MSS=1460TSV=21742713 TSER=0 WS=5

Frame 16 (74 bytes on wire, 74 bytes captured)
Internet Protocol, Src: 172.20.9.80 (172.20.9.80), Dst: 172.20.9.84 (172.20.9.84)
Transmission Control Protocol, Src Port: ftp-data (20), Dst Port: dhct-status (4675), Seq: 0, Len: 0