项目地tcp send() EPIPE错误分析

最近在项目地上发现send在发送数据时，会出现EPIPE错误，将相关的程序抽取出来，通过下面的例子进行测试。

 

参考网络文章：http://hi.baidu.com/tangzhenjiang/blog/item/9700f8ed7475434879f05570.html

测试环境：AIX5.3 64位、SUSE Linux Enterprise Server9

先下载unp源码: wget

tar xzvf *.tar.gz;

configure;make lib.

1、问题分析

     根据《UNIX Network Programming Volume 1, Third Edition: The Sockets Networking》一书对应的5.13“SIGPIPE Signal”小节，SIGPIPE错误发生于：

The rule that applies is:

    When a process writes to a socket that has received an RST, the SIGPIPE signal is sent to the process.

The default action of this signal is to terminate the process, so the process must catch the signal to avoid being involuntarily

terminated.

 

2、客户端测试程序

功能：模拟往服务端发送大量数据。

#include        "unp.h"
#include

#define MAX_DATA_LEN 64*1024*1024
char long_buff[MAX_DATA_LEN];
int SendLongData(int sockfd, char * data, int size)
{                      
        int     total_len;
        int     cur_len;

        total_len = 0;
        while(size != 0)  {
        if (size > 1024) {
                    cur_len = send(sockfd, data+total_len, 1024, 0);
        } else {
                    cur_len = send(sockfd, data+total_len, size, 0);
        }
                if(cur_len < 0 ) {
                        printf("send fail: %s\n", strerror(errno));
                        return  -1;
                }
                size -= cur_len;
                total_len += cur_len;
        }
        return  total_len;
}

void   
cli_process(int sockfd, int send_len)
{      
        int             ret;
        int             len;
        len = send_len*1024;
        sprintf(long_buff, "%011d", len+11);
        memset(long_buff+11, '1', len);

        printf("total send bytes: %d\n", len+11);
        ret = SendLongData(sockfd, long_buff, len+11);
    printf("send ret: %d\n", ret);
        if (ret != (len+11)) {
                printf("SendLongData() fail, total send %d bytes.\n", ret);
        fflush(stdout);
                exit(-1);
        }
    printf("send success!\n");
}

int
main(int argc, char **argv)
{
        int                     sockfd;
        struct sockaddr_in      servaddr;
        int                     send_len;

        /*
        if (argc != 2)
              err_quit("usage: tcpcli ");
        */
       
        /* 忽略EPIPE信号，不然程序会异常退出 */
        signal(SIGPIPE, SIG_IGN);
       

        while(1) {
                /*
                printf("\ninput send bytes(KB): ");
                if(scanf("%d", &send_len) != 1) {
                        printf("input fail\n");
                        continue;
                }
                */
                send_len = 1024; /* 1MB */

                sockfd = Socket(AF_INET, SOCK_STREAM, 0);
                bzero(&servaddr, sizeof(servaddr));
                servaddr.sin_family = AF_INET;
                servaddr.sin_port = htons(SERV_PORT);
                /*servaddr.sin_port = htons(7303);*/
                /*Inet_pton(AF_INET, argv[1], &servaddr.sin_addr);*/
                Inet_pton(AF_INET, "127.0.0.1", &servaddr.sin_addr);

                Connect(sockfd, (SA *) &servaddr, sizeof(servaddr));
        /* 组包（若批量较多的情况下，耗时超过60秒），测试极端情况 */
                sleep(8);

                cli_process(sockfd, send_len);  /* do it all */
                Close(sockfd);
        }

        exit(0);
}

  3、服务端程序

功能：模拟接收客户端发送过来的大量数据。

#include        "unp.h"

#define MAX_DATA_LEN 64*1024*1024

char long_buff[MAX_DATA_LEN];
int RecvLongData(int sockfd, char * data, int size)
{
        int     total_len;
        int     cur_len; 
        total_len = 0;
        while(size != 0)  {
                cur_len = recv(sockfd, data+total_len, size, 0);
                if(cur_len == 0) {
                        printf("recv fail: %s\n", strerror(errno));
                        return  -1;
                } else if(cur_len < 0) {
                        printf("recv fail: %s\n", strerror(errno));
                        return -2;
                }
                size -= cur_len;
                total_len += cur_len;
        }

        return  total_len;
}

void   
srv_process(int sockfd)
{      
        int             ret;
        ssize_t         n;
        int             len;
        /* 检测套接字是否可读 */
        struct timeval tv;
        fd_set         rfds;
        tv.tv_usec = 0;
        tv.tv_sec = 30;
        FD_ZERO(&rfds);
        FD_SET(sockfd, &rfds);
        ret = select(sockfd+1,&rfds,NULL,NULL,&tv);
        if (ret <= 0) {
               printf("select fail: %s\n", strerror(errno));
               return;
        }
        n = recv(sockfd, long_buff, 11, 0);
        if (n <= 0) {
                printf("recv fail: %s\n", strerror(errno));
                return;
        }
        printf("recv %d bytes\n", n);
        long_buff[n] = 0;

        len = atol(long_buff);
        printf("total %d bytes\n", len);

        ret = RecvLongData(sockfd, long_buff, len);
        if (ret != len) {
                printf("RecvLongData() fail, recv %d bytes\n", ret);
                return;
        }
        long_buff[len] = 0;

        printf("total recv %d bytes\n", ret);
}

int
main(int argc, char **argv)
{
        int                             listenfd, connfd;
        pid_t                           childpid;
        socklen_t                       clilen;
        struct sockaddr_in      cliaddr, servaddr;
        void                            sig_chld(int);

        listenfd = Socket(AF_INET, SOCK_STREAM, 0);

        bzero(&servaddr, sizeof(servaddr));
        servaddr.sin_family      = AF_INET;
        servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
        servaddr.sin_port        = htons(SERV_PORT);

        Bind(listenfd, (SA *) &servaddr, sizeof(servaddr));

        Listen(listenfd, LISTENQ);

        Signal(SIGCHLD, SIG_IGN);

        for ( ; ; ) {
                clilen = sizeof(cliaddr);
                if ( (connfd = accept(listenfd, (SA *) &cliaddr, &clilen)) < 0) {
                        if (errno == EINTR)
                                continue;               /* back to for() */
                        else
                                err_sys("accept error");
                }

                if ( (childpid = Fork()) == 0) {        /* child process */
                        Close(listenfd);        /* close listening socket */

                        srv_process(connfd);    /* process the request */

                        Close(connfd);
                        exit(0);
                }

                Close(connfd);                  /* parent closes connected socket */
        }
}

 

4、编译测试

将测试程序放在unpv13e/tcpclieserv目录下，修改Makefile文件，增加如下语句：

tcpserv20:      tcpserv20.o
                ${CC} ${CFLAGS} -o $@ tcpserv20.o ${LIBS}

tcpcli20:       tcpcli20.o
                ${CC} ${CFLAGS} -o $@ tcpcli20.o ${LIBS}

 

执行测试：

# 先启动服务端
$ tcpserv20

# 查看监听服务是否启动
$ netstat -a | grep 9877

# 启动客户端测试程序
$ tcpcli20

测试结果

# 服务端输出
select fail: Error 0 # 表示select返回值0（超时返回）

# 客户端输出

total send bytes: 1048587
send fail: Broken pipe
send ret: -1
SendLongData() fail, total send -1 bytes.

5、结论

经测试，SendData和RecvData应该不会出现问题，再次查找程序，后来发现是由于客户端先创建连接，然后在组包（组包时间太长，超过60秒），但对方服务端那边accpet连接之后，通过

select检测套接字在30秒之内是否可读，若不可读则关闭连接，这样的话，客户端组好报文后发送报文时对方已经将套接字关闭了，所以send系统调用返回EPIPE错误。最后将创建连接放置在

组包之后，即在发送之前再创建连接，解决此问题。