Chinaunix首页 | 论坛 | 博客
  • 博客访问: 507311
  • 博文数量: 111
  • 博客积分: 3160
  • 博客等级: 中校
  • 技术积分: 1982
  • 用 户 组: 普通用户
  • 注册时间: 2010-04-24 11:49
个人简介

低调、勤奋。

文章分类

全部博文(111)

文章存档

2014年(2)

2013年(26)

2012年(38)

2011年(18)

2010年(27)

分类: C/C++

2012-08-30 14:23:01


点击(此处)折叠或打开

  1. #include <stdio.h>
  2. #include <string.h>

  3. #include <stdlib.h>
  4. #include <errno.h>

  5. #include <curl/curl.h>

  6. enum fcurl_type_e {
  7.   CFTYPE_NONE=0,
  8.   CFTYPE_FILE=1,
  9.   CFTYPE_CURL=2
  10. };

  11. struct fcurl_data
  12. {
  13.   enum fcurl_type_e type; /* type of handle */
  14.   union {
  15.     CURL *curl;
  16.     FILE *file;
  17.   } handle; /* handle */

  18.   char *buffer; /* buffer to store cached data*/
  19.   size_t buffer_len; /* currently allocated buffers length */
  20.   size_t buffer_pos; /* end of data in buffer*/
  21.   int still_running; /* Is background url fetch still in progress */
  22. };

  23. typedef struct fcurl_data URL_FILE;

  24. /* exported functions */
  25. URL_FILE *url_fopen(const char *url,const char *operation);
  26. int url_fclose(URL_FILE *file);
  27. int url_feof(URL_FILE *file);
  28. size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
  29. char * url_fgets(char *ptr, size_t size, URL_FILE *file);
  30. void url_rewind(URL_FILE *file);

  31. /* we use a global one for convenience */
  32. CURLM *multi_handle;

  33. /* curl calls this routine to get more data */
  34. static size_t write_callback(char *buffer,
  35.                              size_t size,
  36.                              size_t nitems,
  37.                              void *userp)
  38. {
  39.   char *newbuff;
  40.   size_t rembuff;

  41.   URL_FILE *url = (URL_FILE *)userp;
  42.   size *= nitems;

  43.   rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */

  44.   if(size > rembuff) {
  45.     /* not enough space in buffer */
  46.     newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
  47.     if(newbuff==NULL) {
  48.       fprintf(stderr,"callback buffer grow failed\n");
  49.       size=rembuff;
  50.     }
  51.     else {
  52.       /* realloc suceeded increase buffer size*/
  53.       url->buffer_len+=size - rembuff;
  54.       url->buffer=newbuff;
  55.     }
  56.   }
  57.   
  58.   memcpy(url->buffer + url->buffer_pos, buffer, size);
  59.   url->buffer_pos += size;

  60.   return size;
  61. }

  62. /* use to attempt to fill the read buffer up to requested number of bytes */
  63. static int fill_buffer(URL_FILE *file, size_t want)
  64. {
  65.   fd_set fdread;
  66.   fd_set fdwrite;
  67.   fd_set fdexcep;
  68.   struct timeval timeout;
  69.   int rc;

  70.   /* only attempt to fill buffer if transactions still running and buffer
  71.    * doesnt exceed required size already
  72.    */
  73.   if((!file->still_running) || (file->buffer_pos > want))
  74.     return 0;

  75.   /* attempt to fill buffer */
  76.   do {
  77.     int maxfd = -1;
  78.     long curl_timeo = -1;

  79.     FD_ZERO(&fdread);
  80.     FD_ZERO(&fdwrite);
  81.     FD_ZERO(&fdexcep);

  82.     /* set a suitable timeout to fail on */
  83.     timeout.tv_sec = 60; /* 1 minute */
  84.     timeout.tv_usec = 0;

  85.     curl_multi_timeout(multi_handle, &curl_timeo);
  86.     if(curl_timeo >= 0) {
  87.       timeout.tv_sec = curl_timeo / 1000;
  88.       if(timeout.tv_sec > 1)
  89.         timeout.tv_sec = 1;
  90.       else
  91.         timeout.tv_usec = (curl_timeo % 1000) * 1000;
  92.     }

  93.     /* get file descriptors from the transfers */
  94.     curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);

  95.     /* In a real-world program you OF COURSE check the return code of the
  96.        function calls. On success, the value of maxfd is guaranteed to be
  97.        greater or equal than -1. We call select(maxfd + 1, ...), specially
  98.        in case of (maxfd == -1), we call select(0, ...), which is basically
  99.        equal to sleep. */

  100.     rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);

  101.     switch(rc) {
  102.     case -1:
  103.       /* select error */
  104.       break;

  105.     case 0:
  106.     default:
  107.       /* timeout or readable/writable sockets */
  108.       curl_multi_perform(multi_handle, &file->still_running);
  109.       break;
  110.     }
  111.   } while(file->still_running && (file->buffer_pos < want));
  112.   return 1;
  113. }

  114. /* use to remove want bytes from the front of a files buffer */
  115. static int use_buffer(URL_FILE *file,int want)
  116. {
  117.   /* sort out buffer */
  118.   if((file->buffer_pos - want) <=0) {
  119.     /* ditch buffer - write will recreate */
  120.     if(file->buffer)
  121.       free(file->buffer);

  122.     file->buffer=NULL;
  123.     file->buffer_pos=0;
  124.     file->buffer_len=0;
  125.   }
  126.   else {
  127.     /* move rest down make it available for later */
  128.     memmove(file->buffer,
  129.             &file->buffer[want],
  130.             (file->buffer_pos - want));

  131.     file->buffer_pos -= want;
  132.   }
  133.   return 0;
  134. }

  135. URL_FILE *url_fopen(const char *url,const char *operation)
  136. {
  137.   /* this code could check for URLs or types in the 'url' and
  138.      basicly use the real fopen() for standard files */

  139.   URL_FILE *file;
  140.   (void)operation;

  141.   file = malloc(sizeof(URL_FILE));
  142.   if(!file)
  143.     return NULL;

  144.   memset(file, 0, sizeof(URL_FILE));

  145.   if((file->handle.file=fopen(url,operation)))
  146.     file->type = CFTYPE_FILE; /* marked as URL */

  147.   else {
  148.     file->type = CFTYPE_CURL; /* marked as URL */
  149.     file->handle.curl = curl_easy_init();

  150.     curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
  151.     curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
  152.     curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
  153.     curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);

  154.     if(!multi_handle)
  155.       multi_handle = curl_multi_init();

  156.     curl_multi_add_handle(multi_handle, file->handle.curl);

  157.     /* lets start the fetch */
  158.     curl_multi_perform(multi_handle, &file->still_running);

  159.     if((file->buffer_pos == 0) && (!file->still_running)) {
  160.       /* if still_running is 0 now, we should return NULL */

  161.       /* make sure the easy handle is not in the multi handle anymore */
  162.       curl_multi_remove_handle(multi_handle, file->handle.curl);

  163.       /* cleanup */
  164.       curl_easy_cleanup(file->handle.curl);

  165.       free(file);

  166.       file = NULL;
  167.     }
  168.   }
  169.   return file;
  170. }

  171. int url_fclose(URL_FILE *file)
  172. {
  173.   int ret=0;/* default is good return */

  174.   switch(file->type) {
  175.   case CFTYPE_FILE:
  176.     ret=fclose(file->handle.file); /* passthrough */
  177.     break;

  178.   case CFTYPE_CURL:
  179.     /* make sure the easy handle is not in the multi handle anymore */
  180.     curl_multi_remove_handle(multi_handle, file->handle.curl);

  181.     /* cleanup */
  182.     curl_easy_cleanup(file->handle.curl);
  183.     break;

  184.   default: /* unknown or supported type - oh dear */
  185.     ret=EOF;
  186.     errno=EBADF;
  187.     break;
  188.   }

  189.   if(file->buffer)
  190.     free(file->buffer);/* free any allocated buffer space */

  191.   free(file);

  192.   return ret;
  193. }

  194. int url_feof(URL_FILE *file)
  195. {
  196.   int ret=0;

  197.   switch(file->type) {
  198.   case CFTYPE_FILE:
  199.     ret=feof(file->handle.file);
  200.     break;

  201.   case CFTYPE_CURL:
  202.     if((file->buffer_pos == 0) && (!file->still_running))
  203.       ret = 1;
  204.     break;

  205.   default: /* unknown or supported type - oh dear */
  206.     ret=-1;
  207.     errno=EBADF;
  208.     break;
  209.   }
  210.   return ret;
  211. }

  212. size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
  213. {
  214.   size_t want;

  215.   switch(file->type) {
  216.   case CFTYPE_FILE:
  217.     want=fread(ptr,size,nmemb,file->handle.file);
  218.     break;

  219.   case CFTYPE_CURL:
  220.     want = nmemb * size;

  221.     fill_buffer(file,want);

  222.     /* check if theres data in the buffer - if not fill_buffer()
  223.      * either errored or EOF */
  224.     if(!file->buffer_pos)
  225.       return 0;

  226.     /* ensure only available data is considered */
  227.     if(file->buffer_pos < want)
  228.       want = file->buffer_pos;

  229.     /* xfer data to caller */
  230.     memcpy(ptr, file->buffer, want);

  231.     use_buffer(file,want);

  232.     want = want / size; /* number of items */
  233.     break;

  234.   default: /* unknown or supported type - oh dear */
  235.     want=0;
  236.     errno=EBADF;
  237.     break;

  238.   }
  239.   return want;
  240. }

  241. char *url_fgets(char *ptr, size_t size, URL_FILE *file)
  242. {
  243.   size_t want = size - 1;/* always need to leave room for zero termination */
  244.   size_t loop;

  245.   switch(file->type) {
  246.   case CFTYPE_FILE:
  247.     ptr = fgets(ptr,size,file->handle.file);
  248.     break;

  249.   case CFTYPE_CURL:
  250.     fill_buffer(file,want);

  251.     /* check if theres data in the buffer - if not fill either errored or
  252.      * EOF */
  253.     if(!file->buffer_pos)
  254.       return NULL;

  255.     /* ensure only available data is considered */
  256.     if(file->buffer_pos < want)
  257.       want = file->buffer_pos;

  258.     /*buffer contains data */
  259.     /* look for newline or eof */
  260.     for(loop=0;loop < want;loop++) {
  261.       if(file->buffer[loop] == '\n') {
  262.         want=loop+1;/* include newline */
  263.         break;
  264.       }
  265.     }

  266.     /* xfer data to caller */
  267.     memcpy(ptr, file->buffer, want);
  268.     ptr[want]=0;/* allways null terminate */

  269.     use_buffer(file,want);

  270.     break;

  271.   default: /* unknown or supported type - oh dear */
  272.     ptr=NULL;
  273.     errno=EBADF;
  274.     break;
  275.   }

  276.   return ptr;/*success */
  277. }

  278. void url_rewind(URL_FILE *file)
  279. {
  280.   switch(file->type) {
  281.   case CFTYPE_FILE:
  282.     rewind(file->handle.file); /* passthrough */
  283.     break;

  284.   case CFTYPE_CURL:
  285.     /* halt transaction */
  286.     curl_multi_remove_handle(multi_handle, file->handle.curl);

  287.     /* restart */
  288.     curl_multi_add_handle(multi_handle, file->handle.curl);

  289.     /* ditch buffer - write will recreate - resets stream pos*/
  290.     if(file->buffer)
  291.       free(file->buffer);

  292.     file->buffer=NULL;
  293.     file->buffer_pos=0;
  294.     file->buffer_len=0;

  295.     break;

  296.   default: /* unknown or supported type - oh dear */
  297.     break;
  298.   }
  299. }

  300. /* Small main program to retrive from a url using fgets and fread saving the
  301.  * output to two test files (note the fgets method will corrupt binary files if
  302.  * they contain 0 chars */
  303. int main(int argc, char *argv[])
  304. {
  305.   URL_FILE *handle;
  306.   FILE *outf;

  307.   int nread;
  308.   char buffer[256];
  309.   const char *url;

  310.   if(argc < 2)
  311.     url="";/* default to testurl */
  312.   else
  313.     url=argv[1];/* use passed url */

  314.   /* Copy from url with fread */
  315.   outf=fopen("fread.test","w+");
  316.   if(!outf) {
  317.     perror("couldn't open fread output file\n");
  318.     return 1;
  319.   }

  320.   handle = url_fopen(url, "r");
  321.   if(!handle) {
  322.     printf("couldn't url_fopen() testfile\n");
  323.     fclose(outf);
  324.     return 2;
  325.   }

  326.   do {
  327.     nread = url_fread(buffer, 1,sizeof(buffer), handle);
  328.     fwrite(buffer,1,nread,outf);
  329.   } while(nread);

  330.   url_fclose(handle);

  331.   fclose(outf);

  332.   return 0;/* all done */
  333. }
gcc -o get_url get_url.c -L/usr/local/lib -lcurl
阅读(3234) | 评论(0) | 转发(1) |
给主人留下些什么吧!~~