Python下的Curl

 

1。

   在找pycurl的使用方法时,对初次使用者,很困难,于是想写个简单的demo方便想涉足者使用:
import pycurl
import StringIO
    url=''
    c=pycurl.Curl()
    c.setopt(c.URL, url)
    b = StringIO.StringIO()   
    c.setopt(c.WRITEFUNCTION, b.write)
    c.setopt(c.FOLLOWLOCATION, 1)
    c.setopt(c.HEADER, True)
    c.perform()   
    html=b.getvalue()   
    print html
    b.close()
    c.close()
=========================================================
def test(debug_type, debug_msg):
    print "debug(%d): %s" % (debug_type, debug_msg)

 curl会用到的一些方法:
 c.setopt(c.HTTPHEADER, ["Content-Type: application/x-www-form-urlencoded","X-Requested-With:XMLHttpRequest","Cookie:"+set_cookie[0]])
 c.setopt(c.REFERER, url)
c.setopt(c.POSTFIELDS, params)
c.setopt(c.VERBOSE, 1)

c.setopt(c.POST, 1)
c.setopt(c.DEBUGFUNCTION, test)   


   url = ""
   
    print "Starting downloading", url
    print
    f = open("body", "wb")
    h = open("header", "wb")
    c = pycurl.Curl()
    c.setopt(c.URL, url)
    c.setopt(c.WRITEDATA, f)
    c.setopt(c.NOPROGRESS, 0)
    c.setopt(c.PROGRESSFUNCTION, progress)
    c.setopt(c.FOLLOWLOCATION, 1)
    c.setopt(c.MAXREDIRS, 5)
    c.setopt(c.WRITEHEADER, h)
    c.setopt(c.POST, 1)
    c.setopt(c.OPT_FILETIME, 1)
    c.perform()  
   
    print "HTTP-code:", c.getinfo(c.HTTP_CODE)
    print "Total-time:", c.getinfo(c.TOTAL_TIME)
    print "Download speed: %.2f bytes/second" % c.getinfo(c.SPEED_DOWNLOAD)
    print "Document size: %d bytes" % c.getinfo(c.SIZE_DOWNLOAD)
    print "Effective URL:", c.getinfo(c.EFFECTIVE_URL)
    print "Content-type:", c.getinfo(c.CONTENT_TYPE)
    print "Namelookup-time:", c.getinfo(c.NAMELOOKUP_TIME)
    print "Redirect-time:", c.getinfo(c.REDIRECT_TIME)
    print "Redirect-count:", c.getinfo(c.REDIRECT_COUNT)
    epoch = c.getinfo(c.INFO_FILETIME)
    #print "Filetime: %d (%s)" % (epoch, time.ctime(epoch))
    #print
    print "Header is in file 'header', body is in file 'body'"
   
    c.close()
    f.close()
    h.close()

    #print pycurl.version_info()
    url=''
    c=pycurl.Curl()
    c.setopt(pycurl.URL, url);
   
    b = StringIO.StringIO()   
    c.setopt(pycurl.HTTPHEADER, ["Accept:"])

    c.setopt(pycurl.WRITEFUNCTION, b.write)

    c.setopt(pycurl.FOLLOWLOCATION, 2)
    #c.setopt(pycurl.HEADER, True)
    c.setopt(pycurl.MAXREDIRS, 5)
    #c.setopt(pycurl.USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)")
    #c.setopt(pycurl.REFERER, "")
    #c.setopt(pycurl.CONNECTTIMEOUT, 20)#链接超时   
    #c.setopt(pycurl.TIMEOUT, 20)#下载超时
    #c.setopt(pycurl.COOKIEFILE, "cookie_file_name")   
    #c.setopt(pycurl.COOKIEJAR, "cookie_file_name")
    c.perform()   
    #print ret
    html=b.getvalue()   
    print '-----------'
    print html
========================代理使用

defgetURLContent_pycurl(url):   

   c = pycurl.Curl()

   c.setopt(pycurl.URL,url)

   b = StringIO.StringIO()

   c.setopt(pycurl.WRITEFUNCTION, b.write)

   c.setopt(pycurl.FOLLOWLOCATION, 1)

   c.setopt(pycurl.MAXREDIRS, 5)

   #代理

   #c.setopt(pycurl.PROXY, '')

   #c.setopt(pycurl.PROXYUSERPWD, 'aaa:aaa')

   c.perform()

   returnb.getvalue()

url ='http://blog.csdn.net'

content = getURLContent_pycurl(url)

printcontent

 

2。

这阵子使用python里读rss保存到数据库里，但使用了一段时间urllib觉得慢，在网上说pycurl的速度比urllib快，于是尝试使用，记录下使用方法:

1. #!/usr/bin/env python
2. # -*- coding: utf-8 -*-
3. import StringIO
4. import pycurl 
5. html = StringIO.StringIO()
6. c = pycurl.Curl()
7. myurl=''
8. c.setopt(pycurl.URL, myurl)
9. #写的回调
10. c.setopt(pycurl.WRITEFUNCTION, html.write)
11. c.setopt(pycurl.FOLLOWLOCATION, 1)
12. #最大重定向次数,可以预防重定向陷阱
13. c.setopt(pycurl.MAXREDIRS, 5)
14. #连接超时设置
15. c.setopt(pycurl.CONNECTTIMEOUT, 60)
16. c.setopt(pycurl.TIMEOUT, 300)
17. #模拟浏览器
18. c.setopt(pycurl.USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)")
19. #访问,阻塞到访问结束
20. c.perform()
21. #打印出 200(HTTP状态码)
22. print c.getinfo(pycurl.HTTP_CODE)
23. #输出网页的内容
24. print html.getvalue()
25. #输出网页类型
26. print "Content-type:", c.getinfo(c.CONTENT_TYPE)

安装pycurl到http://pycurl.sourceforge.net/这里去找.
在windows安装的话http://pycurl.sourceforge.net/download/ , 看你使用的版本决定下载那个，我在 windows使用的是python2.4, 所以下载 pycurl-ssl-7.15.5.1.win32-py2.4.exe 。

#-*- coding:utf-8 -*-
import os
import pycurl
import StringIO

html = StringIO.StringIO()
url = r''

c = pycurl.Curl()
c.setopt(pycurl.URL,url)
c.setopt(pycurl.SSL_VERIFYHOST, False)
c.setopt(pycurl.SSL_VERIFYPEER,False)
#c.setopt(pycurl.USERAGENT,r"User-Agent: Dalvik/1.4.0 (Linux; U; Android 2.3.7; Milestone Build/SHOLS_U2_05.26.3)")
c.setopt(pycurl.WRITEFUNCTION, html.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.perform()

print c.getinfo(pycurl.HTTP_CODE), c.getinfo(pycurl.EFFECTIVE_URL)
print html.getvalue()