抓取Web数据：Go与Python

原文地址：http://blog.raphaelzhang.com/2013/11/grab-web-data-go-vs-python/

这几天要帮老婆下载几个上的数据，正好在试用Go和Python，便用了Python和Go写了两个程序来抓取数据。当然，分析数据这种高难度的事是搜索引擎帮忙搞定的，具体说是这篇文章给出了好全的一个视频抓取分析。

用Go写的代码大概在160行左右，Python的大概在110行左右，可以发现比起Python来说Go的代码还是更长一些，而且Python的代码也很易懂，因此如果是后台的脚本程序，还真是用Python更好一些。

其中差别最大的大概就是抓取数据的代码了，下面是Python的：

1 2 3 4 5 6 7 8 9 10 11

def get_web_data(url, referer = None):     try:         f = urllib2.urlopen(create_request(url, referer), timeout = 10)         data = f.read()         f.close()         if data[:6] == '\x1f\x8b\x08\x00\x00\x00':             data = gzip.GzipFile(fileobj = cStringIO.StringIO(data)).read()         return data     except Exception, e:         print >> sys.stderr, 'Cannot read data from {0}: {1}'.format(url, e)         return ''

下面是Go的：

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32

func webData(url string, referer string) string {     httpCh := make(chan *http.Response)     go func() {         defer close(httpCh)         client := &http.Client{}         if resp, err := client.Do(request(url, referer)); err == nil {             httpCh <- resp         }     }()       select {     case r, chanOk := <-httpCh:         if chanOk {             defer r.Body.Close()             body, _ := ioutil.ReadAll(r.Body)             s := string(body)             const gzipHdr = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03"             if len(s) > len(gzipHdr) && s[:len(gzipHdr)] == gzipHdr {                 reader, _ := gzip.NewReader(strings.NewReader(s))                 unzipped, _ := ioutil.ReadAll(reader)                 return string(unzipped)             }             return s         } else {             log.Println("err in http, so quit")         }     case <-time.After(10 * time.Second):         log.Println("timed out reading " + url)     }       return "" }

使用Go代码较长的原因一个是Go没有异常，所以中间会有很多err的忽略和处理，而Python直接用一个大的try/except搞定了，另外一个是urlopen方法有一个timeout参数，而Go没有相应的参数，因此需要自己通过channel来实现，这个也会导致代码比较长。

另外Go还有一个稍微不习惯的地方，python里面列表切片不用管是否超出了边界，例如有一个列表a，即使长度是3，我也可以通过a[:6]来访问a的前六个元素，不用事先判断，这样代码会比较简洁，Go需要先判断len(a)是否至少是6，然后再a[:6]，不然会出错。

当然，代码短不一定都好，例如Python和Go分析下载URL的代码，虽然Python的代码较短，但是从维护性和性能来说，还是Go的更好。下面是Python的：

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36

def get_download_url(url, page_url, result, lock):     lock.acquire()     print 'processing', url     lock.release()     data = get_web_data(url, page_url)     obj = json.loads(data)     if 'data' not in obj or 'allot' not in obj or 'prot' not in obj:         print >> sys.stderr, url + ' does NOT have this video'         return          for i in range(len(obj['data']['su'])):         key_url = 'http://{0}/?prot={1}&file={2}&new={3}'.format(obj['allot'], obj['prot'], obj['data']['clipsURL'][i], obj['data']['su'][i])         data = get_web_data(key_url, page_url).split('|')         if len(data) > 6:             mp4_url = '{0}{1}?key={2}'.format(data[0], obj['data']['su'][i], data[3])             lock.acquire()             result += [mp4_url]             lock.release()         else:             print >>sys.stderr, 'error data: {0}'.format('|'.join(data))      def get_download_urls(page_url, vid):     avails = ['http://my.tv.sohu.com/videinfo.jhtml?m=viewtv&vid={0}', 'http://hot.vrs.sohu.com/vrs_flash.action?vid={0}']     threads = []     lock = threading.Lock()     result = []     for avail in avails:         t = threading.Thread(target = get_download_url, args = (avail.format(vid), page_url, result, lock))         threads += [t]         t.daemon = True         t.start()          for t in threads:         t.join()          return result

而Go由于有goroutine和channel，就非常优雅，如下：

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59

type SohuVideoData struct {     ClipUrl []string `json:"clipsURL"`     Video   []string `json:"su"` }   type SohuVideo struct {     Site string        `json:"allot"`     Prot int           `json:"prot"`     Data SohuVideoData `json:"data"` }   func getDownloadUrl(url string, referer string, dataCh chan string) {     log.Println("processing " + url)     var video SohuVideo     data := webData(url, referer)     err := json.NewDecoder(strings.NewReader(data)).Decode(&video)     if err != nil || video.Site == "" {         log.Printf("%s error, try next\n", url)         return     }       defer close(dataCh)     for i, data := range video.Data.Video {         keyUrl := webData(fmt.Sprintf("http://%s/?prot=%d&file=%s&new=%s", video.Site, video.Prot, video.Data.ClipUrl[i], data), referer)         pieces := strings.Split(keyUrl, "|")         if len(pieces) > 6 {             dataCh <- fmt.Sprintf("%s%s?key=%s", pieces[0], data, pieces[3])         } else {             log.Print("error data: %s", keyUrl)         }     } }   func getDownloadUrls(pageUrl string, vid string) []string {     avails := []string{"http://my.tv.sohu.com/videinfo.jhtml?m=viewtv&vid=%s", "http://hot.vrs.sohu.com/vrs_flash.action?vid=%s"}     urlCh := make(chan string)     for _, avail := range avails {         go getDownloadUrl(fmt.Sprintf(avail, vid), pageUrl, urlCh)     }       result := []string{} Loop:     for {         select {         case url, chanOk := <-urlCh:             if chanOk {                 result = append(result, url)             } else {                 break Loop             }         case <-time.After(10 * time.Second):             log.Println("timeout processing json")             close(urlCh)             break Loop         }     }       return result }

从上面的代码看来，主要是Go解析json数据需要自己先定义几个struct type，但是你可以忽略自己不感兴趣的字段，其实也十分棒的，我觉得这些多出来的代码还是不错的。

总体来说，Go和Python的易读性都不错，而Go在Web数据抓取上缺乏不少方便的模块和参数，模块例如没有lxml，因此解析真实的html比较麻烦，而参数例如上面说到的timeout参数，还有一个就是Go没有内建的字符串编码解码模块，这个太不方便了，完全应该是内建模块才行。用Linux改的iconv什么的在Windows上编译真是麻烦死，我个人是很讨厌mingw和cygwin这种东西的，几年前在正式工作中折腾了这些玩意半年，别扭极了，例如cygwin的下载和配置就很麻烦，还有各种路径和编码的问题。

Python用惯了，一些小语法糖在Go里面没有感觉有点不习惯，例如列表推导，还有例如’+’*80打印重复字符，不过这些都是细节了 

当然，Go的goroutine和channel实在是很强大的，另外Go运行时环境要求简单太多了，因此我完全可以直接把Go编译出来的程序复制给其他人用，不像Python，还需要装一个Python环境(有时候还要装一堆第三方模块，例如lxml/PIL之类的)，真是折腾啊，因此Python也只适合后台脚本。

下面是Python与Go抓取搜狐视频的完整代码，不过这些视频网站变得非常快的，也许过几个月就不能用了，主要是留在这里做个对比而已。

另外需要注意的是Go里面getDownloadUrls的实现假设的是最多只有一个模板会匹配下载视频，而且原始网页中必须要有vid这个js变量的声明，其实在搜狐视频里并不完全是这样的。有的时候会同时匹配两个模板，有的时候会声明vids变量。前者需要修改getDownloadUrls与getDownloadUrl函数，不要在getDownloadUrl里面close(dataCh)，而是在getDownloadUrls里defer close(urlCh)，然后再在getDownloadUrl里面return之前(包括最后的隐式return)向dataCh发送一个长度为零的字符串，在urlCh接收到长度为零的字符串以后就知道有一个goroutine处理完毕了。

这里是Python的。

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113

import sys import urllib2 import json import cStringIO import gzip import threading   import lxml.html   def create_request(url, referer = None):     req = urllib2.Request(         urllib2.quote(url.split('#')[0].encode('utf8'), safe = "%/:=&?~#+!$,;'@()*[]"),         headers = {"Accept": "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",                    "Accept-Charset": "GBK,utf-8;q=0.7,*;q=0.3",                    "Accept-Encoding": "gzip",                    "Accept-Language": "zh-CN,zh;q=0.8",                    "Cache-Control": "max-age=0",                    "Connection": "keep-alive",                    "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",         })          if referer is not None:         req.add_header('Referer', referer)     return req   def get_web_data(url, referer = None):     try:         f = urllib2.urlopen(create_request(url, referer), timeout = 10)         data = f.read()         f.close()         if data[:6] == '\x1f\x8b\x08\x00\x00\x00':             data = gzip.GzipFile(fileobj = cStringIO.StringIO(data)).read()         return data     except Exception, e:         print >> sys.stderr, 'Cannot read data from {0}: {1}'.format(url, e)         return ''   def get_vid(page_url):     html = get_web_data(page_url)     if html == '':         return ''          dom = lxml.html.fromstring(html.decode('gbk', 'ignore'), parser = lxml.html.HTMLParser(remove_comments = True))     scripts = dom.xpath('.//script[not(@src)]')     for s in scripts:         if s.text.strip().startswith('var vid='):             return s.text.strip().split('\n')[0].strip().split('"')[1]       return ''   def get_download_url(url, page_url, result, lock):     lock.acquire()     print 'processing', url     lock.release()     data = get_web_data(url, page_url)     obj = json.loads(data)     if 'data' not in obj or 'allot' not in obj or 'prot' not in obj:         print >> sys.stderr, url + ' does NOT have this video'         return          for i in range(len(obj['data']['su'])):         key_url = 'http://{0}/?prot={1}&file={2}&new={3}'.format(obj['allot'], obj['prot'], obj['data']['clipsURL'][i], obj['data']['su'][i])         data = get_web_data(key_url, page_url).split('|')         if len(data) > 6:             mp4_url = '{0}{1}?key={2}'.format(data[0], obj['data']['su'][i], data[3])             lock.acquire()             result += [mp4_url]             lock.release()         else:             print >>sys.stderr, 'error data: {0}'.format('|'.join(data))      def get_download_urls(page_url, vid):     avails = ['http://my.tv.sohu.com/videinfo.jhtml?m=viewtv&vid={0}', 'http://hot.vrs.sohu.com/vrs_flash.action?vid={0}']     threads = []     lock = threading.Lock()     result = []     for avail in avails:         t = threading.Thread(target = get_download_url, args = (avail.format(vid), page_url, result, lock))         threads += [t]         t.daemon = True         t.start()          for t in threads:         t.join()          return result   if __name__ == '__main__':     if len(sys.argv) != 2:         print >> sys.stderr, 'Please input a sohu video page url, like http://tv.sohu.com/20131122/n390605984.shtml'         exit(1)          video_page_url = sys.argv[1].lower()     if not video_page_url.startswith('http://tv.sohu.com/'):         video_page_url = 'http://tv.sohu.com/' + video_page_url          print 'start grabbing', video_page_url     vid = get_vid(video_page_url)     if vid == '':         print >> sys.stderr, 'vid not found :('         exit(2)          print 'vid =', vid     download_urls = get_download_urls(video_page_url, vid)     ffmpeg_list = []     for i in range(len(download_urls)):         print 'wget "{0}" -O {1}_{2}.mp4'.format(download_urls[i], vid, i+1)         ffmpeg_list.append("file '{0}_{1}.mp4'".format(vid, i+1))          for f in ffmpeg_list:         print f          print 'ffmpeg -f concat -i {0}.txt -c copy {0}.mp4'.format(vid)

下面是Go的：

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155

package main   import (     "compress/gzip"     "encoding/json"     "fmt"     "io/ioutil"     "log"     "net/http"     "os"     "runtime"     "strings"     "time" )   type SohuVideoData struct {     ClipUrl []string `json:"clipsURL"`     Video   []string `json:"su"` }   type SohuVideo struct {     Site string        `json:"allot"`     Prot int           `json:"prot"`     Data SohuVideoData `json:"data"` }   func request(url string, referer string) *http.Request {     req, _ := http.NewRequest("GET", url, nil)     req.Header.Add("Accept", "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")     req.Header.Add("Accept-Encoding", "gzip")     req.Header.Add("Accept-Charset", "GBK,utf-8;q=0.7,*;q=0.3")     req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")     req.Header.Add("Cache-Control", "max-age=0")     req.Header.Add("Connection", "keep-alive")     req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1")     if referer != "" {         req.Header.Add("Referer", referer)     }     return req }   func webData(url string, referer string) string {     httpCh := make(chan *http.Response)     go func() {         defer close(httpCh)         client := &http.Client{}         if resp, err := client.Do(request(url, referer)); err == nil {             httpCh <- resp         }     }()       select {     case r, chanOk := <-httpCh:         if chanOk {             defer r.Body.Close()             body, _ := ioutil.ReadAll(r.Body)             s := string(body)             const gzipHdr = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03"             if len(s) > len(gzipHdr) && s[:len(gzipHdr)] == gzipHdr {                 reader, _ := gzip.NewReader(strings.NewReader(s))                 unzipped, _ := ioutil.ReadAll(reader)                 return string(unzipped)             }             return s         } else {             log.Println("err in http, so quit")         }     case <-time.After(10 * time.Second):         log.Println("timed out reading " + url)     }       return "" }   func getVid(pageUrl string) string {     body := webData(pageUrl, "")     for _, line := range strings.Split(body, "\n") {         if strings.Index(line, "var vid=") != -1 && strings.Index(line, "\"") != -1 {             return strings.Split(line, "\"")[1]         }     }       return "" }   func getDownloadUrl(url string, referer string, dataCh chan string) {     log.Println("processing " + url)     var video SohuVideo     data := webData(url, referer)     err := json.NewDecoder(strings.NewReader(data)).Decode(&video)     if err != nil || video.Site == "" {         log.Printf("%s error, try next\n", url)         return     }       defer close(dataCh)     for i, data := range video.Data.Video {         keyUrl := webData(fmt.Sprintf("http://%s/?prot=%d&file=%s&new=%s", video.Site, video.Prot, video.Data.ClipUrl[i], data), referer)         pieces := strings.Split(keyUrl, "|")         if len(pieces) > 6 {             dataCh <- fmt.Sprintf("%s%s?key=%s", pieces[0], data, pieces[3])         } else {             log.Print("error data: %s", keyUrl)         }     } }   func getDownloadUrls(pageUrl string, vid string) []string {     avails := []string{"http://my.tv.sohu.com/videinfo.jhtml?m=viewtv&vid=%s", "http://hot.vrs.sohu.com/vrs_flash.action?vid=%s"}     urlCh := make(chan string)     for _, avail := range avails {         go getDownloadUrl(fmt.Sprintf(avail, vid), pageUrl, urlCh)     }       result := []string{} Loop:     for {         select {         case url, chanOk := <-urlCh:             if chanOk {                 result = append(result, url)             } else {                 break Loop             }         case <-time.After(10 * time.Second):             log.Println("timeout processing json")             close(urlCh)             break Loop         }     }       return result }   func main() {     if len(os.Args) != 2 || !strings.HasPrefix(strings.ToLower(os.Args[1]), "http://tv.sohu.com/") {         log.Fatal("Input a sohu video page url, like http://tv.sohu.com/20131122/n390605984.shtml")     }       log.Printf("# of CPU: %d\n", runtime.NumCPU())     runtime.GOMAXPROCS(runtime.NumCPU())     pageUrl := strings.ToLower(os.Args[1])     log.Println("Grabbing: " + pageUrl)       vid := getVid(pageUrl)     log.Printf("vid: %s\n", vid)       if vid == "" {         log.Fatal("error getting vid")     }       for i, url := range getDownloadUrls(pageUrl, vid) {         log.Printf("%d: %s", i+1, url)     } }

2013.11.24的补充：

Python抓取Web网页用也是很方便的，但是对我而言，requests的大部分功能也没啥用，urllib2与cookielib大部分时候已经足够使了，而到处安装第三方模块也是非常麻烦的，所以也就很少用requests了。还有一个就是原来python里面检测字符串编码的chardet模块已经没有了，但是现在又有了一个，这个用来做编码检测也是不错的。