scrapy实战百度百科数据抓取

Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架。 可以应用在包括数据挖掘，信息处理或存储历史数据等一系列的程序中。

架构


scrapy中文手册 

1.安装SCRAPY

2.创建一个Scrapy项目

scrapy startproject tutorial

生成
tutorial/
    scrapy.cfg
    tutorial/
        __init__.py
        items.py
        pipelines.py
        settings.py
        spiders/
            __init__.py
            ...

3.新建抓取文件spiders/baike_spider.py,实现代码

# -*- coding: utf-8 -*-
from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy.selector import HtmlXPathSelector
from tutorial.items import BaikeItem

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor

i=0
not_data = 0 #统计未采集到的条数
class BaikeSpider(CrawlSpider):

   name = "baike"
   #减慢爬取速度 为1s
   # download_delay = 1
   allowed_domains = ['baike.baidu.com']
   start_urls = [
       "互联网"
   ]



   rules =[
        Rule(SgmlLinkExtractor(allow=('/?limit=30&index=([\w]+)&offset=([\d]+)\#gotoList', ),)),
        Rule(SgmlLinkExtractor(allow=('/view/', ),
          restrict_xpaths=('//div[@class="list"]')),
          callback='parse_item',
          ),
   ]

   def parse_item(self, response):
    global i,not_data
    i+= 1 #记录抓取条数
    print i
    item = BaikeItem()
    sel = HtmlXPathSelector(response)
    baike_url = str(response.url)
    baike_name = sel.xpath('//div[@id="sec-content0"]/h1/span[@class="lemmaTitleH1"]/text()').extract()
    baike_desc = sel.xpath('//div[@class="card-summary-content"]/div[@class="para"]/text()').extract()[0]


    if not baike_name:
      not_data+=1 #记录未抓取到的条数
      print not_data


    if not baike_desc:
      baike_desc = '未抓取到'


    item['title'] = [n.encode('utf-8') for n in baike_name]
    item['link'] = baike_url.encode('utf-8')
    item['desc'] = baike_desc

    yield item

4.编写item

class CsdnItem(scrapy.Item):
    blog_name = scrapy.Field()
    blog_url = scrapy.Field()

5.保存数据到mysql

# -*- coding: utf-8 -*-

# Define your item pipelines here
#保存文件
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See:

from scrapy import log
from twisted.enterprise import adbapi
from scrapy.http import Request
from scrapy.exceptions import DropItem
import time
import MySQLdb
import MySQLdb.cursors
import json
import codecs

class TutorialPipeline(object):

    def __init__(self):
      self.dbpool = adbapi.ConnectionPool('MySQLdb',
                db = 'test',
                user = 'root',
                passwd = '',
                cursorclass = MySQLdb.cursors.DictCursor,
                charset = 'utf8',
                use_unicode = False
        )


    def process_item(self, item, spider):
        query = self.dbpool.runInteraction(self._conditional_insert, item)
        query.addErrback(self.handle_error)


    def _conditional_insert(self, tx, item):
        if item.get('title'):
            tx.execute(\
                "insert into test(id,title, link, descc ) \
                 values (null,%s,%s,%s)",
                 (item['title'],item['link'],item['desc'])
            )

6.执行代码



7.效果