python网页爬虫之列车时刻表的抓取(5)-错误数据的处理

python网页爬虫之列车时刻表的抓取(5)-错误数据的处理

2010-05-30 19:35

抓取了99%的数据还剩一点点瑕疵要处理掉.

在脚本的运行日志里不和谐的一段:

05-29 18:14 transchedule INFO     train number 5611 processed
05-29 18:14 transchedule ERROR    2:2
05-29 18:14 transchedule INFO     start process
05-29 18:14 transchedule INFO     train number 5612 processed
05-29 18:14 transchedule ERROR    7 黄:2

5611和5612的椑木镇和黄磏,这两个生僻字直接导致了我的脚本少了14条途经车站的数据....数据该升级啦

铁路网的数据:                                       这边是极品时刻表的:

                          

                               
原来他们的数据源是一样的.....到他们家论坛提个BUG去.

把5611和5612存到本地,改掉html的gb2312编码,补好数据,完工啦.

========数据补充脚本========

#! /usr/bin/env python
#coding=utf8
#import urllib2
from HTMLParser import HTMLParser
from pyquery import PyQuery as pq
import sqlite3,urllib2,logging,sys
from datetime import datetime
from decimal import Decimal

# 日志初始化
logFileName='./%s.log'%(datetime.now().strftime("%Y%m%d%H%M%S"))
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename=logFileName,
                    filemode='w')
logger = logging.getLogger('transchedule')
hdlr = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)

global witchTrain # 所属当前页面列车的序列号
witchTrain = 0

# 初始化数据库
conn = sqlite3.connect('trainsInfo.sqlite')
c = conn.cursor()

def getScheduleInfo(i,e):
    global witchTrain # 所属当前页面列车的序列号
    td = pq(e)('td')
    if td.eq(0).text() in ('No.',""):
        witchTrain += 1
        return
    # 解析异常处理
    if len(td) == 2:
        logger.error("%s:%s"%(td.text().encode('gb18030'),len(td)))
        return    
    # 停车时间
    stopTime = datetime.strptime("00:00",'%H:%M')
    try:
        stopTime = datetime.strptime(td.eq(5).text(),'%H:%M')
    except Exception:
        #print 'stop time parse error:%s:%s'%(td.eq(5).text(),td.eq(0).text())
        None
    # 开车时间
    startTime = datetime.strptime("00:00",'%H:%M')
    try:
        startTime = datetime.strptime(td.eq(6).text(),'%H:%M')
    except Exception:
        #print 'start time parse error:%s:%s'%(td.eq(6).text(),td.eq(0).text())
        None
    # 里程
    range = int(td.eq(7).text()[:-2])
    # 硬座
    hardSeatPrice=0.0
    if td.eq(8) and len(td.eq(8).text()) > 1:
        hardSeatPrice=td.eq(8).text()[:-1]
    # 硬卧中铺
    hardBerthPrice=0.0
    if td.eq(9) and len(td.eq(9).text()) > 1 and td.eq(9).text()[:-1] <> "-":
        hardBerthPrice=td.eq(9).text()[:-1]
    # 软座
    softSeatPrice=0.0
    if td.eq(10) and len(td.eq(10).text()) > 1:
        softSeatPrice=td.eq(10).text()[:-1]
    # 软卧下铺
    softBerthPrice='0'
    if td.eq(11) and len(td.eq(11).text()) > 1:
        softBerth=td.eq(11).text()[:-1]
    return [[witchTrain,
            int(td.eq(0).text()),td.eq(1).text(),
            td.eq(4).text(),stopTime,
            startTime,range,
            hardSeatPrice,hardBerthPrice,
            softSeatPrice,softBerthPrice,]]

def insertTrainSchedule(trainInfo,scheduleData,cursor):      
    scheduleData[0]=trainInfo
    try:
      cursor.execute('insert into trains_schedule values(?,?,?,?,?,?,?,?,?,?,?)',scheduleData)
    except Exception,e:
      logger.error("%s %s"%(e,scheduleData[0]))

# 获取页面内的车次
def get561x(pageFilePath):
    # 列车途经站点
    trainSchedule = pq(filename=pageFilePath)('body center div.ResultContent div.ResultContentLeft div.ResultContentLeftContent\
div.ResultTrainCodeContent table').eq(2)('tr')
    trainSchedules = trainSchedule.map(getScheduleInfo)
    global witchTrain
    witchTrain = 0
    for oneTrainSchedule in trainSchedules:
        insertTrainSchedule(pageFilePath[:-5],oneTrainSchedule,c)
        conn.commit()

logger.info("start get data...")
try:
    for oneHtmlFile in ['5611.html','5612.html',]:
        get561x(oneHtmlFile)
except Exception,e:
    logger.error(e)
logger.info("finish...")

导出sqlite数据文件:4,880,384 bytes