python网页爬虫之列车时刻表的抓取(4)-完整的python脚本

python网页爬虫之列车时刻表的抓取(4)-完整的python脚本

2010-05-30 18:28

#! /usr/bin/env python
#coding=utf8
# by 2010/5/30
from HTMLParser import HTMLParser
from pyquery import PyQuery as pq
import sqlite3,urllib2,logging,sys
from datetime import datetime
from decimal import Decimal

# 日志初始化
logFileName='./%s.log'%(datetime.now().strftime("%Y%m%d%H%M%S"))
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename=logFileName,
                    filemode='w')
logger = logging.getLogger('transchedule')
hdlr = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)

global witchTrain # 所属当前页面列车的序列号
witchTrain = 0

# 初始化数据库
conn = sqlite3.connect('trainsInfo.sqlite')
c = conn.cursor()

def getTrainInfo(i,e):   
    result=[]
    # 取车次
    trainNumbers=pq(e)('tr td').eq(2).text().split(' ')
    for oneTNum in trainNumbers:
        result.append([oneTNum])
    # 运行时间
    runtimeMeta=pq(e)('tr td').eq(4).text().split(' ')
    for (counter,oneRuntime) in enumerate(runtimeMeta):
        runtime = int(oneRuntime.split(u'时')[0])*60+int(oneRuntime.split(u'时')[1][0:-1])
        result[counter].append(runtime)
    # 始发站
    startingStations=pq(e)('tr').eq(1)('td').eq(1).text().split(' ')
    for (counter,oneStartingStation) in enumerate(startingStations):
        result[counter].append(oneStartingStation)
    # 终点站
    terminatingStations=pq(e)('tr').eq(1)('td').eq(3).text().split(' ')
    for (counter,oneTerminatingStation) in enumerate(terminatingStations):
        result[counter].append(oneTerminatingStation)
    # 始发时间
    departureTimes=pq(e)('tr').eq(2)('td').eq(1).text().split(' ')
    for (counter,oneDepartureTime) in enumerate(departureTimes):
        result[counter].append(datetime.strptime(oneDepartureTime,'%H:%M'))
    # 到达时间
    arrivalTimes=pq(e)('tr').eq(2)('td').eq(3).text().split(' ')
    for (counter,oneArrivalTime) in enumerate(arrivalTimes):
        result[counter].append(datetime.strptime(oneArrivalTime,'%H:%M'))
    # 类型
    clazzes=pq(e)('tr').eq(3)('td').eq(1).text().split(' ')
    for (counter,oneClazz) in enumerate(clazzes):
        result[counter].append(oneClazz)
    # 全程
    ranges=pq(e)('tr').eq(3)('td').eq(3).text().split(' ')
    for (counter,oneRange) in enumerate(ranges):
        result[counter].append(int(oneRange[0:-2]))
    return result
   
def insertTrainInfo(trainData,cursor):
    try:
        cursor.execute('insert into trains_Info values(?,?,?,?,?,?,?,?)',trainData)
        logger.info("train number %s processed"%trainData[0])
    except Exception,e:
        logger.error("%s %s"%(e,trainData[0]))

def getScheduleInfo(i,e):
    global witchTrain # 所属当前页面列车的序列号
    td = pq(e)('td')
    if td.eq(0).text() in ('No.',""):
        witchTrain += 1
        return
    # 解析异常处理
    if len(td) == 2:
        logger.error("%s:%s"%(td.text().encode('gb18030'),len(td)))
        return    
    # 停车时间
    stopTime = datetime.strptime("00:00",'%H:%M')
    try:
        stopTime = datetime.strptime(td.eq(5).text(),'%H:%M')
    except Exception:
        #print 'stop time parse error:%s:%s'%(td.eq(5).text(),td.eq(0).text())
        None
    # 开车时间
    startTime = datetime.strptime("00:00",'%H:%M')
    try:
        startTime = datetime.strptime(td.eq(6).text(),'%H:%M')
    except Exception:
        #print 'start time parse error:%s:%s'%(td.eq(6).text(),td.eq(0).text())
        None
    # 里程
    range = int(td.eq(7).text()[:-2])
    # 硬座
    hardSeatPrice=0.0
    if td.eq(8) and len(td.eq(8).text()) > 1:
        hardSeatPrice=td.eq(8).text()[:-1]
    # 硬卧中铺
    hardBerthPrice=0.0
    if td.eq(9) and len(td.eq(9).text()) > 1 and td.eq(9).text()[:-1] <> "-":
        hardBerthPrice=td.eq(9).text()[:-1]
    # 软座
    softSeatPrice=0.0
    if td.eq(10) and len(td.eq(10).text()) > 1:
        softSeatPrice=td.eq(10).text()[:-1]
    # 软卧下铺
    softBerthPrice='0'
    if td.eq(11) and len(td.eq(11).text()) > 1:
        softBerth=td.eq(11).text()[:-1]
    return [[witchTrain,
            int(td.eq(0).text()),td.eq(1).text(),
            td.eq(4).text(),stopTime,
            startTime,range,
            hardSeatPrice,hardBerthPrice,
            softSeatPrice,softBerthPrice,]]

def insertTrainSchedule(trainInfos,scheduleData,cursor):      
    scheduleData[0]=trainInfos[scheduleData[0]-1][0]
    try:
      cursor.execute('insert into trains_schedule values(?,?,?,?,?,?,?,?,?,?,?)',scheduleData)
    except Exception,e:
      logger.error("%s %s"%(e,scheduleData[0]))

def getNextPageLink(e):
    d = pq(e)
    if d.text() == u"下一页":
        return d.attr('href')

# 获取页面内的车次
def processTrainsInPage(url):
    # 解析指定的连接   
    d = pq(url=url)
    # 取得车次列表
    lis = d('body center div.ListContent div.ListContentLeft ul li')
    lis.make_links_absolute()
    trains = lis.map(lambda i,e:pq(e)('a').attr('href'))
    # 循环处理每个车次的时刻表
    for oneTrain in trains:
        pageContent=urllib2.urlopen(oneTrain).read().replace("gb2312","gb18030")
        # 列车信息
        trainInfo = pq(pageContent)('body center div.ResultContent div.ResultContentLeft div.ResultContentLeftContent\
div.ResultTrainCodeContent table').eq(1)
        trainInfos = trainInfo.map(getTrainInfo)
        for oneTrainInfo in trainInfos:
            insertTrainInfo(oneTrainInfo,c)
            conn.commit()
        # 列车途经站点
        trainSchedule = pq(pageContent)('body center div.ResultContent div.ResultContentLeft div.ResultContentLeftContent\
div.ResultTrainCodeContent table').eq(2)('tr')
        trainSchedules = trainSchedule.map(getScheduleInfo)
        global witchTrain
        witchTrain = 0
        for oneTrainSchedule in trainSchedules:
            insertTrainSchedule(trainInfos,oneTrainSchedule,c)
            conn.commit()
    # 递归处理下一页
    nextPageTable = d('body center div.ListContent div.ListContentLeft div.ListContentLeftContent').eq(2)('a')
    nextPageTable.make_links_absolute()
    lis=nextPageTable.map(lambda i,e:getNextPageLink(e))
    if lis[0] == url:
        logger.info(lis[0].encode('gb18030'))
    else:
        logger.info("start process %s"%lis[0])
        processTrainsInPage(lis[0])

logger.info("start get data...")
try:
    processTrainsInPage(r"")
except Exception,e:
    logger.error(e)
logger.info("finish...")