beautifulsoup 爬取网络视频数据

测试

#coding:utf-8

from bs4 import BeautifulSoup
import urllib2
import re
import sys

HomePage=""
'''
    获取网站的菜单列表地址
'''
def getMenuList():
    menulist={}
    webdata=urllib2.urlopen(HomePage).read()
    soup=BeautifulSoup(webdata,'lxml')
    menu=soup.find_all(id='menu')
# print urls
    for m in menu:
        url=m.find_all('a')
        for u in url:
            href=u.get('href')
            title=u.get_text()
            if not re.match(r'http:', href):
                href=HomePage+href
# print title,href
            menulist[title]=href
    return menulist

'''
    获取实际视频文件的地址
'''
def getFuliMp4Url(url):
    webdata=urllib2.urlopen(url).read()
    soup=BeautifulSoup(webdata,'lxml')
    urls=soup.find_all(type="text/javascript")

    for u in urls:
        return soup.title.get_text().split('-')[1],u.get_text().split('"')[1]

'''
    获取视频播发器地址
'''
def getFuliPlayer(url):
    webdata=urllib2.urlopen(url).read()
    soup=BeautifulSoup(webdata,'lxml')
    urls=soup.find_all(title='第1集')
    for u in urls:
        href=u.get('href')
        if not re.match(r'http', href):
            href=HomePage+href
        return href

'''
    获取当前页所有播发器页面地址
'''
def getFuliCurrPageUrls(url):
    player_onepage=[]
    webdata=urllib2.urlopen(url).read()
    soup=BeautifulSoup(webdata,'lxml')
    content=soup.find_all(id="content")
    for c in content:
        us=c.find_all('a')
        for u in us:
            addr=u.get('href')
            if re.match(r'http', addr):
# print addr
                player_onepage.append(addr)
    return player_onepage
'''
    获取福利片页面索引
'''
def getFuliPageIndex():
    allpage_url=[]
    #从主页获取 '福利片'页面url
    menu=getMenuList()
    curr_page=menu.get(u'福利片')
    allpage_url.append(curr_page)
    while True:

        webdata=urllib2.urlopen(curr_page).read()
        soup=BeautifulSoup(webdata,'lxml')
        urls=soup.find_all('a')
        for u in urls:
            if u.get_text()==u'下一页':
                next_page_url=u.get('href')
    # print next_page_url
        if not re.match(r'http:', next_page_url):
            next_page_url=HomePage+next_page_url
        if next_page_url:
            print "current"+curr_page
            if next_page_url in allpage_url:
                return allpage_url
            allpage_url.append(next_page_url)
            curr_page=next_page_url

        else:
            return allpage_url

        #测试1个页面
# return allpage_url





if __name__=="__main__":

    import sys
    reload(sys)
    sys.setdefaultencoding('utf8')

    fd=open('fuli_allmp4.txt','w')
    fd.close()
    fd=open('fuli_allmp4.txt','a+')

    fd_notitle=open('fuli_allmp4_notitle.txt','w')
    fd_notitle.close()
    fd_notitle=open('fuli_allmp4_notitle.txt','a+')
    try:
        #获取福利片所有页面索引
        pageindex=getFuliPageIndex()
        #遍历所有页面
        for page in pageindex:
            print page
            #获取当前页面所有视频指向地址
            curr_urls=getFuliCurrPageUrls(page)
            #遍历当前页所有视频播发器地址
            for url in curr_urls:
                #获取当前视频播发器地址
                player_url=getFuliPlayer(url)
                #获取实际视频地址
                mp4title,mp4_url=getFuliMp4Url(player_url)
                print mp4title,mp4_url
                fd.write(mp4title)
                fd.write('|'+mp4_url+'\n')
                fd_notitle.write(mp4_url+'\n')

    except Exception,err:
        print err
# continue
    fd.close()
    fd_notitle.close()

得到地址后使用迅雷打包下载，比较快速。