分类: Python/Ruby
2013-11-11 10:26:00
#coding:utf8
#!/usr/bin/env python
import threading
import httplib
import random
import time
import os
import sys
import urllib2
from Queue import Queue
#
'''
class :RequestThread 请求URL
func :request_url(self) 创建各个线程任务
'''
class RequestThread(threading.Thread):
def __init__(self,thread_name):
threading.Thread.__init__(self)
self.error_num = 0
self.url_num = 0
self.thread_name = thread_name
def run(self):
print("%s---time is %s "%(threading.currentThread(),time.ctime(),))
time.sleep(10)
self.request_url()
#单个线程的任务
def request_url(self):
try:
file = open('urls.txt','r')
for line in file.readlines():
self.url_num +=1
url = line.strip()
html = urllib2.urlopen(url).read()
except urllib2.URLError,e:
print e
self.error_num +=1
''' func: start_thread(thread_num) 生成多个处理线程 int thread_num :传入并发的线程个数 return: time_total 多个并发进程完成一次循环所用时间 error_num 请求失败的页面数 url_num 请求的url数 ''' def start_thread(thread_num): time_start = time.time() threads = [] for i in range(thread_num): thread = RequestThread('thread'+str(i)) thread.setDaemon(True) threads.append(thread) for t in threads: t.start() for t in threads: t.join() time_end = time.time() time_total = time_end - time_start print "每个循环耗时%s秒"%(time_total) #获取类属性 error_num = thread.error_num url_num = thread.url_num return time_total,error_num,url_num''' func: work(work_num) 每个线程做同一个任务循环的次数 int work_num: 任务循环次数 int thread_num:需要并发的进程数 ''' def work(work_num,thread_num): time_all = 0 i=0 while i < work_num: i +=1 time_total,error_num,url_num = start_thread(thread_num) # print time_total time.sleep(2) time_all = time_all + time_total print "第 %s 次循环结束"%(i) print "*****************************************************" print "总共运行耗时:%f秒"%(time_all) print "错误数:%s"%(error_num) print "总计请求了%s个页面"%(work_num*thread_num*url_num) if __name__ == "__main__": #thread_num,创建并发线程的数量 thread_num = 10 work_num = 2 # start_thread(thread_num) work(work_num,thread_num)