多线程糗事百科案例，直接可用，注释明确

port threading#线程模块
import queue#用来队列
import requests#便捷性优于urllib模块
import time
from lxml import etree#数据清洗

#1采集网页线程
class Thread1(threading.Thread):
	def __init__(self,threadName,pageQueue,dataQueue):#初始化
		threading.Thread.__init__(self)#调用threading.Thread
		self.threadName = threadName#线程名
		self.pageQueue=pageQueue#网页队列
		self.dataQueue=dataQueue#数据队列
		self.headers = {"User-Agent" : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}

	def run(self):
		print("启动线程"+self.threadName)
		while not flag1:
			try:
				page=self.pageQueue.get()
				url="https://www.qiushibaike.com/8hr/page/"+str(page)+"/"
				content=requests.get(url,headers=self.headers).text#获取文本形式
				time.sleep(0.5)
				self.dataQueue.put(content)#将数据放入数据队列中
			except Exception as e:
				pass
		print("结束线程"+self.threadName)
#2解析网页线程
class Thread2(threading.Thread):
	def __init__(self,threadName,dataQueue,filename):
		threading.Thread.__init__(self)
		self.threadName = threadName
		self.dataQueue=dataQueue
		self.filename=filename
	def run(self):
		print("启动线程"+self.threadName)
		while not flag2:
			try:
				data1=self.dataQueue.get()
				html=etree.HTML(data1)
				node_list=html.xpath('//div/a[@class="recmd-content"]')
				for node in node_list:
					data=node.text
					self.filename.write(data+"\n")
			except Exception as e:
				pass



		print("结束线程"+self.threadName)
flag1=False#判断页码队列中是否为空
flag2=False#判断数据队列中是否为空


def main():
	pageQueue=queue.Queue(10)
	for i in range(1,11):
		pageQueue.put(i)
	#存放采集结果的数据队列
	dataQueue=queue.Queue()
	#保存到本地的文件
	filename=open(r"C:\Users\AMDS\Desktop\爬虫存储\duanzi.txt","a")
	#启动线程
	t1=Thread1("采集线程",pageQueue,dataQueue)
	t1.start()
	t2=Thread2("解析线程",dataQueue,filename)
	t2.start()
	#当pageQueue为空时，停止采集线程
	while not pageQueue.empty():#如果不为空
		pass
	global flag1
	flag1=True


	#当dataQueue为空时，停止采集线程
	while not dataQueue.empty():#如果不为空
		pass
	global flag2
	flag2=True
	t1.join()
	t2.join()

	filename.close()
	print("结束")
if __name__ == '__main__':
	main()

标签：__,百科,self,threadName,注释,线程,pageQueue,dataQueue,多线程
来源： https://blog.csdn.net/ksueis/article/details/121587074