首页 > 其他分享> > 【完成学校的作业】基于白名单的URL分类逻辑的爬虫.py

【完成学校的作业】基于白名单的URL分类逻辑的爬虫.py

2020-12-08 17:01:45 作者：互联网

import requests
from lxml import etree
import time
import csv
import random
import os
print(os.getpid())

#定义一些全局变量

time_begin=time.time()
#定义开始的时间
mobile_whitelist=[
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg1&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg2&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg3&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg4&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg5&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg6&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg7&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg8&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg9&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg10&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg11&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg12&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg13&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg14&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg15&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg16&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg17&from=lianjia_m&searchFrom=jingjiren’,

#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg18&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg19&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg20&from=lianjia_m&searchFrom=jingjiren’
]
web_whitelist=[
‘https://gz.lianjia.com/ershoufang/pg’,
‘https://gz.lianjia.com/ershoufang/pg1’,
‘https://gz.lianjia.com/ershoufang/pg2’,
‘https://gz.lianjia.com/ershoufang/pg3’,
‘https://gz.lianjia.com/ershoufang/pg4’,
‘https://gz.lianjia.com/ershoufang/pg5’,
‘https://gz.lianjia.com/ershoufang/pg6’,
‘https://gz.lianjia.com/ershoufang/pg8’,
‘https://gz.lianjia.com/ershoufang/pg9’,
‘https://gz.lianjia.com/ershoufang/pg10’,
‘https://gz.lianjia.com/ershoufang/pg11’,
‘https://gz.lianjia.com/ershoufang/pg12’,
‘https://gz.lianjia.com/ershoufang/pg13’,
‘https://gz.lianjia.com/ershoufang/pg14’,
‘https://gz.lianjia.com/ershoufang/pg15’,
‘https://gz.lianjia.com/ershoufang/pg16’,
‘https://gz.lianjia.com/ershoufang/pg17’,
‘https://gz.lianjia.com/ershoufang/pg18’,
‘https://gz.lianjia.com/ershoufang/pg19’,
‘https://gz.lianjia.com/ershoufang/pg20’
]

#撰写解析移动端网页的函数

def web_crawl(url_single):
# 定义一些基础且基本不变的变量
header_1 = {
‘User-Agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36’}

# 定义保存为文件的函数,先暂时不保存，看看时间效率
'''
def data_writer(item):
    with open('lianjia_fangjiaxinxi.csv', 'a', encoding='utf-8', newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(item)
        '''

# 定义下载页面的函数
def download(url_single):
    response_1 = requests.get(url_single, headers=header_1)
    time.sleep(3)  # 防止被反爬虫
    return etree.HTML(response_1.text)  # 返回一个Element对象

#下一段代码能从房源列表页面中解析出房源详情页的URL，然后打开这个URL,从中提取房源的其他具体信息
seletor_1 = download(url_single)#调用上面定义的download函数，下载每一页的内容
house_list = seletor_1.xpath(’//[@id=“content”]/div[1]/ul/li’)#提取每一页包含的房源的列表
for house in house_list:
house_layout = house.xpath(‘div[1]/div[3]/div/text()’)[0]
total_price = house.xpath(’//[@id=“content”]/div[1]/ul/li[1]/div[1]/div[6]/div[1]/span/text()’)[0]
apartment = house.xpath(‘div[1]/div[1]/a/text()’)[0]
# 接着进入每一个房源的详情页面 div[1]/div[1]/a/@href
deep_url = house.xpath(‘div[1]/div[1]/a/@href’)[0]
# 然后再次将这个详情页面的内容下载下来进行数据提取
seletor_2 = download(deep_url)
time.sleep(3)
housekeeper = seletor_2.xpath(’//*[@id=“zuanzhan”]/div[2]/div/div[1]/a/text()’)[0]
item = [housekeeper, house_layout, apartment, total_price]
print(‘正在抓取’ + apartment)
print(item)
#data_writer(item)

#爬取移动版链家网页的经纪人的网址（XHR类型的网址）
#https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg99&from=lianjia_m&searchFrom=jingjiren
#99所在的位置会边，下限1，上限100
def mobile_crawl(url_single):
‘’’
proxies = [{‘http’: ‘123.163.117.243:9999’},
{‘http’: ‘175.42.158.122:9999’},
{‘http’: ‘182.87.45.215:9000’},
{‘http’: ‘175.43.151.247:9999’},
{‘http’:‘60.167.112.150:1133’},
{‘http’:‘218.64.184.219:9000’},
{‘http’:‘58.22.177.217:9999’},
{‘https’:‘183.165.11.155:3000’},
{‘http’:‘114.239.149.34:9999’}]
‘’’

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}

response = requests.get(url_single, proxies=random.choice(proxies), headers=headers)

response = requests.get(url_single,headers=headers)
time.sleep(3)
sel = etree.HTML(response.text)
print(sel)

#定义全部的url:移动端和网页端的URL各20条，其实就是上面两个列表的相加

def main():
all_url_list = [
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg1&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg2&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg3&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg4&from=lianjia_m&searchFrom=jingjiren’,
‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg5&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg6&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg7&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg8&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg9&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg10&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg11&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg12&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg13&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg14&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg15&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg16&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg17&from=lianjia_m&searchFrom=jingjiren’,
#‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg18&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg19&from=lianjia_m&searchFrom=jingjiren’,
# ‘https://m.lianjia.com/liverpool/api/jingjiren/getList?cityId=110000&condition=%252Fao12pg20&from=lianjia_m&searchFrom=jingjiren’,
#‘https://gz.lianjia.com/ershoufang/pg’,
#‘https://gz.lianjia.com/ershoufang/pg1’,
#‘https://gz.lianjia.com/ershoufang/pg2’,
#‘https://gz.lianjia.com/ershoufang/pg3’,
# ‘https://gz.lianjia.com/ershoufang/pg4’,
# ‘https://gz.lianjia.com/ershoufang/pg5’,
# ‘https://gz.lianjia.com/ershoufang/pg6’,
# ‘https://gz.lianjia.com/ershoufang/pg8’,
# ‘https://gz.lianjia.com/ershoufang/pg9’,
# ‘https://gz.lianjia.com/ershoufang/pg10’,
#‘https://gz.lianjia.com/ershoufang/pg11’,
# ‘https://gz.lianjia.com/ershoufang/pg12’,
# ‘https://gz.lianjia.com/ershoufang/pg13’,
# ‘https://gz.lianjia.com/ershoufang/pg14’,
# ‘https://gz.lianjia.com/ershoufang/pg15’,
‘https://gz.lianjia.com/ershoufang/pg16’,
‘https://gz.lianjia.com/ershoufang/pg17’,
‘https://gz.lianjia.com/ershoufang/pg18’,
‘https://gz.lianjia.com/ershoufang/pg19’,
‘https://gz.lianjia.com/ershoufang/pg20’]

for url_single in all_url_list:
    if url_single in web_whitelist:
        web_crawl(url_single)
    elif url_single in mobile_whitelist:
        mobile_crawl(url_single)

main()
#定义程序结束的时间
time_end=time.time()
time_consuming=time_end-time_begin
print(‘采用白名单进行URL分类的爬虫程序一共消耗时间为：’+str(time_consuming))

标签：110000,jingjiren,URL,py,cityId,https,白名单,com,lianjia
来源： https://blog.csdn.net/weixin_44452160/article/details/110876903