前提

如果爬取解析的数据不在同一张页面中。（深度爬取）

实战

使用古诗词网站进行模拟

import scrapy
from bossPro.items import BossproItem
class BossSpider(scrapy.Spider):
    name = 'boss'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.shicimingju.com/category/all']

    url = 'https://www.shicimingju.com/chaxun/zuozhe/1_%d.html'
    page_num = 2

    # 回调函数接受item
    def parse_detail(self, response):
        item = response.meta['item']

        detail = response.xpath('//*[@id="main_right"]/div[1]/div[2]/div[1]/div/text()').extract()
        detail = ''.join(detail)
        # print(job_desc)
        item['detail'] = detail
        yield item

    #数据解析处理
    def parse(self, response):
        list_data = response.xpath('//*[@id="main_left"]/div')
        for li in list_data:
            name = li.xpath('./div[@class="zuozhe_list_item"]/h3/a/text()').extract_first()
            detail_url = li.xpath('./div[@class="zuozhe_list_item"]/h3/a/@href').extract_first()
            detail_url = 'https://www.shicimingju.com' + str(detail_url) #有空值需要处理下，如果没有空值可以不用str

            item = BossproItem()
            item['name'] = name
            # 对详情页发请求获取详情页的页面源码数据
            # 手动请求的发送
            # 请求传参：meta={}，可以将meta字典传递给请求对应的回调函数
            yield scrapy.Request(detail_url, callback=self.parse_detail, meta={'item': item})

        #分页操作
        if self.page_num <= 3:
            new_url = format(self.url%self.page_num)
            self.page_num += 1
            yield scrapy.Request(new_url,callback=self.parse)

标签：传参,请求,url,detail,item,scrapy,div,response,name
来源： https://www.cnblogs.com/simon1993/p/16307636.html