天堂图片全部图片爬取

#1.导包
import requests,os
from lxml import etree
from urllib.request import urlretrieve
from urllib.parse import urljoin
import zmail
import zipfile
headers = {'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
               'Accept - Encoding':'gzip, deflate',
               'Accept-Language':'zh-Hans-CN, zh-Hans; q=0.5',
               'Connection':'Keep-Alive',
               'Host':'zhannei.baidu.com',
               'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'}

#2.获取网页源码
url = 'http://www.ivsky.com/tupian/'
response = requests.get(url,headers = headers)
html_content = response.content
#3.解析大分类标题和地址
root = etree.HTML(html_content)
big_hrefs = root.xpath('//ul[@class="tpmenu"]/li/a/@href')#标签后面属性的值
big_titles = root.xpath('//ul[@class="tpmenu"]/li/a/text()')

for big_title,big_href in zip(big_titles,big_hrefs):
   os.makedirs(f'天堂图片/{big_title}',exist_ok=True)
   big_href = urljoin(url,big_href)

   response = requests.get(big_href,headers = headers)
   html_content = response.content
   root = etree.HTML(html_content)
   small_hrefs = root.xpath('//div[@class="sline"]/div/a/@href')
   small_titles = root.xpath('//div[@class="sline"]/div/a/text()')

   for small_href,small_title in zip(small_hrefs,small_titles):
      os.makedirs(f'天堂图片/{big_title}/{small_title}',exist_ok=True)
      small_href = urljoin(url,small_href)
      response = requests.get(small_href,headers = headers)
      html_content = response.content
      root = etree.HTML(html_content)
      img_srcs = root.xpath("//ul[@class = 'pli']/li/div/a/img/@src")

      for src in img_srcs:
         urlretrieve(src,f'天堂图片/{big_title}/{small_title}/'+src.split('/')[-1])
         high_src = src.replace('/t/', '/pre/')
         urlretrieve(high_src, f'天堂图片网/{big_title}/{small_title}/' + 'big_' + high_src.split('/')[-1])

标签：content,title,big,爬取,href,small,src,天堂,图片
来源： https://blog.csdn.net/qq_38537101/article/details/86592366