批量爬取贴吧图片 糗事百科 煎蛋网

批量爬取贴吧图片
from urllib import request
import re
#                                   %e5%9b%be%e7%89%87
url = "http://tieba.baidu.com/f?kw=%E6%91%84%E5%BD%B1%E5%90%A7"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
}

req = request.Request(url = url,headers=headers)

response = request.urlopen(req)

html = response.read().decode("utf-8")

img_link = re.findall(r'<img src="(.*?)"',html)

# print(img_link)
# str.startswith()

for link in img_link:
    # 判断链接是否是以http开头的
    if link.startswith("http"):
        print("开始爬取:%s"%link)
        request.urlretrieve(url = link,filename = '../images/'+link[-10:])
    else:
        pass


糗事百科

from urllib import request
import re
url = "https://www.qiushibaike.com/pic/page/%s/"
headers = {
    "User-Agent"": ""Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"",
}
for i in range(1,11):
    req = request.Request(url=url%i,headers=headers)
    response = request.urlopen(req)
    html = response.read().decode("utf-8")
    img_link = re.findall(r'<img src="(.*?)"',html)
    for link in img_link:
        if link.startswith("//pic"):
            links="http:"+link
            try:
                request.urlretrieve(url=links,filename="./images/"+links[-10:])
            except:
                pass
        else:
            print(link,"图片路径不正确")


煎蛋网

from urllib import request
import re
url = "http://jandan.net/pic/page-127"
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Cookie": "_ga=GA1.2.1246691030.1543560067; _gid=GA1.2.1877436102.1559200246",
    "Host": "jandan.net",
    "Pragma": "no-cache",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
}
req = request.Request(url=url,headers=headers)
response = request.urlopen(req)
html = response.read().decode("utf-8")
link = re.findall(r'<img .* org_src="(.*?)"',html)
for var in link:
    links="http:"+var
    print("正在下载：%s"%(links))
    request.urlretrieve(links,"./image/"+links[-10:])

标签：煎蛋,links,url,百科,req,request,爬取,headers,link
来源： https://www.cnblogs.com/wyf2019/p/10951937.html