编程语言
首页 > 编程语言> > Python批量下载壁纸,保存壁纸,flask搭建壁纸网站

Python批量下载壁纸,保存壁纸,flask搭建壁纸网站

作者:互联网

小提示:需更换新的请求头,下载时同目录创建img文件夹存图片

 

1. 获取壁纸数据

"""
思路
1. 请求网页,拿到源代码 ok   requests.get(网址)
2. 将源代码解析成网页        lxml的etree.HTML(源代码)
3. 从网页中提取数据         HTML.xpath('复制来的xpath/text()')
4. 翻页,全站数据爬取       翻页一般修改url就可以实现
5. 数据保存
"""
from pprint import pprint
import csv

"""
1. 获取小图片页面的大图页面的地址
2. 将域名拼接上去,构成大图片的页面地址
3. 请求大图片的地址,将大图片的网址保存
4. 下载图片
"""

import requests
from lxml import etree

baseUrl = 'http://www.netbian.com'

# 伪装成浏览器  需要更换请求头
cookies = {
    '__yjs_duid': '1_33e223172d0308c509f12b4f304f2d491651476976719',
    'yjs_js_security_passport': 'f75d121a02abe7650a974a503fabb1b5f24977f8_1652958780_js',
    'Hm_lvt_0f461eb489c245a31c209d36e41fcc0f': '1652672709,1652774098,1652870042,1652958781',
    'Hm_lpvt_0f461eb489c245a31c209d36e41fcc0f': '1652958781',
}

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Referer': 'http://www.netbian.com/',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36',
}


def page(url):
    res = requests.get(url, headers=headers, cookies=cookies)

    html = res.text
    HTML = etree.HTML(html)

    # 提取数据 href属性
    for i in range(1, 21):
        if i == 3:
            continue

        href = HTML.xpath('//*[@id="main"]/div[3]/ul/li[{}]/a/@href'.format(i))

        # 提取小图片地址
        small = HTML.xpath('//*[@id="main"]/div[3]/ul/li[{}]/a/img/@src'.format(i))

        # 将域名拼接上去
        detail_url = baseUrl + href[0]
        # print(detail_url)

        # 请求详情页,拿大图地址
        detail = requests.get(detail_url, headers=headers, cookies=cookies)
        detail.encoding = "gbk"
        detail_html = detail.text
        detail_HTML = etree.HTML(detail_html)

        # 提取图片
        big = detail_HTML.xpath('//*[@id="main"]/div[3]/div/p/a/img/@src')

        # 提取图片标题
        title = detail_HTML.xpath('//*[@id="main"]/div[3]/div/p/a/img/@title')
        print(title, small, big)
        data.append(
            {"title": title[0], "small": small[0], "big": big[0], "category": category}
        )


if __name__ == '__main__':
    category = "动漫"
    data = []
    for p in range(2, 5):
        main_url = "http://www.netbian.com/dongman/"
        url = main_url + 'index_{}.htm'.format(p)
        page(url)

    pprint(data)

    # 保存数据
    # 1. 创建表头
    header_list = ["title", "small", "big", "category"]
    # 打开文件
    with open("img_data_a.csv", 'w', encoding="utf-8-sig", newline="") as f:
        # 创建csv的写对象
        writer = csv.DictWriter(f, header_list)
        # 写入表头(a模式 第一次爬的需要写入表头,第二次不需要表头,请把写入表头注释了)
        writer.writeheader()
        # 写入数据
        writer.writerows(data)
获取数据

2. 批量下载

# 导入 csv 库
import csv
import requests
import time

# 伪装成浏览器  需要更换请求头
cookies = {
    '__yjs_duid': '1_33e223172d0308c509f12b4f304f2d491651476976719',
    'yjs_js_security_passport': 'f75d121a02abe7650a974a503fabb1b5f24977f8_1652958780_js',
    'Hm_lvt_0f461eb489c245a31c209d36e41fcc0f': '1652672709,1652774098,1652870042,1652958781',
    'Hm_lpvt_0f461eb489c245a31c209d36e41fcc0f': '1652958781',
}

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Referer': 'http://www.netbian.com/',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36',
}


def progressbar(url, filepath='./必须加上扩展名'):
    start = time.time()  # 下载开始时间
    response = requests.get(url, stream=True, headers=headers, cookies=cookies)  # stream=True必须写上
    size = 0  # 初始化已下载大小
    chunk_size = 1024  # 每次下载的数据大小
    content_size = int(response.headers['content-length'])  # 下载文件总大小
    try:
        if response.status_code == 200:  # 判断是否响应成功
            print('开始下载,[文件大小]:{size:.2f} MB'.format(
                size=content_size / chunk_size / 1024))  # 开始下载,显示下载文件大小
            # filepath = '下载/222.mp4'  #注:必须加上扩展名
            with open(filepath, 'wb') as file:  # 显示进度条
                for data in response.iter_content(chunk_size=chunk_size):
                    file.write(data)
                    size += len(data)
                    print('\r' + '[下载进度]:%s%.2f%%' % (
                        '>' * int(size * 50 / content_size), float(size / content_size * 100)), end=' ')
        end = time.time()  # 下载结束时间
        print('完成!用时: %.2f秒' % (end - start))  # 输出下载用时时间
    except Exception:
        pass


# 打开文件
with open("img_data_a.csv", encoding="utf-8-sig", mode="r") as f:
    # 基于打开的文件,创建csv.DictReader实例
    reader = csv.DictReader(f)

    # 输出信息
    for row in reader:
        title = row.get("title")
        big_url = row.get("big")
        print("下载:", title)
        # 下载
        # res = requests.get(big_url, headers=headers, cookies=cookies)
        #
        # with open("img/{}.jpg".format(title), 'wb') as f:
        #     f.write(res.content)

        # 用进度条下载  需要当前目录创建img文件夹存放下载的图片
        u = big_url
        progressbar(url=u, filepath="img/{}.jpg".format(title))
下载图片

3. 线程池批量获取全站壁纸地址

import requests
from lxml import etree
from concurrent.futures import ThreadPoolExecutor
import csv

baseUrl = 'http://www.netbian.com'

# 伪装成浏览器 需要更换

cookies = {
    '__yjs_duid': '1_33e223172d0308c509f12b4f304f2d491651476976719',
    'Hm_lvt_0f461eb489c245a31c209d36e41fcc0f': '1652672709,1652774098,1652870042',
    'trenvecookieclassrecord': '^%^2C4^%^2C',
    'trenvecookieinforecord': '^%^2C4-14978^%^2C',
    'yjs_js_security_passport': '9376682f8d181fc1c094828cbcf9858097ffe69e_1652876557_js',
    'Hm_lpvt_0f461eb489c245a31c209d36e41fcc0f': '1652876558',
}

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Referer': 'http://www.netbian.com/fengjing/index_3.htm',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36',
}


def home_page(url):
    res = requests.get(url, headers=headers, cookies=cookies)
    html = res.text
    HTML = etree.HTML(html)
    lis = HTML.xpath('//*[@id="main"]/div[3]/ul//li')
    # print(lis)
    for i in range(1, len(lis)):
        # 提取数据 href属性
        if i == 3:
            continue

        href = HTML.xpath('//*[@id="main"]/div[3]/ul/li[1]/a/@href')
        small = HTML.xpath('//*[@id="main"]/div[3]/ul/li[4]/a/img/@src')[0]
        # print(href)
        # 将域名拼接上去
        detail_url = baseUrl + href[0]
        print(detail_url)
        detail_date = {'small': small}

        detail(detail_url, detail_date)


def detail(detail_url, detail_date):
    res = requests.get(detail_url, headers=headers, cookies=cookies)
    res.encoding = "gbk"
    html = res.text
    HTML = etree.HTML(html)
    img_url = HTML.xpath('//*[@id="main"]/div[3]/div/p/a/img/@src')[0]
    title = HTML.xpath('//*[@id="main"]/div[3]/div/p/a/img/@title')[0]
    # print(img_url)
    detail_date["title"] = title
    detail_date["big"] = img_url
    detail_date['category'] = current
    print(current)
    print(title)
    data.append(detail_date)


def job(url):
    print(url)
    for page in range(1, 10000):
        try:
            if page == 1:
                home_page(url)
            else:
                url_next = url + 'index_{}.htm'.format(page)
                home_page(url_next)
        except:
            print("没有了")
            return


def save():
    # 表头
    header_list = ["title", "category", "small", 'big']
    with open("img_data_all.csv", 'w', encoding="utf-8") as f:
        writer = csv.DictWriter(f, header_list)
        writer.writeheader()

        # 写入数据
        writer.writerows(data)


if __name__ == '__main__':
    category = [
        "/rili/",
        "/dongman/",
        "/fengjing/",
        "/meinv/",
        "/youxi/",
        "/yingshi/",
        "/dongtai/",
        "/weimei/",
        "/sheji/",
        "/keai/",
        "/qiche/",
        "/huahui/",
        "/dongwu/",
        "/jieri/",
        "/renwu/",
        "/meishi/",
        "/shuiguo/",
        "/jianzhu/",
        "/tiyu/",
        "/junshi/",
        "/feizhuliu/",
        "/qita/",
        "/s/wangzherongyao/",
        "/s/huyan/",
        "/s/lol/", ]
    data = []
    pool = ThreadPoolExecutor(50)
    for lei in category:
        current = lei
        url = baseUrl + lei
        # job(url)
        pool.submit(job, url)
    pool.shutdown()
    print("-- 爬取结束 --".center(20, "*"))
    print("开始写入")
    save()
    print("写入完成")
全站/多线程

4. 存数据库

 

5. 搭建壁纸网站

标签:img,title,Python,detail,url,HTML,flask,print,壁纸
来源: https://www.cnblogs.com/zwnsyw/p/16295765.html