scrapy-splash

官网：https://github.com/scrapy-plugins/scrapy-splash

1、安装：

pip install scrapy-splash

2、运行splash

docker run -p 8050:8050 scrapinghub/splash

3、配置setting文件

DOWNLOADER_MIDDLEWARES = {
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}

SPIDER_MIDDLEWARES = {
    'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}

SPLASH_URL = 'http://10.63.32.49:8050/'

# 过滤
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'

# 缓存
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

标签：8050,DOWNLOADER,scrapy,splash,32.49,MIDDLEWARES
来源： https://www.cnblogs.com/wt7018/p/11897162.html