基本上不需要修改原有代码，添加如下代码即可。
注：在settings.py 中需要注释点原有的 USER_AGENT 配置。
############### ##settings.py## ############### ### 配置IP代理池 IPPOOL = [ {"ipaddr":"x.x.x.x:端口"}, {"ipaddr":"x.x.x.x:端口"}, ] ### 配置User-agent 池 UAPOOL = [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)" ] ###配置DOWNLOADER_MIDDLEWARES，注意此处需要修改为自己的爬虫名称 DOWNLOADER_MIDDLEWARES = { '自己的爬虫名称.middlewares.IPPOOLS': 125, '自己的爬虫名称.middlewares.Uamid': 127, } ###################### ##配置 middlewares.py## ######################
import random from .settings import IPPOOL from .settings import UAPOOL from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware ### IP代理池 class IPPOOLS(HttpProxyMiddleware): def __init__(self, ip=""): self.ip = ip def process_request(self, request, spider): thisip = random.choice(IPPOOL) #print("当前使用的IP为： " + thisip["ipaddr"]) request.meta["proxy"] = "http://" + thisip["ipaddr"] ### 用户代理池 class Uamid(UserAgentMiddleware): def __init__(self, user_agent=""): self.user_agent = user_agent def process_request(self, request, spider): thisua = random.choice(UAPOOL) #print("当前使用的User-Agent是： " + thisua) request.headers.setdefault("User-Agent", thisua)