web-scraping - Global ItemLoader - 在多个蜘蛛之间共享
问题描述
我对 Scrapy/Python 还很陌生。
我想抓取几个网站,但我只会从每个网站“日期”“cota”和“名称”中获取三个项目,这些项目每天更新并且始终具有相同的 xpath
刮完所有这些后,我想导出到 csv 文件,但是使用我的代码,我得到以下格式
但我想要这样的东西
我特别询问了如何在多个蜘蛛之间共享同一个 ItemLoader,因为这是我想到的,但我对其他选择持开放态度。
这是我目前为两个网站编写的脚本,稍后我将添加更多蜘蛛:
顺便说一句,使用这样的代码,鉴于scrapy是异步的,是否有可能将值混淆?
# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.loader import ItemLoader
class fundo(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
name = scrapy.Field()
cota = scrapy.Field()
date = scrapy.Field()
class ModalSpider(scrapy.Spider):
name = 'modal'
allowed_domains = ['modalasset.com.br']
start_urls = ['http://modalasset.com.br/cotas-diarias/']
def parse(self, response):
l = ItemLoader(item=fundo(),response=response)
name = response.xpath("//tr[@class='row-6 even']/td/a/text()").extract_first()
date = response.xpath("//tr[@class='row-6 even']/td/text()")[0].extract()
cota = response.xpath("//tr[@class='row-6 even']/td/text()")[1].extract()
l.add_value('name', name)
l.add_value('date', date)
l.add_value('cota', cota)
return l.load_item()
class KapitaloSpider(scrapy.Spider):
name = 'kapitalo'
allowed_domains = ['kapitalo.com.br/relatorios.']
start_urls = ['http://kapitalo.com.br/relatorios.html']
def parse(self, response):
l = ItemLoader(item=fundo(),response=response)
name = response.xpath("//tr[@class='odd']")[1].xpath("td//text()")[0].extract()
date = response.xpath("//*[@class='event layout_full block bygone']/h2/text()")[0].extract()
date = date.replace(' Cotas do Dia: ','')
cota = response.xpath("//tr[@class='odd']")[1].xpath("td//text()")[1].extract()
l.add_value('name', name)
l.add_value('date', date)
l.add_value('cota', cota)
return l.load_item()
process = CrawlerProcess({
'FEED_FORMAT': 'csv',
'FEED_URI': 'result.csv'
})
process.crawl(ModalSpider)
process.crawl(KapitaloSpider)
process.start() # the script will block here until all crawling jobs are finished
我尝试的另一种方法是使用以下代码,但 add_value 替换了 ItemLoader 中的旧值,无法弄清楚原因。所以它只返回上一个网站的值。我宁愿使用第一个代码,因为它允许我使用不同种类的蜘蛛,并且对于其中一个网站,我可能需要使用 Selenium。
# -*- coding: utf-8 -*-
import scrapy
from scrapy.loader import ItemLoader
from scrapy.http import Request
class FundoItem(scrapy.Item):
name = scrapy.Field()
date = scrapy.Field()
cota = scrapy.Field()
class RankingSpider(scrapy.Spider):
name = 'Ranking'
allowed_domains = ['modalasset.com.br',
'kapitalo.com.br'
]
start_urls = ['http://modalasset.com.br/cotas-diarias/']
def parse(self, response):
l = ItemLoader(item=FundoItem(),response=response)
name = response.xpath("//tr[@class='row-6 even']/td/a/text()").extract_first()
date = response.xpath("//tr[@class='row-6 even']/td/text()")[0].extract()
cota = response.xpath("//tr[@class='row-6 even']/td/text()")[1].extract()
#item['name'] = name
#item['date'] = date
#item['cota'] = cota
l.add_value('name', name)
l.add_value('date', date)
l.add_value('cota', cota)
yield Request(url = "http://kapitalo.com.br/relatorios.html",
callback = self.parse_2,
meta={'item':l.load_item()})
def parse_2 (self,response):
name = response.xpath("//tr[@class='odd']")[1].xpath("td//text()")[0].extract()
date = response.xpath("//*[@class='event layout_full block bygone']/h2/text()")[0].extract()
date = date.replace(' Cotas do Dia: ','')
cota = response.xpath("//tr[@class='odd']")[1].xpath("td//text()")[1].extract()
l = ItemLoader(item=response.meta['item'])
l.add_value('name', name)
l.add_value('date', date)
l.add_value('cota', cota)
return l.load_item()
解决方案
看起来这是一个问题,因为 Windows 如何解释 csv 文件中的换行符。我通过使用以下代码而不是 FEED_EXPORT 解决了问题:
import csv
with open('test.csv',mode='a',newline='\n') as csv_file:
writer = csv.writer(csv_file)
writer.writerow([name,date,cota,url])
解决了newline = '\n'
问题。
最终代码变成了:
#Scrapy framework
#CrawlerProcess to run multiple spiders
#csv to export
#sys,inspect to find all the classes(spiders) in this script
import scrapy
from scrapy.crawler import CrawlerProcess
import csv
import sys, inspect
import datetime
#SPIDER DEFINITIONS
class ModalSpider(scrapy.Spider):
name = 'modal'
allowed_domains = ['modalasset.com.br']
start_urls = ['http://modalasset.com.br/cotas-diarias/']
def parse(self, response):
name = response.xpath("//tr[@class='row-6 even']/td/a/text()").extract_first()
date = response.xpath("//tr[@class='row-6 even']/td/text()")[0].extract()
cota = response.xpath("//tr[@class='row-6 even']/td/text()")[2].extract()
write(name,date,cota,response.request.url)
class KapitaloSpider(scrapy.Spider):
name = 'kapitalo'
allowed_domains = ['kapitalo.com.br/relatorios.']
start_urls = ['http://kapitalo.com.br/relatorios.html']
def parse(self, response):
#Zeta FIQ FIM
name = response.xpath("//tr[@class='odd']")[1].xpath("td//text()")[0].extract()
date = response.xpath("//*[@class='event layout_full block bygone']/h2/text()")[0].extract()
date = date.replace(' Cotas do Dia: ','')
date = date.replace('.','/')
cota = response.xpath("//tr[@class='odd']")[1].xpath("td//text()")[1].extract()
write(name,date,cota,response.request.url)
#Kappa FIN FIQ FIM
name = response.xpath("//tr[@class='odd']")[0].xpath("td//text()")[0].extract()
cota = response.xpath("//tr[@class='odd']")[0].xpath("td//text()")[1].extract()
write(name,date,cota,response.request.url)
#write to csv file
#newline='\n' so it won't jump any lines between the entries
def write(name,date,cota,url):
with open('test.csv',mode='a',newline='\n') as csv_file:
writer = csv.writer(csv_file)
writer.writerow([name,date,cota,url])
def crawl():
#create the columns in the csv file
with open('test.csv',mode="w",newline='\n') as csv_file:
writer = csv.writer(csv_file)
writer.writerow(['Nome do Fundo','Data','Cota do dia','URL'])
#get all the members from the script
#if it's a class and it's inside the main (it's a spider)
#then it crawls
process = CrawlerProcess()
for name, obj in inspect.getmembers(sys.modules[__name__]):
if inspect.isclass(obj):
if obj.__module__ == '__main__':
process.crawl(obj)
process.start()
crawl()
推荐阅读
- ruby-on-rails - 由于 LoadError: libpcap.so.0.8 导致创建数据库的 docker 问题
- python-2.7 - 如何实现下一页和上一页逻辑
- bash - 如何在 x 循环 x 秒后暂停循环
- powershell - PowerShell 脚本在粘贴到 PowerShell 窗口时运行,但在从快捷方式运行时不运行
- css - Rails 字体未在移动设备上加载但在桌面上显示
- php - 使用 php 中的标头函数回显多个错误和成功消息
- c# - WPF Button Style Trigger fired twice
- laravel - PHP Laravel 5 CLI 终止信号覆盖
- node.js - 在没有用户登录 react.js 和 redux 的情况下将商品添加到购物车
- spring - 超时后重新启动步骤(或作业)