python - 在scrapy中使用自定义名称保存下载的文件

Question

我是scrapy的新手。我使用下面的代码下载了一些文件。我想更改下载文件的名称，但我不知道怎么做。

例如，我想要一个包含名称的列表，并使用它来重命名我下载的文件。

任何帮助将不胜感激

我的蜘蛛

import scrapy from scrapy.loader 
import ItemLoader from demo_downloader.items 
import DemoDownloaderItem

class FileDownloader(scrapy.Spider):
    name = "file_downloader"
    def start_requests(self):
        urls = [
            "https://www.data.gouv.fr/en/datasets/bases-de-donnees-annuelles-des-accidents-corporels-de-la-circulation-routiere-annees-de-2005-a-2019/#_"
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        for link in response.xpath('//article[@class = "card resource-card "]'):
            name = link.xpath('.//h4[@class="ellipsis"]/text()').extract_first()
            if ".csv" in name:
                loader = ItemLoader(item=DemoDownloaderItem(), selector=link)
                absolute_url = link.xpath(".//a[@class = 'btn btn-sm btn-primary']//@href").extract_first()
                loader.add_value("file_urls", absolute_url)
                loader.add_value("files", name)
                yield loader.load_item()

项目.py

from scrapy.item import Field, Item

class DemoDownloaderItem(Item):
    file_urls = Field()
    files = Field()

管道.py

from itemadapter import ItemAdapter


class DemoDownloaderPipeline:
    def process_item(self, item, spider):
        return item

设置.py

BOT_NAME = 'demo_downloader'
SPIDER_MODULES = ['demo_downloader.spiders']
NEWSPIDER_MODULE = 'demo_downloader.spiders'
ROBOTSTXT_OBEY = False
ITEM_PIPELINES = {
   'scrapy.pipelines.files.FilesPipeline': 1
}
DOWNLOAD_TIMEOUT = 1200
FILES_STORE = "C:\\Users\\EL\\Desktop\\work\\demo_downloader"
MEDIA_ALLOW_REDIRECTS = True

标签： pythonweb-scrapingscrapyscrapy-pipeline