Scrapy_糗事百科

1.进入桌面

1 cd C:\Users\Mr_wa\Desktop

2.新建项目

scrapy startproject qsbk

3.新建爬虫

cd qsbk

scrapy genspider qsbk_spider qiushibaike.com

4.修改settings.py

1 ROBOTSTXT_OBEY = False
2 
3 DEFAULT_REQUEST_HEADERS = {
4   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
5   'Accept-Language': 'en',
6   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
7 }

5.书写qsbk_spider.py（start_urls）

 1 import scrapy
 2 
 3 
 4 class QsbkSpiderSpider(scrapy.Spider):
 5     name = 'qsbk_spider'
 6     allowed_domains = ['qiushibaike.com']
 7     start_urls = ['https://www.qiushibaike.com/text/page/1/']
 8 
 9     def parse(self, response):
10         pass

6.根目录新建运行文件start.py

1 from scrapy import cmdline
2 
3 cmdline.execute(['scrapy', 'crawl', 'qsbk_spider'])

7.爬虫测试（查看response类型）

 1 import scrapy
 2 
 3 
 4 class QsbkSpiderSpider(scrapy.Spider):
 5     name = 'qsbk_spider'
 6     allowed_domains = ['qiushibaike.com']
 7     start_urls = ['https://www.qiushibaike.com/text/page/1/']
 8 
 9     def parse(self, response):
10         print("===================")
11         print(type(response))
12         print("===================")

8.在start.py中运行结果

===================
<class 'scrapy.http.response.html.HtmlResponse'>
===================

8.xpath解析数据

获取作者和段子内容

 1 import scrapy
 2 
 3 
 4 class QsbkSpiderSpider(scrapy.Spider):
 5     name = 'qsbk_spider'
 6     allowed_domains = ['qiushibaike.com']
 7     start_urls = ['https://www.qiushibaike.com/text/page/1/']
 8 
 9     def parse(self, response):
10         # content_list : SelectorList
11         content_list = response.xpath("//div[@class='col1 old-style-col1']/div")
12         # content : selector
13         for content in content_list:
14             author = content.xpath(".//h2/text()").get().strip()
15             text = content.xpath(".//div[@class='content']//text()").getall()
16             text = "".join(text).strip()
17             duanzi = {'author': author, 'content': text}
18             print(duanzi)

9.存储数据

在item.py中定义数据保存的类型，（而不是上面的字典 duanzi = {'author': author, 'content': text}）

1 import scrapy
2 
3 
4 class QsbkItem(scrapy.Item):
5     # define the fields for your item here like:
6     # name = scrapy.Field()
7     author = scrapy.Field()
8     text = scrapy.Field()

qsbk_spider.py

import scrapy
from ..items import QsbkItem


class QsbkSpiderSpider(scrapy.Spider):
    name = 'qsbk_spider'
    allowed_domains = ['qiushibaike.com']
    start_urls = ['https://www.qiushibaike.com/text/page/1/']

    def parse(self, response):
        # content_list : SelectorList
        content_list = response.xpath("//div[@class='col1 old-style-col1']/div")
        # content : selector
        for content in content_list:
            author = content.xpath(".//h2/text()").get().strip()
            text = content.xpath(".//div[@class='content']//text()").getall()
            text = "".join(text).strip()
            item = QsbkItem(author=author, content=text)
            yield item

settings.py 取消注释

ITEM_PIPELINES = {
   'qsbk.pipelines.QsbkPipeline': 300,
}

pipelines.py 保存数据

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import json 

class QsbkPipeline:

    def process_item(self, item, spider):
        with open('qsbk.json', 'a', encoding='utf-8') as f:
            f.write(json.dumps(dict(item), ensure_ascii=False))
        return item

保存为json

爬取多页，并保存为csv格式

qsbk_spider.py

import scrapy
from ..items import QsbkItem


class QsbkSpiderSpider(scrapy.Spider):
    name = 'qsbk_spider'
    allowed_domains = ['qiushibaike.com']
    start_urls = ['https://www.qiushibaike.com/text/page/1/']

    def parse(self, response):
        # content_list : SelectorList
        content_list = response.xpath("//div[@class='col1 old-style-col1']/div")
        # content : selector
        for content in content_list:
            author = content.xpath(".//h2/text()").get().strip()
            text = content.xpath(".//div[@class='content']//text()").getall()
            text = "".join(text).strip()
            item = QsbkItem(author=author, content=text)

            yield item

        next_url = response.xpath("//ul[@class='pagination']/li[last()]/a/@href").get()
        
        if next_url:
            next_url = 'https://www.qiushibaike.com' + next_url
            yield scrapy.Request(next_url, callback=self.parse)

pipelines.py

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import csv 

class QsbkPipeline:
    def process_item(self, item, spider):
        with open('qsbk.csv', 'a', encoding='utf-8', newline="") as f:
            writer = csv.writer(f)
            writer.writerow([item["author"], item["content"]])
        return item

Scrapy_糗事百科

推荐阅读