python - 如何使用 Scrapy 更快地制作 Selenium driver.get

Question

下面是一个用于监控某个网站的 python 脚本，一旦某个特定的类（在这个例子中 '.ant-card-body'）发生任何变化，就做一些事情。

但是，driver.get()完全加载网页大约需要 14 秒...

你能帮我用什么样的解决方法来让它更快，立即检测元素变化吗？

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from scrapy.crawler import CrawlerProcess
import scrapy
import time

class BasicSpider(scrapy.Spider):
    name = "basic"
    allowed_domains = [""]
    start_urls = ['https://solanamonkeysclub.com/#/#mint']

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        chrome_options = Options()
        chrome_options.add_argument("--headless")
        self.driver = webdriver.Chrome('chromedriver.exe', options=chrome_options)

    def parse(self, response, **kwargs):
        # constantly get webpage and check if an element has been changed
        while True:
            start = time.time()
            self.driver.get(response.url)
            end = time.time()
            print(end - start)   # takes 14 SECONDS...

            new_elems = self.driver.find_elements_by_css_selector('.ant-card-body')
            for new_elem in new_elems:
                print(f'{"new_elem":10} : {new_elem.text}')
                if new_elem.text != 0:
                    pass

process = CrawlerProcess()
process.crawl(BasicSpider)
process.start()

标签： pythonseleniumweb-scrapingbeautifulsoup