python - Python 中的 Selenium 超时异常

Question

我有以下功能：

def get_info(url):
    options = webdriver.ChromeOptions()
    options.headless = True
    chrome_browser = webdriver.Chrome('./chromedriver', chrome_options=options)
    chrome_browser.get(url)
    name = chrome_browser.find_element_by_xpath("//h1[contains(@class,'text-center medium-text-left')]").text
    winter = WebDriverWait(chrome_browser, 10).until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="main-content"]/div[1]/div[1]/div/div[2]/div[5]/div['
                                                  '2]/div/div[1]/div[3]/div')))
    chrome_browser.quit()
    return winter, name

我想从这个网站上的冬季/春季/夏季等图表中获取宽度百分比：https ://www.fragrantica.com/perfume/Christian-Dior/Sauvage-Eau-de-Parfum-48100.html

所以我希望这个函数在 HTML 页面上返回香水的名称和冬季行。季节收视率在网站上加载似乎有点慢，所以我尝试添加等到 HTML 行出现。当我在冬季收视率图表上单击检查时，我得到了这个元素：

<div style="border-radius: 0.2rem; height: 0.3rem; background: rgb(120, 214, 240); width: 90.3491%; opacity: 1;"></div>

首先，BeautifulSoup 没有找到它，所以我尝试了 Selenium。Selenium 没有找到它，并且在使用 WebDriverWait 时它只是向我显示此错误：

Traceback (most recent call last):
  File "D:/Fragrance selector/main.py", line 16, in <module>
    s = info.get_info('https://www.fragrantica.com/perfume/Christian-Dior/Sauvage-Eau-de-Parfum-48100.html')
  File "D:\Fragrance selector\fragrance_info_from_net.py", line 24, in get_info
    winter = WebDriverWait(chrome_browser, 10).until(
  File "D:\Fragrance selector\venv\lib\site-packages\selenium\webdriver\support\wait.py", line 80, in until
    raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message: 

我真的对这个问题没有想法。我对如何从收视率中获得宽度百分比没有更多的想法。如果你们中的一些人能帮助我解决这个问题，我将不胜感激。

标签： pythonhtmlseleniumweb-scraping

Answer 1

我将 WebDriverWait 中的等待时间修改为 30 秒，并添加了一个 get_attribute 以返回正确的数据。请参阅下面的示例：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options

options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--start-maximized')
options.add_argument('--disable-extensions')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--ignore-certificate-errors')
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=options)
driver.get('https://www.fragrantica.com/perfume/Christian-Dior/Sauvage-Eau-de-Parfum-48100.html')
name = driver.find_element(By.XPATH, "//h1[contains(@class,'text-center medium-text-left')]").text
winter = WebDriverWait(driver, 30).until(ec.presence_of_element_located((By.XPATH, '//*[@id="main-content"]/div[1]/div[1]/div/div[2]/div[4]/div[2]/div/div[1]/div[3]/div/div')))
t = winter.get_attribute('style')
driver.quit()

要将所有评级（面孔和季节）放入列表中，请使用以下代码：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options

new = []
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--start-maximized')
options.add_argument('--disable-extensions')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--ignore-certificate-errors')
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=options)
driver.get('https://www.fragrantica.com/perfume/Christian-Dior/Sauvage-Eau-de-Parfum-48100.html')
name = driver.find_element(By.XPATH, "//h1[contains(@class,'text-center medium-text-left')]").text
WebDriverWait(driver, 30).until(ec.presence_of_element_located((By.XPATH, '//*[@id="main-content"]/div[1]/div[1]/div/div[2]/div[4]/div[2]/div/div[1]/div[3]/div/div')))
ratings = driver.find_elements(By.XPATH, './/div[@style="width: 100%; height: 0.3rem; border-radius: 0.2rem; background: rgba(204, 224, 239, 0.4);"]')
for style in ratings:
    new.append(style.find_element(By.TAG_NAME, 'div').get_attribute('style'))


print(new)
driver.quit()

这将给出条形显示的百分比的有序列表（从 Faces 开始，然后移至 Seasons）。