python-3.x - 如何使用 Selenium (Python) 抓取多个页面

Question

我已经看到了几种从网站上抓取多个页面的解决方案，但无法使其在我的代码上运行。

目前，我有这段代码，正在抓取第一页。我想创建一个循环来抓取网站的所有页面（从第 1 页到第 5 页）

import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup


options = Options()
options.add_argument("window-size=1400,600")
from fake_useragent import UserAgent
ua = UserAgent()
a = ua.random
user_agent = ua.random
print(user_agent)
options.add_argument(f'user-agent={user_agent}')

driver = webdriver.Chrome('/Users/raduulea/Documents/chromedriver', options=options)
driver.get('https://www.immoweb.be/fr/recherche/immeuble-de-rapport/a-vendre/liege/4000?page=1')

import time
time.sleep(10)

html = driver.page_source
soup = BeautifulSoup(html, 'html.parser')

results = soup.find_all("div", {"class":"result-xl"})
title=[]
address=[]
price=[]
surface=[]
desc=[]

for result in results:
    title.append(result.find("div", {"class":"title-bar-left"}).get_text().strip())
    address.append(result.find("span", {"result-adress"}).get_text().strip())
    price.append(result.find("div", {"class":"xl-price rangePrice"}).get_text().strip())
    surface.append(result.find("div", {"class":"xl-surface-ch"}).get_text().strip())
    desc.append(result.find("div", {"class":"xl-desc"}).get_text().strip())


df = pd.DataFrame({"Title":title,"Address":address,"Price:":price,"Surface" : surface,"Description":desc})
df.to_csv("output.csv")

标签： python-3.xselenium-webdriverweb-scrapingbeautifulsoup

Answer 1

试试下面的代码。它将遍历所有页面，而不仅仅是 5 个页面。如果可用，请检查下一个按钮，单击它，否则会打破诡计循环。

import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup

options = Options()
options.add_argument("window-size=1400,600")
from fake_useragent import UserAgent

ua = UserAgent()
a = ua.random
user_agent = ua.random
print(user_agent)
options.add_argument(f'user-agent={user_agent}')

driver = webdriver.Chrome('/Users/raduulea/Documents/chromedriver', options=options)

driver.get('https://www.immoweb.be/fr/recherche/immeuble-de-rapport/a-vendre')

import time

time.sleep(10)

Title = []
address = []
price = []
surface = []
desc = []
page=2
while True:
    time.sleep(10)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    results = soup.find_all("div", {"class": "result-xl"})
    for result in results:
        Title.append(result.find("div", {"class": "title-bar-left"}).get_text().strip())
        address.append(result.find("span", {"result-adress"}).get_text().strip())
        price.append(result.find("div", {"class": "xl-price rangePrice"}).get_text().strip())
        surface.append(result.find("div", {"class": "xl-surface-ch"}).get_text().strip())
        desc.append(result.find("div", {"class": "xl-desc"}).get_text().strip())
    if len(driver.find_elements_by_css_selector("a.next")) > 0:
        url = "https://www.immoweb.be/fr/recherche/immeuble-de-rapport/a-vendre/?page={}".format(page)
        driver.get(url)
        page += 1
        #It will traverse for only 5 pages as you are after if want more page just comment the below if block
        if int(page)>5:
        break
    else:
        break

df = pd.DataFrame({"Title": Title, "Address": address, "Price:": price, "Surface": surface, "Description": desc})
df.to_csv("output.csv")