python - 转到下一页,但它不使用 Selenium 和 Scrapy 抓取其元素
问题描述
我正在尝试使用 Selenium 抓取所有页面并单击下一页按钮。但是,当我转到下一页时,URL 不会改变。我可以移动到所有页面,但我只能从第一页抓取项目,不知道如何让它适用于所有页面。关于我应该做什么的任何建议?
先感谢您!
编码:
class MilieuProperties(scrapy.Spider):
name = 'milieu_properties'
start_urls = [
# FOR SALE
'https://www.milieuproperties.com/search-results.aspx?paramb=ADVANCE%20SEARCH:%20Province%20(Western%20Cape),%20%20Area%20(Cape%20Town)',
'https://www.milieuproperties.com/RentalByCategory.aspx'
]
def __init__(self):
#headless options
options = Options()
options.add_argument('--no-sandbox')
options.add_argument("--headless")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
self.driver = webdriver.Chrome('path',options=options)
def parse(self,response):
self.driver.get(response.url)
current_page_number = self.driver.find_element_by_css_selector('#ContentPlaceHolder1_lvDataPager1>span').text
while True:
try:
elem = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="ContentPlaceHolder1_lvDataPager1"]/a[text()="Next" and not(@class)]')))
elem.click()
except TimeoutException:
break
WebDriverWait(self.driver, 10).until(lambda driver: self.driver.find_element_by_css_selector('#ContentPlaceHolder1_lvDataPager1>span').text != current_page_number)
current_page_number = self.driver.find_element_by_css_selector('#ContentPlaceHolder1_lvDataPager1>span').text
offering = response.css('span#ContentPlaceHolder1_lblbreadcum::text').get()
try:
offering = 'rent' if 'Rental' in offering else 'buy'
except TypeError:
offering = 'buy'
base_link = response.request.url.split('/')
try:
base_link = base_link[0] + '//' + base_link[2] + '/'
except:
pass
for p in response.xpath('//div[@class="ct-itemProducts ct-u-marginBottom30 ct-hover"]'):
link = base_link + p.css('a::attr(href)').get()
yield scrapy.Request(
link,
callback=self.parse_property,
meta={'item': {
'url': link,
'offering': offering,
}},
)
def parse_property(self, response):
item = response.meta.get('item')
. . .
解决方案
您可以在不使用 Scrapy 的情况下获取数据。试试这个代码:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome()
links = []
url = 'https://www.milieuproperties.com/search-results.aspx?paramb=ADVANCE%20SEARCH:%20Province%20(Western%20Cape),%20%20Area%20(Cape%20Town)'
driver.get(url)
current_page_number = driver.find_element_by_css_selector('#ContentPlaceHolder1_lvDataPager1>span').text
while True:
links.extend([link.get_attribute('href') for link in driver.find_elements_by_css_selector('.hoverdetail a')])
try:
elem = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="ContentPlaceHolder1_lvDataPager1"]/a[text()="Next" and not(@class)]')))
elem.click()
except TimeoutException:
break
WebDriverWait(driver, 10).until(lambda driver: driver.find_element_by_css_selector('#ContentPlaceHolder1_lvDataPager1>span').text != current_page_number)
current_page_number = driver.find_element_by_css_selector('#ContentPlaceHolder1_lvDataPager1>span').text
print(links)
推荐阅读
- networking - 如何让传感器/海龟运行功能优先考虑当前最高能量的传感器/海龟?网标
- nestjs - 如何使用 { scope: Scope.REQUEST } 对服务上的每个请求调用方法?
- python - 熊猫:无法根据其他列的条件设置单元格值
- php - 在循环中渲染树枝组件
- c++ - Code Chef DSA 学习系列 : Multiple of 3 Multhree
- postgresql - 将 hasura 连接到现有的 postgresql
- python - 安装 scikit-image 时如何解决此错误?
- postgresql - On any of my Heroku apps, several `heroku pg:` commands fail immediately with "Cannot read property 'includes' of undefined."
- php - change the final number of the permalink by +1 php
- python - Low accuracy for triplet loss in image recognition