python - 在第 6/7 页后处理 Cookie 弹出窗口

问题描述

在这个网站上的一些会员的帮助下，我已经建立了一个房地产数据的网络抓取。

它工作得很好，但是在爬到第 6/7 页或更长时间之后，弹出一个典型的 cookie 警告的 cookie，并且似乎破坏了我在 CSV 文件中的输出。

有没有办法处理弹出窗口？

from selenium import webdriver
from bs4 import BeautifulSoup
import re
import time
import requests
import pandas as pd
#open('output.csv', 'w').close()

browser = webdriver.Chrome('C:/Users/907133/Pythonstuff/chromedriver')
browser.set_window_position(0,0)

def jaap_spider(max_pages):
    page = 1
    while page <= max_pages:
        url = 'https://www.jaap.nl/koophuizen/noord+holland/groot-amsterdam/amsterdam/p{}'.format(page)
        #browser.delete_all_cookies()
        browser.get(url)
        #session = requests.Session()
        #res1 = session.post(url, post_data)
        #res2 = session.get(url1)
        time.sleep(15)
        #input('Press Enter after bypassing Captcha')
        soup = BeautifulSoup(browser.page_source, 'html.parser')
        info = soup.find_all('div', {'class':'property-info'})
        inside = soup.find_all('a', {'class': 'property-inner'},{'href'})
    # Make empty lists with header lines
    outputlist_l1 = [['street', 'address', 'price', 'pricetag']]
    outputlist_l2 = [['soort', 'bouwjaar', 'woonoppervlakte', 'inhoud', 'perceel']]

    for huis in info:
        street = huis.find('h2')
        street = ' '.join(street.get_text(separator='\r\n', strip=True).split()[:+3])  
        address = huis.find('div')
        address = address.find('div').text.strip()
        price = huis.find('div', {'class': 'price-info'})
        price = price.find('div').text.strip()
        price = re.findall(r'\d', price)
        price = ''.join(price)
        pricetag = huis.find('div', {'class': 'property-price'})
        pricetag = pricetag.find('span').text.strip()
        outputlist_l1.append([street, address, price, pricetag])

    for items in inside:
        #browser.delete_all_cookies()
        href = items.get('href')
        url1 = href.format(page)
        browser.get(url1)
        kenmerken = BeautifulSoup(browser.page_source, 'html.parser')
        details = kenmerken.find_all ('div', {'class':'detail-tab-content kenmerken'})
        try:
            tr = details[0].find_all ('td', {'class': 'value'})
        except IndexError:
            size_space = 'Unknown'

        for inhoud in tr:
            soort = tr[0].get_text(separator='\n', strip=True)
            bouwjaar = tr[1].get_text(separator='\n', strip=True)
            woonoppervlakte = tr[2].get_text(separator='\n', strip=True)
            inhoud = tr[3].get_text(separator='\n', strip=True)
            perceel = tr[4].get_text(separator='\n', strip=True)
        l2 = ('{},{},{},{},{}'.format(soort, bouwjaar, woonoppervlakte, inhoud, perceel))
        outputlist_l2.append([soort, bouwjaar, woonoppervlakte, inhoud, perceel])

    page += 1    
# Merge outputlist_l1 with outputlist_l2
outputlist = [a + b for a, b in zip(outputlist_l1, outputlist_l2)]

# transform to Pandas dataframe and export as csv
#saveFile = open('output.csv', 'a')
df = pd.DataFrame(outputlist[1:], columns=outputlist[0])
df.to_csv('output.csv', index=False)
#saveFile.close()

jaap_spider(15)

网站中的 cookie 脚本：

标签： pythonseleniumbeautifulsoup

python - 在第 6/7 页后处理 Cookie 弹出窗口

问题描述

解决方案

推荐阅读