python - 如何将 Web 元素存储在数据框中的不同页面上？

Question

我正在抓取 crunchbase（ev 行业），对于列表中的每家公司，我想点击他们的个人资料存储其创始人的姓名，点击每个姓名并存储该人的性别和教育背景. 我知道它需要一个嵌套循环我只是不知道如何将数据按顺序存储在不同的页面上......这是我到目前为止想出的。

#imports
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import pandas as pd
import time

#driver path
PATH = "C:/Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(PATH)

#access crunchbase ui
driver.get("https://www.crunchbase.com/search/organizations/field/organization.companies/categories/electric-vehicle")
driver.maximize_window()
time.sleep(5)
print(driver.title)

time.sleep(3)
   
#await element location

WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.XPATH, ('//a[@aria-label="Next"][@aria-disabled="false"][@type="button"]'))))
    
#next page
page = driver.find_element_by_xpath('/html/body/chrome/div/mat-sidenav-container/mat-sidenav-content/div/search/page-layout/div/div/form/div[2]/results/div/div/div[1]/div/results-info/h3/a[2]')

company_list = [] ###create dictionary
   
counter = 0
for _ in range(2):
    if counter == 1:
        break
    counter += 1
    
    if page.is_displayed():
        
        time.sleep(25)
        
        #webscrape through iterations
        all_rows = driver.find_elements_by_css_selector("grid-row")
                                                      
        for row in all_rows:
            companyname = row.find_element_by_xpath('.//*[@class="identifier-label"]')
            companyname.click()
            time.sleep(25) 
            company=driver.find_element_by_xpath('.//*[@class="profile-name"]')
            ActionChains(driver).move_to_element(company).perform()
            company_list.append(company)
        print("next")
        page.click()
    
    
#create dataframe    
df = pd.DataFrame(company_list)

print(df)

#create excel writer object
writer=pd.ExcelWriter('crunchbasedemo.xlsx')

#export to excel
df.to_excel(writer)

writer.save()



print("It's alive!")

标签： pythonhtmlpandasseleniumselenium-webdriver