首页 > 解决方案 > Despite using Explicit wait TimeoutException Error

问题描述

I want to scrape data from https://www.dawn.com/business. I want to fetch all div data from site using selenium data from first div fetch and store in document file sucessfully but 4th div cannot be fetched and i recieved Error

File "D:\anconda\lib\site-packages\selenium\webdriver\support\wait.py", line 80, in until raise TimeoutException(message, screen, stacktrace) TimeoutException

I use explicit wait but it dose not work. Kindly look in to code and help me out what wrong with this .

"""
Created on Sun Nov 10 22:05:24 2019

@author: Tanoli
"""

import docx
import time

# selenium
from selenium import webdriver
#keys
from selenium.webdriver.common.keys import Keys
#Explicit wait import
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
#Explicitwait
driver.implicitly_wait(10)

chrome_path= r"C:\Users\Tanoli\AppData\Local\Temp\Rar$DRa8876.13156\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)
doc=docx.Document()
#waiting for page loads

#open in selenium brwoser
driver.get("https://www.dawn.com/business")
#open first div on business page
First_Div_Open=driver.find_element_by_xpath('/html/body/div[2]/div/main/div[1]/div[1]/div[1]/article/h2/a').click()
#Find all paragraph element on detail page    
element=driver.find_elements_by_tag_name('p');
#add heading to paragraph
First_Div_Heading=driver.find_element_by_xpath('/html/body/div[2]/div/div[1]/main/div/div/article/div[1]/h2/a').text
doc.add_heading(First_Div_Heading)
#fecting paragraph on first para
for info in element:
    doc.add_paragraph(info.text)
#saving file
doc.save(First_Div_Heading +'.docx')

#moving back to business page
driver.back()
#wait for 5 sec on page
time.sleep(5)
#Starting div number 2
Second_Div_Open=driver.find_element_by_xpath('/html/body/div[2]/div/main/div[1]/div[1]/div[2]/article[1]/h2/a').click()
time.sleep(5)
#Find all paragraph element on detail page    
element1=driver.find_elements_by_tag_name('p');
#add heading to paragraph
Second_Div_Heading=driver.find_element_by_xpath('/html/body/div[2]/div/div[1]/main/div/div/article/div[1]/h2/a').text
#add heading in file
doc.add_heading(Second_Div_Heading)
#fecting paragraph on first para
for para1 in element1:
    doc.add_paragraph(para1.text)
#saving file
doc.save(Second_Div_Heading +'.docx')
#end of second div

#moving back to business page
driver.back()
#wait for 5 sec on page

time.sleep(10)
#Starting div number 3
Third_Div_Open=driver.find_element_by_xpath('/html/body/div[2]/div/main/div[1]/div[1]/div[2]/article[2]/h2/a').click()
#Find all paragraph element on detail page    
element2=driver.find_elements_by_tag_name('p');
#add heading to paragraph
Third_Div_Heading=driver.find_element_by_xpath('/html/body/div[2]/div/div[1]/main/div/div/article/div[1]/h2/a').text
#add heading in file
doc.add_heading(Third_Div_Heading)
#fecting paragraph on first para
for para2 in element2:
    doc.add_paragraph(para2.text)
#saving file
doc.save(Third_Div_Heading +'.docx')
#end of second div
time.sleep(10)
#moving back to business page
driver.back()
#wait for 5 sec on page
time.sleep(30)
Fourth_Div_Open= WebDriverWait(driver, 60).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[3]/div/main/div[1]/div[1]/div[2]/article[3]/h2/a')))
Fourth_Div_Open.click()
#Starting div number 4
#Fourth_Div_Open=driver.find_element_by_xpath('/html/body/div[3]/div/main/div[1]/div[1]/div[2]/article[4]/h2/a').click()
#Find all paragraph element on detail page    
element3=driver.find_elements_by_tag_name('p');
#add heading to paragraph
Fourth_Div_Heading=driver.find_element_by_xpath('/html/body/div[3]/div/div[1]/main/div/div/article/div[1]/h2/a').text
#add heading in file
doc.add_heading(Fourth_Div_Heading)
#fecting paragraph on first para
for para3 in element3:
    doc.add_paragraph(para3.text)
#saving file
doc.save(Fourth_Div_Heading +'.docx')
#end of second div

标签: selenium

解决方案


推荐阅读