python - 网页抓取交互式内容python

由于元素附在页面上，因此您也可以在不单击按钮的情况下获取位置。

诱导WebDriverWait() 并等待presence_of_all_elements_located() 并跟随css selector

然后迭代这些元素并获取元素的textContent属性并进行一些字符串操作以获取location列表。

代码：

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

chrome_path = r"C:\Users\O63308\Desktop\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)
driver.get("https://www.irishconcrete.ie/members-directory/")

products=WebDriverWait(driver,20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,"span[data-bind='html:$data.title']")))
print("Total Number of products : " +str(len(products)))
resultproduct=[product.text for product in products]
print(resultproduct)
locelements=WebDriverWait(driver,20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,".address+p")))
print("Total Number of locations : " +str(len(locelements)))
resultlocation=[locelement.get_attribute("textContent").splitlines()[0].split("Location:")[-1].strip() for locelement in locelements]
print(resultlocation)

输出：

Total Number of products : 175
['ARDFERT QUARRY PRODUCTS', 'AUSTIN GROGAN & SONS', 'BANAGHER PRECAST CONCRETE', 'BD FLOOD (CAVAN)', 'BD FLOOD (CROOKEDWOOD)', 'BD FLOOD (DRUMLISH)', 'BD FLOOD (KNOCKMANT)', 'BD FLOOD (OLDCASTLE)', 'BD FLOOD (RHODE)', 'BENNETTSBRIDGE LIMESTONE', 'BOGGAN SAND & GRAVEL', 'BOOTH CONCRETE (HEAD OFFICE)', 'BOOTH CONCRETE (PORTLAOISE)', 'BREEDON CEMENT IRELAND*', 'CANNON CONCRETE PRODUCTS', 'CARNAROSS SAND & GRAVEL', 'CARROLL QUARRY LTD', 'CASSIDY BROTHERS (BUNCRANA)', 'CASSIDY BROTHERS (CRANFORD)', 'CASSIDY BROTHERS (LETTERKENNY)', 'CASSIDY CONCRETE', 'CMC QUARRIES (GARRAFRAUNS)', 'CMC QUARRIES (MILLTOWN)', 'CMC QUARRIES (TUAM)', 'CONCAST PRECAST GROUP', 'CONDRON CONCRETE', 'COPPEEN CONCRETE', 'CORCORAN CONCRETE', 'COSHLA QUARRIES', 'DAVID K TROTTER & SONS', 'DOYLE CONCRETE (BUILDER PROVIDERS)', 'DOYLE CONCRETE (HEAD OFFICE)', 'DOYLE CONCRETE (WATERFORD DEPOT)', 'DRIMOLEAGUE CONCRETE WORKS', 'DUCON CONCRETE', 'DUFFY CONCRETE PRODUCTS', 'ESKER READYMIX', 'FARRELL CONCRETE', "FINBARR O'NEILL (HEAD OFFICE)", "FINBARR O'NEILL (OVENS)", 'FLANAGAN CONCRETE PRODUCTS', 'FLOOD PRECAST', 'FOGARTY CONCRETE', 'GLEESON CONCRETE', 'GLEESON PRECAST', 'GLEESON QUARRIES', 'GREANEY CONCRETE (SHANAGOLDEN)', 'GREANEY CONCRETE (TRALEE)', 'HANLON CONCRETE', 'HANLY QUARRIES', 'HARRINGTON CONCRETE & QUARRIES (GALWAY)', 'HARRINGTON CONCRETE & QUARRIES (MAYO)', 'HARRINGTON CONCRETE & QUARRIES (SLIGO)', 'HENNESSY CONCRETE', 'IRISH CEMENT * (DROGHEDA)', 'IRISH CEMENT * (LIMERICK)', 'IRISH INDUSTRIAL EXPLOSIVES *', 'KELLYS OF FANTANE', 'KEOHANE READYMIX (BALLYGURTEEN)', 'KEOHANE READYMIX (INNISHANNON)', 'KERRIGAN QUARRIES', 'KILCARRIG QUARRIES', 'KILDEA CONCRETE', 'KILKENNY BLOCK', 'KILLESHAL PRECAST CONCRETE', 'KILSARAN (BALLINAKILL)', 'KILSARAN (BALLINASCORNEY)', 'KILSARAN (BELLEWSTOWN)', 'KILSARAN (BROWNSTOWN)', 'KILSARAN (CASTLETOWN)', 'KILSARAN (CLONARD)', 'KILSARAN (DULEEK)', 'KILSARAN (ENFIELD)', 'KILSARAN (GALLSTOWN)', 'KILSARAN (HEADQUATERS)', 'KILSARAN (KILCULLEN)', 'KILSARAN (KILFEAKLE)', 'KILSARAN (KILLINICK)', 'KILSARAN (LUCAN)', 'KILSARAN (MILLENIUM PARK)', 'KILSARAN (NAUL)', 'KILSARAN (NAVAN)', 'KILSARAN (PORTLAOISE)', 'KILSARAN (ROSSMORE)', 'KILSARAN (SOUTH BANK ROAD)', 'KILSARAN (TALLAGHT)', 'KILSARAN (THE CURRAGH)', 'KILSARAN (TULLAMORE)', 'KILSARAN (WATERFORD)', 'LAGAN CONCRETE', 'LAGAN MATERIALS', 'LENNON QUARRIES', 'LOUGHNANE CONCRETE', 'LOUGHNANE CONCRETE (NENAGH)', 'MCGRATH CONCRETE PRODUCTS (HERBERSTOWN)', 'MCGRATH CONCRETE PRODUCTS (KILFINNANE)', 'MCGRATH CONCRETE PRODUCTS (O C MILLS)', 'MCGRATHS LIMESTONE (CONG)', 'MCGRATHS LIMESTONE (ROBEEN)', 'MCKEONS SAND & GRAVEL', 'MF QUIRKE & SONS', 'MICHAEL CRONIN READYMIX (CASTLEISLAND)', 'MICHAEL CRONIN READYMIX (KILLARNEY)', 'MICHAEL CRONIN READYMIX (MILLSTREET)', 'MOLLOY CONCRETE', 'MORTIMER QUARRIES', 'MOYLOUGH CONCRETE PRODUCTS', 'MULLAFARRY QUARRY', "O'CONNELL QUARRIES (ARDNACRUSHA)", "O'CONNELL QUARRIES (O'BRIENSBRIDGE)", "O'REILLY CONCRETE (BARLEYSTONE PAVING)", "O'REILLY CONCRETE (KINGSCOURT)", "O'REILLY CONCRETE (TRIM)", "O'ROURKE BROTHERS", 'ORAN PRECAST', 'P MORONEY & SONS', 'PATRICK MCCAFFREY & SONS', 'PLAZAMONT T/A DAN MORRISSEY (BRAY)', 'PLAZAMONT T/A DAN MORRISSEY (CLONMELSH)', 'PLAZAMONT T/A DAN MORRISSEY (RATHDRUM)', 'QUINN BUILDING PRODUCTS', 'QUINN CEMENT *', 'READY MIXED CONCRETE IRELAND', 'RHYNE ROCK', 'ROADSTONE (ALLEN, NAAS)', 'ROADSTONE (ARKLOW)', 'ROADSTONE (BALLINTRA)', 'ROADSTONE (BALLYADAMS)', 'ROADSTONE (BALLYEGAN)', 'ROADSTONE (BALLYGARVAN)', 'ROADSTONE (BALLYKNOCKANE)', 'ROADSTONE (BALLYQUINN)', 'ROADSTONE (BARLEY HILL)', 'ROADSTONE (BELGARD)', 'ROADSTONE (BENNETTSBRIDGE)', 'ROADSTONE (BLESSINGTON)', 'ROADSTONE (BOYLE)', 'ROADSTONE (BRAY)', 'ROADSTONE (BUNRATTY)', 'ROADSTONE (CAM)', 'ROADSTONE (CAPPAGH)', 'ROADSTONE (CARNDONAGH)', 'ROADSTONE (CARRIGTWOHILL)', 'ROADSTONE (CASTLEBAR)', 'ROADSTONE (CASTLEMINE)', 'ROADSTONE (CASTLEMORE)', 'ROADSTONE (CLASSIS)', 'ROADSTONE (DULEEK)', 'ROADSTONE (ENNISCORTHY)', 'ROADSTONE (FOYNES)', 'ROADSTONE (GALWAY)', 'ROADSTONE (HEAD OFFICE)', 'ROADSTONE (HUNTSTOWN)', 'ROADSTONE (KILCHREEST)', 'ROADSTONE (KILLARNEY)', 'ROADSTONE (KILLORGLIN)', 'ROADSTONE (KILLOUGH)', 'ROADSTONE (KILMACOW)', 'ROADSTONE (KILMUCKRIDGE)', 'ROADSTONE (LAGHEY)', 'ROADSTONE (MALLOW)', 'ROADSTONE (MIDLETON)', 'ROADSTONE (MILLSTREET)', 'ROADSTONE (MOYNE)', 'ROADSTONE (MULLAGHCRONE)', 'ROADSTONE (SHANAGOLDEN)', 'ROADSTONE (SLANE)', 'ROADSTONE (SWORDS)', 'ROADSTONE (TULLAMORE)', 'RYAN BROTHERS (ROADSTONE)', 'SHAY MURTAGH PRECAST', 'TECHRETE IRELAND', 'WEXFORD BLOCK', 'WRIGHT CONCRETE PRODUCTS', 'WRIGHT QUARRY PRODUCTS']
Total Number of locations : 175
['Sackville, Ardfert, Co. Kerry', 'Cave, Ballyhaunis, Co. Mayo', 'Queen Street, Banagher, Co.Offaly', 'Lavey, Co. Cavan', 'Crookedwood, Co. Westmeath', 'Drumlish, Co.Longford', 'Knockmant, Co.Westmeath', 'The Murrens, Oldcastle, Co. Meath', 'Rhode, Co.Offaly', 'Kilree, Sheastown, Co.Kilkenny, R95 EY0R', 'Kilmacree, Drinagh, Co.Wexford', 'Ballymullen, Abbeyleix, Co.Laois', 'Clonminham Industrial Estate, Portlaoise, Co.Laois', 'Lansdown Cement Works, Killaskillen, Kinnegad, Co. Meath', 'Tonroe, Oranmore, Co.Galway', 'Carnaross, Kells, Co. Meath', 'Knockcollier, Castletown, Aghafin, Portlaoise, Co.Laois', 'Gransha, Buncrana, Co.Donegal', 'Devlinmore, Cranford, Co.Donegal', 'Magheraboy, Letterkenny, Co.Donegal', 'Moyle Plant, Falcarragh, Co.Donegal', 'Garrafrauns, Dunmore, Co.Galway', 'Grageachullaire, Milltown, Co.Galway', 'Ballygaddy Road, Tuam,\xa0Co.Galway', 'Head Office, Hazelhatch, Newcastle, Co.Dublin', 'Arden Road, Tullamore, Co.Offaly', 'Coppeen, Enniskeane, Co. Cork, P47 E093', 'Ballinrobe Road, Buckwaria, Westport, Co.Mayo', 'Cashla, Athenry, Co.Galway', 'Manorhamilton, Co.Leitrim', 'Castle House (behind Pierce Hire), Cork Road, Co. Waterford', 'Hugginstown, Co.Kilkenny', 'The Old Mart, Old Kilmeadon Road, Co.Waterford', 'Bredagh Cross, Drimoleague, Co.Cork', 'Ballymaquirk, Kanturk, Co.Cork, P51 V024', 'Jenkinstown, Dundalk, Co.Louth', 'Athenry, Co.Galway', 'Stonepark, Endrim, Ferbane, Co.Offaly', 'Poulavone, Ballincollig, Co.Cork, P31 F991', 'Clashanure, Ovens, Co.Cork, P31 NH32', 'Feighcullen, Rathangan, Co.Kildare', 'Hilltown, Oldcastle, Co.Meath', 'Gurrane, Templederry, Nenagh, Co.Tipperary', 'Donohill, Co.Tipperary', 'Golden, Cashel, Co.Tipperary', 'Laffansbridge, Thurles, Co.Tipperary', 'Stokesfield, Shanagolden, Co.Limerick', 'Manor West Retail Park, Tralee, Co.Kerry', 'Lowtown, Robertstown, Naas, Co.Kildare', 'Laragin, Elphn, Co.Roscommon', 'Claregalway, Co.Galway', 'Kilkelly, Co.Mayo', 'Ballisodare, Co.Sligo', 'Sheastown, Co.Kilkenny', 'Platin, Drogheda, Co. Louth', 'Castlemungret, Co. Limerick.', 'Clonagh, Enfield, Co. Meath, A83 DY62', 'Fantane, Borrisoleigh, Thurles, Co.Tipperary', 'Ballygurteen, Clonakilty, Co.Cork', 'Dromkeen, Innishannon, Co.Cork', 'Killargue, Dromahair, Co.Leitrim', 'Kilcarrig, Bagnelstown, Co.Carlow', 'Bealnamulla, Athlone, Co.Roscommon, N37 XV88', 'Troyswood, Freshford Road, Co.Kilkenny', 'Killeshal, Daingean, Co.Offaly, R35 YK85', 'Ballinakill, Co.Laois', 'Ballinascorney, Brittas, Co Dublin', 'Mullagh, Bellewstown, Co.Meath', 'Brownstown, Kilcullen, Co.Kildare', 'Castletown, Rathmolyon, Co.Meath', 'Clonard, Kilrathmurray, Co.Meath', 'Annagor, Duleek, Co. Meath', 'Rathcore, Enfield, Co.Meath', 'Gallstown Quarry, Grangebellew, Drogheda, Co.Louth', 'Piercetown, Dunboyne, Co.Meath', 'Halverstown, Kilcullen, Co.Kildare', 'N74, Knockballynoe West, Co.Tipperary', 'Walshestown, Killlinick, Co. Wexford', 'Adamstown, Lucan, Co. Dublin', 'Grange, Lucan, Co.Dublin', 'Naul, Co.Meath', 'Proudstown, Navan, Co. Meath', 'Portlaoise, Rathleague, Co.Laois', 'Rossmore, Carrigtwohill, Co.Cork', 'Ringsend, South Lotts Road, Dublin 4', 'Tallaght, Dublin 24', 'Ballysax, The Curragh, Co. Kildare', 'Bunaterin, Tullamore, Co.Offaly', 'Slieverue, Co. Waterford', 'Milebush, Midleton, Co.Cork', 'Rosemount Business Park, Ballycoolin Road, Dublin 11', 'Bunnahowen, Ballina, Co.Mayo', 'Tullamore Road, Birr, Co.Offaly', 'Ardcroney, Nenagh, Co.Tipperary', 'Rathjordan, Herberstown, Co.Limerick', 'Ballycourtney, Kilfinnane, Co.Limerick', "Fortane Beg, O'Callaghan Mills, Co. Clare", 'Cregaree, Cong, Co.Mayo, F31 W425', 'Newbrook, Claremorris, Co.Mayo, F12 CF72', 'Culliaghbeg Sand Pit, Culliaghbeg, Ballinasloe, Co.Galway', 'Rangue, Killorglin, Co.Kerry', 'Castleisland, Co.Kerry', 'Coolcaslagh, Killarney, Co.Kerry', 'Millstreet, Co.Cork', 'Connolly Street Lower, Ballina, Co.Mayo', 'Belclare, Tuam, Co.Galway', 'Cloonascragh, Athenry Road, Tuam, Co.Galway', 'Killala, Co.Mayo', 'Ballycar, Ardnacrusha, Co.Clare', "Montpellier, O'Briensbridge, Co.Limerick", 'Ballieboro Road, Kingscourt, Co.Cavan', 'Larchfield, Kingscourt, Co.Cavan', 'Oakstown, Trim, Co.Meath', 'Poulregan, Castlebridge, Co.Wexford', 'Deerpark Industrial Estate, Oranmore, Co.Galway', 'Derra, Broadford, Co.Clare', 'Ballymagroarty, Ballintra, Co.Donegal', 'Kilpedder,\xa0Bray, Co.Wicklow', 'Clonmelsh Quarry, Carlow, Co.Carlow', 'Balleese Wood Quarry, Rathdrum, Co. Wicklow', '235 Ballyconnell Road,\xa0Derrylin, Co. Fermanagh,\xa0BT92 9GP', '235 Ballyconnell Road,\xa0Derrylin, Co. Fermanagh,\xa0BT92 9GP', 'Ballaverty, Riverstown, Co. Louth', 'Rhyne, Killoe, Co.Longford', 'Allen, Naas, Co.Kildare', 'Arklow, Co.Wicklow', 'Ballintra, Co.Donegal', 'Ballyadams, Co.Laois', 'Ballyegan, Co.Kerry', 'Ballygarvan, Co.Cork', 'Ballyknockane, Co.Tipperary', 'Ballyquinn, Co.Clare', 'Barley Hill, Co.Meath', 'Belgard Central/Belgard Weighbridge, Tallaght, Dublin 24', 'Bennettsbridge, Co.Kilkenny', 'Dorans Pit, Blessington, Co.Wicklow', 'Boyle, Co.Roscommon', 'Fassaroe, Bray, Co.Wicklow', 'Bunratty, Co.Clare', 'Cam, Co.Roscommon', 'Cappagh, Co.Waterford', 'Carndonagh, Co.Donegal', 'Carrigtwohill, Co.Cork', 'Castlebar, Co.Mayo', 'Castlemine, Co.Roscommon', 'Castlemore, Co.Cork', 'Classis, Ovens, Co.Cork', 'Duleek, Co.Meath', 'Brownswood, Enniscorthy, Co.Wexford', 'Joseph Hogan, Ballylin, Foynes, Co.Limerick', 'Two Mile Ditch, Tuam Road, Co. Galway', 'Head Office, Fortunestown, Tallaght, Dublin 24', 'Huntstown Quarry, Finglas, Dublin 11', 'Kilchreest, Co.Galway', 'Killarney, Co. Kerry', 'Killorglin, Co.Kerry', 'Killough, Co.Tipperary', 'Kilmacow, Co.Kilkenny', 'Kilmuckridge, Co.Wexford', 'Laghey, Co.Donegal', 'Mallow, Co.Cork', 'Midleton, Co.Cork', 'Keim, Millstreet, Co.Cork', 'Moyne, Co.Longford', 'Mullaghcrone, Co.Meath', 'Creeves Quarry, Shanagolden, Co.Limerick', 'Slane, Co.Meath', 'Feltrim, Swords, Co.Dublin', 'Tullamore, Co.Offaly', 'Toonagh, Ennis, Co. Clare', 'Raharney, Mullingar, Co.Westmeath', 'Stephenstown Industrial Park, Balbriggan, Co.Dublin, K32 W665', 'Castlebridge, Co.Wexford', 'Aclint Bridge, Carrickmacross, Co.Monaghan', 'Swanns Cross, Rockcorry, Co.Monaghan']

如果您想在单个列表中执行此操作，请尝试此操作。

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

chrome_path = r"C:\Users\O63308\Desktop\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)

driver.get("https://www.irishconcrete.ie/members-directory/")

products=WebDriverWait(driver,20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,"span[data-bind='html:$data.title']")))
print("Total Number of products : " +str(len(products)))
locelements=WebDriverWait(driver,20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,".address+p")))
print("Total Number of locations : " +str(len(locelements)))

results=[]

for product,location in zip(products,locelements):
    results.append(product.text)
    results.append(location.get_attribute("textContent").splitlines()[0].split("Location:")[-1].strip())

print(results)

python - 网页抓取交互式内容python

问题描述

解决方案

推荐阅读