python - 无法在 Lazada Marketplace 上抓取数据

Question

我想使用美丽的汤从 Lazada 抓取网站数据，但它没有返回任何结果。此外，该网站有时会弹出一条消息以检测异常流量并要求我将栏向右滑动。我该如何解决这个问题？

图片在这里

import time
import random
from random import randint
import requests
from bs4 import BeautifulSoup #A python library to help you to exract HTML information


headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
import xlrd
import pandas as pd

results=[]
data=[]
data_name=[]


url="https://www.lazada.sg/catalog/?_keyori=ss&from=input&page=1&q=lego+super+heroes+76160&sort=priceasc"
 
r=requests.get(url)
soup=BeautifulSoup(r.text)
number_result_div=soup.findAll("div", {"class":" index__tips___1OXnm"})
producttext_div=soup.findAll("div", {"class":"ListItem__title___3ZDDK"})
productprice_div=soup.findAll("div", {"class":"ListItem__price___24YgJ"})

for i in number_result_div:
    result=soup.find("span", {"data-spm-anchor-id":"a2o42.searchlist.0.i2.7ccd5675UbMz6P"}).text.strip()[:-16]
    results.append(result)
    break
    

for i in producttext_div:
        
    yeah=soup.find("a", {"data-spm-anchor-id":"a2o42.searchlist.list.2"}).text.strip()
    data_name.append(yeah)
    break
    
    
for i in productprice_div: 
        
    lowest=i.find("span",{"class":"index__currency___Q78Jz"}).text.strip()
    data.append(lowest)
    break
        
print(results)
print(data_name)
print(data)

标签： pythonbeautifulsoupweb-crawler