node.js - Puppeteer 不会提取所有元素
问题描述
我正在用 NodeJS 编写一个脚本来提取过去 24 小时内交易量最大的加密货币。我想在这样的数组中提取名称、股票代码和 24 小时百分比的列:
[{ name: 'Bitcoin', ticker: 'BTC', percentage: '20.62%' },
{ name: 'Ethereum', ticker: 'ETH', percentage: '10.19%' },
...
]
我的脚本看起来像这样,但是一旦你执行它,它就会跳过一些行。有谁知道为什么它随机跳过一些行?有一个更好的方法吗?
let cryptoData = []
const browser = await puppeteer.launch({ args: ['--no-sandbox'], headless: true })
const page = await browser.newPage()
await page.setViewport({ width: 1536, height: 850 })
await page.goto('https://coinmarketcap.com/', { waitUntil: 'networkidle2' })
// Wait for tickers table to fully load
await page.waitForSelector('tr:nth-child(1) > td > .cmc-link > .sc-16r8icm-0 > .sc-16r8icm-0 > .sc-1eb5slv-0')
// Sort the list 24h descending
await page.waitForSelector('.stickyTop:nth-child(5) > div > .sc-9dqrx-0 > .sc-9dqrx-1 > .sc-1eb5slv-0')
await page.click('.stickyTop:nth-child(5) > div > .sc-9dqrx-0 > .sc-9dqrx-1 > .sc-1eb5slv-0')
// Wait for tickers table to fully load
await page.waitForSelector('tr:nth-child(1) > td > .cmc-link > .sc-16r8icm-0 > .sc-16r8icm-0 > .sc-1eb5slv-0')
let data = await page.evaluate(() => {
let tempData = []
for (let index = 1; index <= 100; index++) {
let name = document.querySelector(`tr:nth-child(${index}) > td > .cmc-link > .sc-16r8icm-0 > .sc-16r8icm-0 > .sc-1eb5slv-0`)
let ticker = document.querySelector(`tr:nth-child(${index}) > td > .cmc-link > .sc-16r8icm-0 > .sc-16r8icm-0 > .sc-1teo54s-2 > .sc-1eb5slv-0`)
let percentage = document.querySelector(`.cmc-table > tbody > tr:nth-child(${index}) > td > .iqsl6q-0`)
if (name && ticker && percentage) {
name = name.innerText
tempData.push({
id: index,
name,
ticker,
percentage,
})
}
}
return tempData
})
console.log(data)
await browser.close()
解决方案
问题是 Puppeteer 没有加载整个页面,需要滚动到页面底部来加载延迟加载数据。
为此,我使用了另一个用户的回复:
https://stackoverflow.com/a/53527984
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({
headless: false
});
const page = await browser.newPage();
await page.goto('https://www.yoursite.com');
await page.setViewport({
width: 1200,
height: 800
});
await autoScroll(page);
await page.screenshot({
path: 'yoursite.png',
fullPage: true
});
await browser.close();
})();
async function autoScroll(page){
await page.evaluate(async () => {
await new Promise((resolve, reject) => {
var totalHeight = 0;
var distance = 100;
var timer = setInterval(() => {
var scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve();
}
}, 100);
});
});
}
推荐阅读
- elasticsearch - 将日志从 AWS 发送到 Elasticcloud
- java - 如何拒绝输入
- python - 这段代码对 python 线程模块安全吗?
- html - 我是否错误地使用了 flexbox?- 2 列 flexbox 错误
- pre-commit-hook - 是否可以列出受预提交运行影响的文件?
- flutter - 如何在颤振中工作 zonedSchedule 本地通知
- android - 如何使用 Flutter Driver 编写测试以等待关闭应用程序
- javascript - 从命令行控制浏览器 javascript 执行
- c# - 实现 Hangfire.FluentNHibernateStorage 时出现 ORA-00955 错误
- java - 出站 Http Enpoint 的 Spring 集成测试