javascript - 使用 phantomJS 抓取 javascript 网页
问题描述
我正在尝试使用 phantomJS 抓取动态网页。下面是我要抓取的代码和网址。该代码适用于其他 url,但这个总是以空白 html 文档的形式返回。有谁知道如何解决这一问题?
我对javascript不是很熟悉,所以这段代码是从其他地方复制的。我已将超时时间从 2.5 秒增加到 30 秒,但并没有什么不同。
var url ='https://www.amazon.com/gp/profile/amzn1.account.AFJ6MBZ5CSY4R6K4USNMQ7JWEQCA/';
var page = new WebPage()
var fs = require('fs');
page.open(url, function (status) {
just_wait();
});
function just_wait() {
setTimeout(function() {
fs.write('page.html', page.content, 'w');
phantom.exit();
}, 30000);
}
解决方案
这就是我解决此类问题的方法。
应用程序.js
var url ='https://www.amazon.com/gp/profile/amzn1.account.AFJ6MBZ5CSY4R6K4USNMQ7JWEQCA/';
var steps=[];
var testindex = 0;
var loadInProgress = false;
//This is set to true when a page is still loading
/*********SETTINGS*********************/
var settings = require('./settings');
var webPage = require('webpage');
var page = webPage.create();
var fs = require('fs');
page.settings.userAgent = settings.userAgents.desktop;
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;
//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
page.viewportSize = {
width: settings.viewport.desktop.width,
height: settings.viewport.desktop.height
};
/*********SETTINGS END*****************/
console.log('All settings Loaded, Start With Execution');
/**********DEFINE STEPS THAT PHANTOM SHOULD DO***********************/
steps = [
function(){
console.log("Step 1 - Load Page => "+url);
page.open(url, function(status){
if(status === 'success'){
console.log('Loaded');
}else{
console.log('Error Loading Page. Try Logging In Again');
phantom.exit(0);
}
});
},
function(){
page.render('./test.png');
},
];
/**********END STEPS THAT PHANTOM SHOULD DO***********************/
interval = setInterval(executeRequestsStepByStep, 3000);
function executeRequestsStepByStep(){
if(loadInProgress == false && typeof steps[testindex] == "function") {
steps[testindex]();
testindex++;
return;
}
if(typeof steps[testindex] != "function") {
console.log("Quiting");
fs.write('page.html', page.content, 'w');
phantom.exit(0);
}
}
/*
* These listeners are very important in order to phantom work properly.
* Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
};
page.onLoadFinished = function() {
loadInProgress = false;
};
page.onConsoleMessage = function(msg) {
// console.log(msg);
};
phantom.onError = function(msg, trace) {
var msgStack = ['PHANTOM ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function +')' : ''));
});
}
console.error(msgStack.join('\n'));
phantom.exit(1);
};
设置.js
module.exports = {
viewport: {
desktop: {
height: 663,
width: 1200
}
},
userAgents: {
desktop: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'
}
};
我已经对此进行了测试,并且效果很好。
推荐阅读
- c# - 在 C# 中显示来自 JSON 的所有名称列表
- uiview - 自定义 UIView 的大小不正确(框架、边界)
- amazon-web-services - 尝试在本地访问 DynamoDB 时超时
- android - 如何在 Android 中使用 Mokito 覆盖/自定义函数的行为
- android - 为什么模块没有显示在应用程序模块依赖项中
- java - 关于 3-Sum 算法,数组访问次数如何计算(1/2 N^3),增长顺序如何计算(N^3)?
- vim - 如何使用通用 ctags 在标签栏中显示 case 语句
- c - 如何在 C 中使用带有文本文件的循环填充字符数组
- android - 片段中的片段/工具栏显示在错误的位置
- flutter - 图像选择器颤振 web 1.9