javascript - Puppeteer 中的 API 请求因 401 错误而被阻止
问题描述
我正在尝试使用 pupeteer 从 instacart API 获取数据,如果我在我的 chrome 浏览器中并转到此 URL: https ://www.instacart.com/products/2736298-red-seedless-grapes-4-lb
并在 devtools 浏览器中运行此代码:
async() => {
try {
let apiUrl = `https://www.instacart.com/v3/containers/items/121555457?source=web`;
let header = {
"headers": {
"accept": "*/*",
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"x-requested-with": "XMLHttpRequest"
},
"method": "GET",
"mode": "cors",
"credentials": "include"
}
let resp = await fetch(apiUrl, header);
console.log(resp)
return resp;
} catch(err) {
console.log('err caught:',err)
return err;
}
}
它向以下 URL 发出请求:https ://www.instacart.com/v3/containers/items/121555457?source=web ,我可以在我的 chrome 浏览器中正常访问它并且它有数据。但是,如果我尝试在 puppeteer 浏览器窗口中发出访问此 URl 的获取请求,该请求会返回 401 错误:
Response {type: "basic", url: "https://www.instacart.com/v3/containers/items/121555457?source=web", redirected: false, status: 401, ok: false, …}
body: (...)
bodyUsed: false
headers: Headers {}
ok: false
redirected: false
status: 401
statusText: ""
type: "basic"
url: "https://www.instacart.com/v3/containers/items/121555457?source=web"
我一直在尝试在启动 puppeteer 时添加 args 来解决这个问题,但到目前为止没有运气,下面是我完整的 puppeteer 代码,我知道我可以在我的个人 chrome 浏览器中正常访问这个网站,但是它在 puppeteer 中被阻止了,任何人都可以帮我在 puppeteer 中获取这些数据?
const puppeteer = require('puppeteer');
console.log('begin')
init();
async function init(){
var browser = await puppeteer.launch({
executablePath:'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
devtools: true,
headless: false,
args: ['--lang=en-US,en','--no-sandbox','--disable-setuid-sandbox','--disable-web-security','--disable-features=IsolateOrigins','--disable-site-isolation-trials']
});
let page = await browser.newPage();
await page.goto('https://www.instacart.com/store/items/item_121555457', {
waitUntil: 'networkidle2', timeout: 0
});
let apiResp = await page.evaluate( async() => {
try {
let apiUrl = `https://www.instacart.com/v3/containers/items/121555457?source=web`;
let header = {
"headers": {
"accept": "*/*",
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"x-requested-with": "XMLHttpRequest"
},
"method": "GET",
"mode": "cors",
"credentials": "include"
}
let resp = await fetch(apiUrl, header);
console.log(resp)
return resp;
} catch(err) {
console.log('err caught:',err)
return err;
}
});
console.log('apiResp=',apiResp)
}