node.js - 我试图抓取的网站阻止了我，因为我正在使用自动化工具，我该如何解决这个问题？

问题描述

当我尝试使用 puppeteer 抓取某些页面时，我被 chegg.com 阻止。有没有办法解决这个问题？也许在铬中使用隐身浏览器？

当我尝试登录并阻止我登录时，它会给出错误。

我还记得阅读有关标题或删除标题以解决问题的内容？

我只打算每分钟刮一次这个页面，也许每小时刮几次。不是很经常。

const puppeteer = require("puppeteer");
const CREDS = require("./creds");
var SlackBot = require("slackbots");
var channel = "testing";
var tryAgain =
  "Try again, nerd. Post a plain text chegg link Ex. https://www.chegg.com/homework-help/questions-and-answers/assume-unc...";
var tryAgainTwo =
  "Try again, nerd. Post a plain text chegg link Ex. <https://www.chegg.com/homework-help/questions-and-answers/assume-unc>...";
var imageName = "workplease2";

var bot = new SlackBot({
  token: "--SNIP--",
  name: "cheggy"
});

// bot.on("start", function() {
//   bot.postMessageToChannel(channel, "Hello world!");
// });

bot.on("message", function(data) {
  if (data.type !== "message") {
    return;
  }

  handleMessage(data.text);
});

function handleMessage(message) {
  if (
    message.includes("chegg.com/") &&
    message.includes("Try again, nerd.") === false
  ) {
    bot.postMessageToChannel(
      channel,
      "Give me a sec to find that for you. XOXO"
    );
    run();
  } else {
    if (
      message === tryAgainTwo ||
      message.includes("sec to find") ||
      message.includes(imageName)
    ) {
      return;
    }
    bot.postMessageToChannel(channel, tryAgain);
    console.log("XXXXXXXXXXXXXXXXXXXXthis is the message!" + message);
    return;
  }
}

function sendGreeting() {
  var greeting = getGreeting();
  bot.postMessageToChannel(channel, greeting);
}

function getGreeting() {
  var greetings = [
    "hello!",
    "hi there!",
    "cheerio!",
    "how do you do!",
    "¡hola!"
  ];
  return greetings[Math.floor(Math.random() * greetings.length)];
}

async function timeout(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function run() {
  const browser = await puppeteer.launch({
    headless: false
  });

  const page = await browser.newPage();

  await page.goto("https://www.chegg.com/auth?action=login&reset_password=0");

  const USERNAME_SELECTOR = "#emailForSignIn";
  const PASSWORD_SELECTOR = "#passwordForSignIn";
  //const BUTTON_SELECTOR = '#eggshell-19 > form > div > div > div > footer > button';

  await page.click(USERNAME_SELECTOR);
  await page.keyboard.type(CREDS.email);

  await page.click(PASSWORD_SELECTOR);
  await page.keyboard.type(CREDS.password);

  //await page.click(BUTTON_SELECTOR);
  await page.keyboard.press(String.fromCharCode(13));

  await page.waitForNavigation();

  await page.goto(
    "https://www.chegg.com/homework-help/questions-and-answers/assume-uncle-holds-one-stock-east-coast-bank-ecb-thinks-little-risk-agree-stock-relatively-q9069609"
  );

  await timeout(1000);

  await page.screenshot({ path: "./myfolder/" + imageName, fullPage: true });

  browser.close();

  console.log("do you reach me?");

  var fs = require("fs");
  var request = require("request");

  var SLACK_TOKEN = "--SNIP--";
  var SLACK_CHANNEL = "general";
  var filepath = "./myfolder/workplease2.png";

  var options = {
    method: "POST",
    url: "https://slack.com/api/files.upload",
    headers: { "cache-control": "no-cache" },
    formData: {
      token: SLACK_TOKEN,
      channels: SLACK_CHANNEL,
      file: fs.createReadStream(filepath)
    }
  };

  request(options, function(error, response, body) {
    if (error) throw new Error(error);

    console.log(body);
  });
}

标签： node.jsweb-scrapingpuppeteer

const browser = await puppeteer.launch({ headless: false, devtools: true, slowMo: 250 userDataDir: 'C:\\userData' // userDataDir <string> Path to a User Data Directory. }); const page = await browser.pages(); await page[0].setUserAgent('5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36');

node.js - 我试图抓取的网站阻止了我，因为我正在使用自动化工具，我该如何解决这个问题？

问题描述

解决方案

推荐阅读