javascript - Puppeteer：setDefaultNavigationTimeout 为 0 仍然超时

Question

每次我运行这个脚本都会超时。setDefaultNavigationTimeout真的可以防止超时吗？

我正在浏览大约 26 个 URL，每个页面都有大量图像。无法想象 Puppeteer 不能仅仅因为图像太重就无法处理这些页面？

const url = 'test.com';
const jsonReturn = [];


async function runScraper() {

  const browser = await puppeteer.launch(prodConfig);

  const page = await browser.newPage({
    timeout: 0
  });

  page.setDefaultNavigationTimeout(0);
  await page.goto(url, { waitUntil: 'domcontentloaded' });
  await page.waitForSelector('.featured-shows-featured-show');

  let featuredShowsURLs = await page.$$eval('.featured-shows-featured-show > a', (links) => {
    return links.map(link => {
      return link.href;
    });
  });


  featuredShowsURLs = _.uniq(featuredShowsURLs)

  for (const featuredShowsURL of featuredShowsURLs) {
    const page = await browser.newPage({
      timeout: 0
    });
    try {
      await page.goto(featuredShowsURL);
      await page.waitForSelector('.show-title');
    } catch (e) {
      featuredShowsURL;
      debugger;
    }

    const showTitle = await findAndReturnSelectorText('.show-title', page);
    const showDates = await findAndReturnSelectorText('.show-dates', page);
    const showLocation = await findAndReturnSelectorText('.show-location', page);
    const showGallery = await findAndReturnSelectorText('.entity-link', page);
    const showDetail = await findAndReturnSelectorText('.show-press-release', page);

    const newItem = {
      showTitle,
      showDates,
      showLocation,
      showGallery,
      showDetail,
    };

    const id = hash(newItem);

    jsonReturn.push({
      ...newItem,
      id
    });
  }

  await browser.close();
}

runScraper();

标签： javascriptpuppeteer