javascript - 保存到文件导致异步功能

Question

我有一个简单的异步函数，我从 URL 中“抓取”站点。一切正常，但现在我想将结果保存到我的 txt 文件中。

我试图做简单的数组，我可以推送每个结果也错误；

现在我遇到了一个问题，我应该在哪里写入文件。

我尝试将它放到一个单独的函数中，然后在我的异步函数中执行 await 函数，但写入文件的函数我总是首先触发。

有完整的代码

const https = require("https");
const fs = require("fs");
const readline = require("readline");
const path = require("path");


let urls = [];
let results = [];

(async function readUrls() {
  const fileStream = fs.createReadStream("urls.txt");

  const rl = readline.createInterface({
    input: fileStream,
    crlfDelay: Infinity,
  });

  for await (let line of rl) {
    urls.push(line);
  }
  for await (let url of urls) {
    https
      .get(url, (res) => {
        const {
          statusCode
        } = res;
        const contentType = res.headers["content-type"];

        let error;
        if (statusCode !== 200) {
          error = new Error("Request Failed.\n" + `Status Code: ${statusCode}`);
        }
        if (error) {
          const firstPath = url.split("/")[7];
          //there is array
          results.push(firstPath);
          //--------------
          console.error("data : " + firstPath + " - " + " nothing found");
          res.resume();
          return;
        }
        res.setEncoding("utf8");
        let rawData = "";
        res.on("data", (chunk) => {
          rawData += chunk;
        });
        (async () => {
          await res.on("end", () => {
            try {
              const parsedData = JSON.parse(rawData);
              const parsedResult = parsedData["data"]["id"] + " - " + parsedData["data"]["price"];
              //there is array
              results.push(parsedResult);
              //--------------
              console.log("data : " + parsedData["data"]["id"] + " - " + parsedData["data"]["price"]);
            } catch (e) {
              console.error(e.message);
            }
          });
        })();
      })
      .on("error", (e) => {
        console.error(`Got error: ${e.message}`);
      });
  }
})();

有我的简单功能写入文件

fs.writeFile('result.txt', results, +(new Date()), function (err) {
    if (err) {
        console.log("Error occurred", err);
    }
    console.log("File write successfull");
});

我试着做点什么

async function secondFunction(){
  await firstFunction();
  // wait for firstFunction...
};

我想达到什么目标？我想从我的文本文件中抓取每个 url 并获取 ID 和价格（这是对浏览器没有 html 的简单 JSON 响应 - 它有效） 最后我想将所有内容保存到文本文件中。

标签： javascriptnode.jsasync-awaitfs

Answer 1

我制作了一个使用 node-fetch 来调用 url 的代码版本。我更喜欢这个，因为它类似于可以在网络上使用的

要使用它，您应该安装它： npm install node-fetch

    const fetch = require("node-fetch"); // I prefer to use node-fetch for my calls
    const fs = require("fs");
    const readline = require("readline");
    const path = require("path");


    let urls = [];
    let results = [];

    (async function readUrls() {
      const fileStream = fs.createReadStream("urls.txt");

      const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity,
      });

      for await (let line of rl) {
        urls.push(line);
      }
      // Make the calls one after the other
      for (let url of urls) {
        try {
          // We can call the urls with node-fetch and await the response
          const res = await fetch(url);
          const { status } = res;
          let error;
          if (status !== 200)
            error = new Error("Request Failed.\n" + `Status Code: ${statusCode}`);
          if (error) {
            const firstPath = url.split('/')[7];
            results.push(firstPath);
            console.error("data : " + firstPath + " - " + " nothing found");
            // As we are inside a loop here, we use continue instead of return
            continue;
          }
          try {
            // Here we try to take the response as json
            const parsedData = await res.json();
            const parsedResult = parsedData["data"]["id"] + " - " + parsedData["data"]["price"];
            //there is array
            results.push(parsedResult);
            //--------------
            console.log(`Data: ${parsedResult}`);
          } catch (e) {
            // In case we can't get the response as json we log the error
            console.error(e.message);
          }
        } catch (httpError) {
          //This is for when the call to fetch fails for some reason
          console.error(httpError.message);
        }  
      }
      // Here we join the results to a string so that we can save it properly to the file
      const resultAsText = results.join("\n");
      // Then after all the urls are processed we can write them to a file
      fs.writeFile('result.txt', resultAsText, 'utf8', function (err) {
        if (err) {
          console.log("Error occurred", err);
        } else {
          console.log("File write successfull");
        }
      });
    })();