javascript - 如何加快获取javascript数组的数据

Question

我想获取 20000 个文件的创建日期并将其存储在一个数组中。
完成的总时间是35 分钟，相当长的时间。（图像处理时间）
有没有办法以更快的处理时间创建阵列？

当前的逻辑是否有任何问题来获取如下所示的文件创建日期数组？
①数组声明：var arr = [];
②我使用下面的代码来获取文件创建日期：

var fs = global.get('fs');
// Get file creation date time
msg.createdDate = fs.statSync(msg.pathFName).birthtime;
return msg;

③ 将创建日期存储在一个数组中。

我的流程：
图像流程

[
    {
        "id": "de157360.11d49",
        "type": "tab",
        "label": "フロー 1",
        "disabled": false,
        "info": ""
    },
    {
        "id": "6a8db36.f55e14c",
        "type": "inject",
        "z": "de157360.11d49",
        "name": "",
        "props": [
            {
                "p": "payload"
            },
            {
                "p": "topic",
                "vt": "str"
            }
        ],
        "repeat": "",
        "crontab": "",
        "once": false,
        "onceDelay": 0.1,
        "topic": "",
        "payload": "",
        "payloadType": "str",
        "x": 750,
        "y": 160,
        "wires": [
            [
                "4907d69b.07f738"
            ]
        ]
    },
    {
        "id": "4907d69b.07f738",
        "type": "change",
        "z": "de157360.11d49",
        "name": "",
        "rules": [
            {
                "t": "set",
                "p": "pathFiles",
                "pt": "msg",
                "to": "D:\\Data\\",
                "tot": "str"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 910,
        "y": 160,
        "wires": [
            [
                "d68da128.cd6d9"
            ]
        ]
    },
    {
        "id": "d68da128.cd6d9",
        "type": "fs-ops-dir",
        "z": "de157360.11d49",
        "name": "",
        "path": "pathFiles",
        "pathType": "msg",
        "filter": "*",
        "filterType": "str",
        "dir": "files",
        "dirType": "msg",
        "x": 1080,
        "y": 160,
        "wires": [
            [
                "764296f2.688338"
            ]
        ]
    },
    {
        "id": "764296f2.688338",
        "type": "change",
        "z": "de157360.11d49",
        "name": "f=0",
        "rules": [
            {
                "t": "set",
                "p": "f",
                "pt": "msg",
                "to": "0",
                "tot": "num"
            },
            {
                "t": "set",
                "p": "arrObjFiles",
                "pt": "msg",
                "to": "[]",
                "tot": "json"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 1210,
        "y": 160,
        "wires": [
            [
                "ab664988.c25da8",
                "232fdda0.b11e22"
            ]
        ]
    },
    {
        "id": "1912092c.fbab77",
        "type": "change",
        "z": "de157360.11d49",
        "name": "f++",
        "rules": [
            {
                "t": "set",
                "p": "f",
                "pt": "msg",
                "to": "$.f + 1\t",
                "tot": "jsonata"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 1650,
        "y": 60,
        "wires": [
            [
                "ab664988.c25da8"
            ]
        ]
    },
    {
        "id": "ab664988.c25da8",
        "type": "switch",
        "z": "de157360.11d49",
        "name": "u<number of files",
        "property": "f",
        "propertyType": "msg",
        "rules": [
            {
                "t": "lt",
                "v": "files.length",
                "vt": "msg"
            },
            {
                "t": "else"
            }
        ],
        "checkall": "true",
        "repair": false,
        "outputs": 2,
        "x": 1410,
        "y": 160,
        "wires": [
            [
                "15154524.a6828b"
            ],
            [
                "c7625457.8ccb18"
            ]
        ]
    },
    {
        "id": "f8030294.f6334",
        "type": "function",
        "z": "de157360.11d49",
        "name": "Get file creation date time",
        "func": "var fs = global.get('fs');\n// Get file creation date time\nmsg.createdDate = fs.statSync(msg.pathFName).birthtime;\nreturn msg;",
        "outputs": 1,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "x": 1830,
        "y": 160,
        "wires": [
            [
                "59879c16.6c7564"
            ]
        ]
    },
    {
        "id": "59879c16.6c7564",
        "type": "change",
        "z": "de157360.11d49",
        "name": "Storage file creation date time",
        "rules": [
            {
                "t": "set",
                "p": "objFiles",
                "pt": "msg",
                "to": "{}",
                "tot": "json"
            },
            {
                "t": "set",
                "p": "objFiles",
                "pt": "msg",
                "to": "$merge([$.objFiles,{'fileName': $.fileName,'createdDate': $.createdDate}])",
                "tot": "jsonata"
            },
            {
                "t": "set",
                "p": "arrObjFiles",
                "pt": "msg",
                "to": "$append(arrObjFiles, [objFiles])",
                "tot": "jsonata"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 2090,
        "y": 160,
        "wires": [
            [
                "1912092c.fbab77"
            ]
        ]
    },
    {
        "id": "15154524.a6828b",
        "type": "change",
        "z": "de157360.11d49",
        "name": "",
        "rules": [
            {
                "t": "set",
                "p": "fileName",
                "pt": "msg",
                "to": "$.files[$$.f]",
                "tot": "jsonata"
            },
            {
                "t": "set",
                "p": "pathFName",
                "pt": "msg",
                "to": "pathFiles & fileName",
                "tot": "jsonata"
            }
        ],
        "action": "",
        "property": "",
        "from": "",
        "to": "",
        "reg": false,
        "x": 1620,
        "y": 160,
        "wires": [
            [
                "f8030294.f6334"
            ]
        ]
    },
    {
        "id": "c7625457.8ccb18",
        "type": "debug",
        "z": "de157360.11d49",
        "name": "END",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "arrObjFiles",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 1590,
        "y": 220,
        "wires": []
    },
    {
        "id": "232fdda0.b11e22",
        "type": "debug",
        "z": "de157360.11d49",
        "name": "START",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "files.length",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 1380,
        "y": 220,
        "wires": []
    }
]

标签： javascriptnode.jsnode-red

Answer 1

您正在使用fs.statSyncwhich 是一个同步函数，这意味着每次调用它时，所有代码执行都会停止，直到该函数完成。研究使用fs.stat（异步版本），映射文件路径数组，并使用Promise.all。

使用fs.stat（异步版本）功能，您可以一次启动多个文件的调用，以便总体上发生得更快（因为可以一次加载多个文件而无需等待超慢的文件）

这是我的意思的一个例子，你可以在浏览器中运行：

const arr = ["hello", "i", "am", "here"];

// This is an example of a function that returns a promise.
// This function returns a promise that resolves after `seconds` amount of seconds, returning the value of `word` to uppercase
const sleepAndUppercase = (seconds, word) => new Promise(
  (resolve) => setTimeout(
     () => resolve(word.toUpperCase()), seconds*1000
   )
);

// This wraps the array of promises
Promise.all(
  // this converts the array of words into an array of promises generated by the sleepAndUppercase function defined above
  arr.map((word, i) => sleepAndUppercase(i, word))
  // the next line waits until all the promies are done, and uses the result as the variable listOfWords,
  // which is an array of the words capitalized
).then((listOfWords) => {
    // we can log the results
    console.log(listOfWords);
    // we can create variables based off of those results
    const sentence = listOfWords.join(" ");
    console.log(sentence);
  });

console.log("Promises started")

现在让它更接近你的问题：

在这个例子中，每次调用fs.stat需要 5 秒来解决，但是所有 4 个结果一起出来的速度比 5 * 4 秒快得多，因为多个调用同时开始。

// This object is just here so that this snippet makes sense on StackOverflow.
// Do not copy this object into your code.
const fs = {
  stat: (pathFName) => {
    const birthtime = Date.now() - Math.floor(Math.random() * 100000000);
    return new Promise((resolve, reject) => {
      setTimeout(() => resolve({birthtime}), 5000)
    })
  }
}

// start timer
let timer = 0;
const timerInterval = setInterval(() => document.querySelector("#ptimer").innerText = `${timer += 0.02}s`, 20);

const msgs = [
  {pathFName: "foo.txt"},
  {pathFName: "bar.png"},
  {pathFName: "baz.html"},
  {pathFName: "idk.jpg"},
  {pathFName: "pepe.sql"},
];

(async () => {
  const msgsWithTimeStamps = await Promise.all(
    msgs.map(async (msg) => {
      const {birthtime} = await fs.stat(msg.pathFName);
      return {...msg, birthtime}
    })
  )
  console.log(msgsWithTimeStamps);
  // stop timer
  clearInterval(timerInterval);
})()

<p id="ptimer">0s</p>