node.js - Nodejs > Gulp > through2 > 限制为 16 个文件？

问题描述

更新好的，这似乎与 through2 的“highWaterMark”属性有关。基本上，这意味着“不要缓冲超过 x 个文件，等待有人使用它，然后才接受另一批文件”。由于它是按设计以这种方式工作的，因此正在审查此问题中的代码段。必须有更好的方法来处理许多文件。

快速修复，允许 8000 个文件：

  through.obj({ highWaterMark: 8000 }, (file, enc, next) => { ... })

原始问题

我正在使用 gulp 任务来创建翻译文件。它扫描src文件夹中的文件并在源文件中找到的每种语言*.i18n.json保存一个。.json

它工作正常 - 直到找到超过 16 个文件。它through2用于处理每个文件。请参阅下面的源代码。该方法processAll18nFiles()是一个自定义管道，它接收匹配的输入文件，读取每个文件的内容，即时构建结果字典，然后最终将其交给on('finish)处理程序以编写字典。

在 windows 和 mac 上测试。我的方法似乎有一个限制，因为它适用于 16 个或更少的文件。

仍在寻找，欢迎提供线索:-)

源文件示例：signs.i18n.json

{
  "path": "profile.signs",
  "data": {
    "title": {
      "fr": "mes signes précurseurs",
      "en": "my warning signs"
    },
    "add": {
      "fr": "ajouter un nouveau signe",
      "en": "add a new warning sign"
    }
  }
}

输出文件示例：en.json

{"profile":{"signs":{"title":"my warning signs","add":"add a new warning sign"}}}

gulpfile.js

const fs = require('fs');
const path = require('path');
const gulp = require('gulp');
const watch = require('gulp-watch');
const through = require('through2');

const searchPatternFolder = 'src/app/**/*.i18n.json';
const outputFolder = path.join('src', 'assets', 'i18n');

gulp.task('default', () => {
  console.log('Ionosphere Gulp tasks');
  console.log(' > gulp i18n         builds the i18n file.');
  console.log(' > gulp i18n:watch   watches i18n file and trigger build.');
});

gulp.task('i18n:watch', () => watch(searchPatternFolder, { ignoreInitial: false }, () => gulp.start('i18n')));
gulp.task('i18n', done => processAll18nFiles(done));

function processAll18nFiles(done) {
  const dictionary = {};
  console.log('[i18n] Rebuilding...');
  gulp
    .src(searchPatternFolder)
    .pipe(
      through.obj((file, enc, next) => {
        console.log('doing ', file.path);
        const i18n = JSON.parse(file.contents.toString('utf8'));
        composeDictionary(dictionary, i18n.data, i18n.path.split('.'));
        next(null, file);
      })
    )
    .on('finish', () => {
      const writes = [];
      Object.keys(dictionary).forEach(langKey => {
        console.log('lang key ', langKey);
        writes.push(writeDictionary(langKey, dictionary[langKey]));
      });
      Promise.all(writes)
        .then(data => done())
        .catch(err => console.log('ERROR ', err));
    });
}

function composeDictionary(dictionary, data, path) {
  Object.keys(data)
    .map(key => ({ key, data: data[key] }))
    .forEach(({ key, data }) => {
      if (isString(data)) {
        setDictionaryEntry(dictionary, key, path, data);
      } else {
        composeDictionary(dictionary, data, [...path, key]);
      }
    });
}

function isString(x) {
  return Object.prototype.toString.call(x) === '[object String]';
}

function initDictionaryEntry(key, dictionary) {
  if (!dictionary[key]) {
    dictionary[key] = {};
  }
  return dictionary[key];
}

function setDictionaryEntry(dictionary, langKey, path, data) {
  initDictionaryEntry(langKey, dictionary);
  let subDict = dictionary[langKey];
  path.forEach(subKey => {
    isLastToken = path[path.length - 1] === subKey;
    if (isLastToken) {
      subDict[subKey] = data;
    } else {
      subDict = initDictionaryEntry(subKey, subDict);
    }
  });
}

function writeDictionary(lang, data) {
  return new Promise((resolve, reject) => {
    fs.writeFile(
      path.join(outputFolder, lang + '.json'),
      JSON.stringify(data),
      'utf8',
      err => (err ? reject(err) : resolve())
    );
  });
}

标签： node.jsgulp

gulp .src(searchPatternFolder) .pipe( through.obj({ highWaterMark: 4, objectMode: true }, (file, enc, next) => { const { data, path } = JSON.parse(file.contents.toString('utf8')); next(null, { data, path }); }) ) // The next line handles the "consumption" of upstream pipings .on('data', ({ data, path }) => ++count && composeDictionary(dictionary, data, path.split('.'))) .on('end', () => Promise.all(Object.keys(dictionary).map(langKey => writeDictionary(langKey, dictionary[langKey]))) .then(() => { console.log(`[i18n] Done, ${count} files processed, language count: ${Object.keys(dictionary).length}`); done(); }) .catch(err => console.log('ERROR ', err)) );

node.js - Nodejs > Gulp > through2 > 限制为 16 个文件？

问题描述

解决方案

推荐阅读