javascript - ftp 目录下载触发最大调用堆栈超出错误
问题描述
我目前正在使用 NodeJS 编写备份脚本。该脚本使用 FTP/FTPS 递归下载目录及其文件和子目录。我正在使用basic-ftp包进行 FTP 调用。
当我尝试下载一个包含很多子目录的大目录时,我得到了Maximum call stack size exceeded
错误,但我找不到它发生的原因和位置。我没有看到任何无限循环或任何丢失的返回调用。经过数小时的调试,我没有更多的想法。
我不使用downloadDirTo
basic-ftp的方法,因为我不想在发生错误后停止下载。当发生错误时,它应该继续运行,并且应该将错误添加到日志文件中。
存储库在这里:https ://github.com/julianpoemp/webspace-backup 。
一旦 FTPManager 准备好,我就调用 doBackup 方法(参见 BackupManager 中的方法)。该方法调用 FTPManager 中定义的 downloadFolder 方法。
export class BackupManager {
private ftpManager: FtpManager;
constructor() {
osLocale().then((locale) => {
ConsoleOutput.info(`locale is ${locale}`);
moment.locale(locale);
}).catch((error) => {
ConsoleOutput.error(error);
});
this.ftpManager = new FtpManager(AppSettings.settings.backup.root, {
host: AppSettings.settings.server.host,
port: AppSettings.settings.server.port,
user: AppSettings.settings.server.user,
password: AppSettings.settings.server.password,
pasvTimeout: AppSettings.settings.server.pasvTimeout
});
this.ftpManager.afterManagerIsReady().then(() => {
this.doBackup();
}).catch((error) => {
ConsoleOutput.error(error);
});
}
public doBackup() {
let errors = '';
if (fs.existsSync(path.join(AppSettings.appPath, 'errors.log'))) {
fs.unlinkSync(path.join(AppSettings.appPath, 'errors.log'));
}
if (fs.existsSync(path.join(AppSettings.appPath, 'statistics.txt'))) {
fs.unlinkSync(path.join(AppSettings.appPath, 'statistics.txt'));
}
const subscr = this.ftpManager.error.subscribe((message: string) => {
ConsoleOutput.error(`${moment().format('L LTS')}: ${message}`);
const line = `${moment().format('L LTS')}:\t${message}\n`;
errors += line;
fs.appendFile(path.join(AppSettings.appPath, 'errors.log'), line, {
encoding: 'Utf8'
}, () => {
});
});
let name = AppSettings.settings.backup.root.substring(0, AppSettings.settings.backup.root.lastIndexOf('/'));
name = name.substring(name.lastIndexOf('/') + 1);
const downloadPath = (AppSettings.settings.backup.downloadPath === '') ? AppSettings.appPath : AppSettings.settings.backup.downloadPath;
ConsoleOutput.info(`Remote path: ${AppSettings.settings.backup.root}\nDownload path: ${downloadPath}\n`);
this.ftpManager.statistics.started = Date.now();
this.ftpManager.downloadFolder(AppSettings.settings.backup.root, path.join(downloadPath, name)).then(() => {
this.ftpManager.statistics.ended = Date.now();
this.ftpManager.statistics.duration = (this.ftpManager.statistics.ended - this.ftpManager.statistics.started) / 1000 / 60;
ConsoleOutput.success('Backup finished!');
const statistics = `\n-- Statistics: --
Started: ${moment(this.ftpManager.statistics.started).format('L LTS')}
Ended: ${moment(this.ftpManager.statistics.ended).format('L LTS')}
Duration: ${this.ftpManager.getTimeString(this.ftpManager.statistics.duration * 60 * 1000)} (H:m:s)
Folders: ${this.ftpManager.statistics.folders}
Files: ${this.ftpManager.statistics.files}
Errors: ${errors.split('\n').length - 1}`;
ConsoleOutput.log('\n' + statistics);
fs.writeFileSync(path.join(AppSettings.appPath, 'statistics.txt'), statistics, {
encoding: 'utf-8'
});
if (errors !== '') {
ConsoleOutput.error(`There are errors. Please read the errors.log file for further information.`);
}
subscr.unsubscribe();
this.ftpManager.close();
}).catch((error) => {
ConsoleOutput.error(error);
this.ftpManager.close();
});
}
}
import * as ftp from 'basic-ftp';
import {FileInfo} from 'basic-ftp';
import * as Path from 'path';
import * as fs from 'fs';
import {Subject} from 'rxjs';
import {FtpEntry, FTPFolder} from './ftp-entry';
import {ConsoleOutput} from './ConsoleOutput';
import moment = require('moment');
export class FtpManager {
private isReady = false;
private _client: ftp.Client;
private currentDirectory = '';
public readyChange: Subject<boolean>;
public error: Subject<string>;
private connectionOptions: FTPConnectionOptions;
public statistics = {
folders: 0,
files: 0,
started: 0,
ended: 0,
duration: 0
};
private recursives = 0;
constructor(path: string, options: FTPConnectionOptions) {
this._client = new ftp.Client();
this._client.ftp.verbose = false;
this.readyChange = new Subject<boolean>();
this.error = new Subject<string>();
this.currentDirectory = path;
this.connectionOptions = options;
this.connect().then(() => {
this.isReady = true;
this.gotTo(path).then(() => {
this.onReady();
}).catch((error) => {
ConsoleOutput.error('ERROR: ' + error);
this.onConnectionFailed();
});
});
}
private connect(): Promise<void> {
return new Promise<void>((resolve, reject) => {
this._client.access({
host: this.connectionOptions.host,
user: this.connectionOptions.user,
password: this.connectionOptions.password,
secure: true
}).then(() => {
resolve();
}).catch((error) => {
reject(error);
});
});
}
private onReady = () => {
this.isReady = true;
this.readyChange.next(true);
};
private onConnectionFailed() {
this.isReady = false;
this.readyChange.next(false);
}
public close() {
this._client.close();
}
public async gotTo(path: string) {
return new Promise<void>((resolve, reject) => {
if (this.isReady) {
ConsoleOutput.info(`open ${path}`);
this._client.cd(path).then(() => {
this._client.pwd().then((dir) => {
this.currentDirectory = dir;
resolve();
}).catch((error) => {
reject(error);
});
}).catch((error) => {
reject(error);
});
} else {
reject(`FTPManager is not ready. gotTo ${path}`);
}
});
}
public async listEntries(path: string): Promise<FileInfo[]> {
if (this.isReady) {
return this._client.list(path);
} else {
throw new Error('FtpManager is not ready. list entries');
}
}
public afterManagerIsReady(): Promise<void> {
return new Promise<void>((resolve, reject) => {
if (this.isReady) {
resolve();
} else {
this.readyChange.subscribe(() => {
resolve();
},
(error) => {
reject(error);
},
() => {
});
}
});
}
public async downloadFolder(remotePath: string, downloadPath: string) {
this.recursives++;
if (this.recursives % 100 === 99) {
ConsoleOutput.info('WAIT');
await this.wait(0);
}
if (!fs.existsSync(downloadPath)) {
fs.mkdirSync(downloadPath);
}
try {
const list = await this.listEntries(remotePath);
for (const fileInfo of list) {
if (fileInfo.isDirectory) {
const folderPath = remotePath + fileInfo.name + '/';
try {
await this.downloadFolder(folderPath, Path.join(downloadPath, fileInfo.name));
this.statistics.folders++;
ConsoleOutput.success(`${this.getCurrentTimeString()}===> Directory downloaded: ${remotePath}\n`);
} catch (e) {
this.error.next(e);
}
} else if (fileInfo.isFile) {
try {
const filePath = remotePath + fileInfo.name;
if (this.recursives % 100 === 99) {
ConsoleOutput.info('WAIT');
await this.wait(0);
}
await this.downloadFile(filePath, downloadPath, fileInfo);
} catch (e) {
this.error.next(e);
}
}
}
return true;
} catch (e) {
this.error.next(e);
return true;
}
}
public async downloadFile(path: string, downloadPath: string, fileInfo: FileInfo) {
this.recursives++;
if (fs.existsSync(downloadPath)) {
const handler = (info) => {
let procent = Math.round((info.bytes / fileInfo.size) * 10000) / 100;
if (isNaN(procent)) {
procent = 0;
}
let procentStr = '';
if (procent < 10) {
procentStr = '__';
} else if (procent < 100) {
procentStr = '_';
}
procentStr += procent.toFixed(2);
ConsoleOutput.log(`${this.getCurrentTimeString()}---> ${info.type} (${procentStr}%): ${info.name}`);
};
if (this._client.closed) {
try {
await this.connect();
} catch (e) {
throw new Error(e);
}
}
this._client.trackProgress(handler);
try {
await this._client.downloadTo(Path.join(downloadPath, fileInfo.name), path);
this._client.trackProgress(undefined);
this.statistics.files++;
return true;
} catch (e) {
throw new Error(e);
}
} else {
throw new Error('downloadPath does not exist');
}
}
public chmod(path: string, permission: string): Promise<void> {
return new Promise<void>((resolve, reject) => {
this._client.send(`SITE CHMOD ${permission} ${path}`).then(() => {
console.log(`changed chmod of ${path} to ${permission}`);
resolve();
}).catch((error) => {
reject(error);
});
});
}
public getCurrentTimeString(): string {
const duration = Date.now() - this.statistics.started;
return moment().format('L LTS') + ' | Duration: ' + this.getTimeString(duration) + ' ';
}
public getTimeString(timespan: number) {
if (timespan < 0) {
timespan = 0;
}
let result = '';
const minutes: string = this.formatNumber(this.getMinutes(timespan), 2);
const seconds: string = this.formatNumber(this.getSeconds(timespan), 2);
const hours: string = this.formatNumber(this.getHours(timespan), 2);
result += hours + ':' + minutes + ':' + seconds;
return result;
}
private formatNumber = (num, length): string => {
let result = '' + num.toFixed(0);
while (result.length < length) {
result = '0' + result;
}
return result;
};
private getSeconds(timespan: number): number {
return Math.floor(timespan / 1000) % 60;
}
private getMinutes(timespan: number): number {
return Math.floor(timespan / 1000 / 60) % 60;
}
private getHours(timespan: number): number {
return Math.floor(timespan / 1000 / 60 / 60);
}
public async wait(time: number): Promise<void> {
return new Promise<void>((resolve) => {
setTimeout(() => {
resolve();
}, time);
});
}
}
export interface FTPConnectionOptions {
host: string;
port: number;
user: string;
password: string;
pasvTimeout: number;
}
解决方案
问题
在FtpManager.downloadFolder
函数内部,我看到downloadFolder
使用await
. 您的Maximum call stack exceeded
错误可能来自那里,因为您的初始调用需要在遍历所有子目录时将所有内容保存在内存中。
建议的解决方案
await
您可以使用如下算法设置队列系统,而不是递归地对所有内容进行处理:
- 将当前文件夹添加到队列
- 虽然该队列不为空:
- 获取队列中的第一个文件夹(并从中删除)
- 列出其中的所有条目
- 下载所有文件
- 将所有子文件夹添加到队列中
这使您可以循环下载大量文件夹,而不是使用递归。每个循环迭代都将独立运行,这意味着根目录下载的结果将不依赖于其中的 deeeeeep 文件树。
使用队列管理器
NodeJS 有很多队列管理器模块,它们允许您拥有并发、超时等。我过去使用的一个简单地命名为queue。它有很多有用的功能,但需要更多的工作才能在您的项目中实现。因此,对于这个答案,我没有使用外部队列模块,因此您可以看到它背后的逻辑。随意搜索queue
, job
, concurrency
...
例子
我想直接在你自己的代码中实现这个逻辑,但我不使用 Typescript,所以我想我会做一个简单的文件夹复制功能,它使用相同的逻辑。
注意:为简单起见,我没有添加任何错误处理,这只是一个概念证明!你可以在我的 Github 上找到一个使用这个的演示项目。
以下是我的做法:
const fs = require('fs-extra');
const Path = require('path');
class CopyManager {
constructor() {
// Create a queue accessible by all methods
this.folderQueue = [];
}
/**
* Copies a directory
* @param {String} remotePath
* @param {String} downloadPath
*/
async copyFolder(remotePath, downloadPath) {
// Add the folder to the queue
this.folderQueue.push({ remotePath, downloadPath });
// While the queue contains folders to download
while (this.folderQueue.length > 0) {
// Download them
const { remotePath, downloadPath } = this.folderQueue.shift();
console.log(`Copy directory: ${remotePath} to ${downloadPath}`);
await this._copyFolderAux(remotePath, downloadPath);
}
}
/**
* Private internal method which copies the files from a folder,
* but if it finds subfolders, simply adds them to the folderQueue
* @param {String} remotePath
* @param {String} downloadPath
*/
async _copyFolderAux(remotePath, downloadPath) {
await fs.mkdir(downloadPath);
const list = await this.listEntries(remotePath);
for (const fileInfo of list) {
if (fileInfo.isDirectory) {
const folderPath = Path.join(remotePath, fileInfo.name);
const targetPath = Path.join(downloadPath, fileInfo.name);
// Push the folder to the queue
this.folderQueue.push({ remotePath: folderPath, downloadPath: targetPath });
} else if (fileInfo.isFile) {
const filePath = Path.join(remotePath, fileInfo.name);
await this.copyFile(filePath, downloadPath, fileInfo);
}
}
}
/**
* Copies a file
* @param {String} filePath
* @param {String} downloadPath
* @param {Object} fileInfo
*/
async copyFile(filePath, downloadPath, fileInfo) {
const targetPath = Path.join(downloadPath, fileInfo.name);
console.log(`Copy file: ${filePath} to ${targetPath}`);
return await fs.copy(filePath, targetPath);
}
/**
* Lists entries from a folder
* @param {String} remotePath
*/
async listEntries(remotePath) {
const fileNames = await fs.readdir(remotePath);
return Promise.all(
fileNames.map(async name => {
const stats = await fs.lstat(Path.join(remotePath, name));
return {
name,
isDirectory: stats.isDirectory(),
isFile: stats.isFile()
};
})
);
}
}
module.exports = CopyManager;
推荐阅读
- javascript - 具有父组件操作的角度更改子组件模板
- python - AttributeError:“CollectionReference”对象没有属性“流”
- coq - 如何在不同的索引中找到价值
- firebase - 在更新数据期间未编辑时,firebase 数据库的值更新为 null
- javascript - 如何在等待响应时在网站中呈现加载页面
- java - 将 double[] 转换为不可变列表的成本更低的方法
- google-apps-script - 在目标工作表中运行编辑功能后添加时间戳-Google 表格
- angularjs - 如何在加载角度 1.4 数据表之前在列中添加锚标记
- html - 页面 CSS 宽度大于 100%
- kotlin - 在 Kotlin 中声明一个不是表达式的函数