node.js - 如何使用节点js在本地机器上下载aws S3文件夹中的所有文件

Question

我有一个 S3 存储桶，其中有一个包含一些文件的文件夹，我想将该文件夹中的所有文件下载到本地计算机文件夹中，我尝试了单个文件，它正在工作如何下载多个文件。

根据关键文件夹A中的以下代码有10个文件，我想将所有十个文件下载到我在 s3.getObject(params).createReadStream().pipe(ws); 中提到的本地文件夹目录；

我的代码：

const downloadObject = () => {
  var params = { Bucket: "Sample", Key:"folderA/"};
  const ws = fs.createWriteStream(`${__dirname}/localfolder/`);
  const s3Stream = s3.getObject(params).createReadStream().pipe(ws);
  s3Stream.on("error", (err) => {
    ws.end();
  });
  s3Stream.on("close", () => {
    console.log(`downloaded successfully from s3 at ${new Date()}`);
    ws.end();
  });
};

预期输出：

s3 -> bucket/folderA/<10 个文件> localmachine -> localfolder/<需要本地所有 10 个文件>

标签： node.jsamazon-s3

Answer 1

它有很多，

首先，您需要列出所有存储桶，然后遍历所有存储桶（如果您只想要一个）。如果找不到，请创建一个本地目录等。

然后找出存储桶中的所有文件，然后循环遍历它们，在每个路径上获取对象并存储它。

以下是如何使用minio js 客户端（调用相同）根据您的需要进行调整，显然文件夹路径会有所不同。

/**
 * S3 images pull script
 */

const fs = require('fs')
const path = require('path')
const util = require('util')
const readFile = util.promisify(fs.readFile)
const writeFile = util.promisify(fs.writeFile)

//
const rootPath = path.join(__dirname, '..')
const publicPath = path.join(rootPath, 'public', 'images')

//
require('dotenv').config({
  path: path.join(rootPath, '.env')
})

// minio client S3
const s3 = new(require('minio')).Client({
  endPoint: process.env.S3_HOST,
  port: parseInt(process.env.S3_PORT, 10),
  useSSL: process.env.S3_USE_SSL === 'true',
  accessKey: process.env.S3_ACCESS_KEY,
  secretKey: process.env.S3_ACCESS_SECRET,
  region: process.env.S3_REGION
})

/**
 * Functions
 */

const mkdir = dirPath => {
  dirPath.split(path.sep).reduce((prevPath, folder) => {
    const currentPath = path.join(prevPath, folder, path.sep);
    if (!fs.existsSync(currentPath)) {
      fs.mkdirSync(currentPath);
    }
    return currentPath
  }, '')
}

// list objects in bucket
const listObjects = bucket => new Promise(async (resolve, reject) => {
  //
  bucket.objects = []
  bucket.total_objects = 0
  bucket.total_size = 0
  //
  let stream = await s3.listObjectsV2(bucket.name, '', true)
  //
  stream.on('data', obj => {
    if (obj && (obj.name || obj.prefix)) {
      bucket.objects.push(obj)
      bucket.total_objects++
      bucket.total_size = bucket.total_size + obj.size
    }
  })
  //
  stream.on('end', () => resolve(bucket))
  stream.on('error', e => reject(e))
})

// get an objects data
const getObject = (bucket, name) => new Promise((resolve, reject) => {
  s3.getObject(bucket, name, (err, stream) => {
    if (err) reject(err)
    //
    let chunks = []
    stream.on('data', chunk => chunks.push(chunk))
    stream.on('end', () => resolve(Buffer.concat(chunks || [])))
    stream.on('error', e => reject(e))
  })
})

/**
 *
 */
async function main() {

  // get buckets
  console.log(`Fetching buckets from: ${process.env.S3_HOST}`)
  let buckets = []
  try {
    buckets = await s3.listBuckets()

    console.log(buckets.length + ' buckets found')
  } catch (e) {
    return console.error(e)
  }

  // create local folders if not exists
  console.log(`Creating local folders in ./api/public/images/ if not exists`)
  try {
    for (let bucket of buckets) {
      //
      bucket.local = path.join(publicPath, bucket.name)
      try {
        await fs.promises.access(bucket.local)
      } catch (e) {
        if (e.code === 'ENOENT') {
          console.log(`Creating local folder: ${bucket.local}`)
          await fs.promises.mkdir(bucket.local)
        } else
          bucket.error = e.message
      }
    }
  } catch (e) {
    return console.error(e)
  }

  // fetch all bucket objects
  console.log(`Populating bucket objects`)
  try {
    for (let bucket of buckets) {
      bucket = await listObjects(bucket)
    }
  } catch (e) {
    console.log(e)
  }

  // loop over buckets and download all objects
  try {
    for (let bucket of buckets) {
      console.log(`Downloading bucket: ${bucket.name}`)
      // loop over and download
      for (let object of bucket.objects) {
        // if object name has prefix
        let dir = path.dirname(object.name)
        if (dir !== '.') {
          try {
            await fs.promises.access(path.join(bucket.local, dir))
          } catch (e) {
            if (e.code === 'ENOENT') {
              console.log(`Creating local folder: ${bucket.local}`)
              mkdir(path.join(bucket.local, dir))
            }
          }
        }

        //
        console.log(`Downloading object[${bucket.name}]: ${object.name}`)
        await writeFile(path.join(bucket.local, object.name), await getObject(bucket.name, object.name))
      }
    }

    console.log(`Completed!`)
  } catch (e) {
    console.log(e)
  }
}

main()