c++ - 使用 aws-sdk-cpp 上传文件列表

Question

我正在尝试使用 cpp sdk 在 aws s3 上上传文件列表。但是只有少数文件正在上传，并且在上传所有文件之前就被终止了。我正在使用 aws s3 cpp sdk 的上传 api。

 bool OnUpload_test(bool status, const char* carouselname, void* userdata){
        cout << "status" << status<< endl;
        cout << "Objectname =" << carouselname << endl;
        return true;

}


bool upload_thread(Proxy* proxy, const char* file,const char* path, int duration)
{
        char output_file[512];
        snprintf(output_file,512,"%s/%s",path,file);
        proxy->Upload(file, output_file,duration,OnUpload_test,nullptr);
        return true;
}

int main(){
     
        char filename[9][128];
        int i = 0, num_thread = 10,j=0,k=0;

        thread *thread_count[num_thread];
        Proxy *m_proxy = new Proxy("123","bucket_name",20);

        //creating thread to test upload
        for (int i = 1, j = 0; i<=9; i++, j++)
        {
             
                snprintf(filename[j],128,"centos%d.mp4",i);
                cout << "count = " << j << endl;

                thread_count[i] = new thread(upload_thread, m_proxy,(const char*) filename[j],"path",20);
           

        }
        for (i = 1; i<=9; i++)
        {
                thread_count[i]->join();
               

        }
       sleep(120);
}

上传函数 s3 Api 将数据上传到 s3。文件大小平均为 40 mb。我正在尝试上传 9 个文件。

///////////////////////////////////////// /////////////////

 bool UploadData::Uploaddata(string bucketName, string objname, string 
           objdata,int duration,const char* id)
{


        Aws::String bcktName(bucketName.c_str(), bucketName.size());
        Aws::String obj(objname.c_str(), objname.size());
        Aws::String objdt(objdata.c_str(), objdata.size());

        Aws::Client::ClientConfiguration config;
        config.connectTimeoutMs = 5000000;
        config.requestTimeoutMs = 6000000;

        const Aws::String user_region = "ap-south-1";
        config.region = user_region;
        Aws::S3::S3Client client(config);

        Aws::S3::Model::PutObjectRequest request;

        request.SetBucket(bcktName);
        request.SetKey(obj);

        
        const std::shared_ptr<Aws::IOStream> input_data = Aws::MakeShared<Aws::FStream>(obj.c_str(),objdt.c_str(),std::ios_base::in | std::ios_base::binary);
        

        input_data->seekg(0, input_data->end);

        int fsize = input_data->tellg();
        input_data->seekg(0, input_data->beg);
        
        request.SetContentLength(fsize);
        request.SetBody(input_data);


        auto outcome = client.PutObject(request);
        if (!outcome.IsSuccess()) {
                std::cout <<"UploadObject error = "<< outcome.GetError().GetMessage().c_str() << std::endl;
                return false;
        }
}

标签： c++amazon-web-servicesamazon-s3aws-sdk-cpp

Answer 1

由于您没有发布代码的其他相关部分，例如 S3 客户端配置，我只能猜测.. 也许超时还不够？尝试在客户端增加requestTimeoutMs和connectTimeoutMs配置。

编辑

您的代码不是 mwe（例如..什么是代理？）。AWS SDK 没有初始化和关闭。

这就是使用线程对我有用的方法：

#include <iostream>
#include <fstream>
#include <thread>
#include <vector>
#include <unistd.h>
#include <aws/core/Aws.h>
#include <aws/core/auth/AWSCredentialsProvider.h>
#include <aws/core/utils/StringUtils.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/s3/model/Bucket.h>

using namespace std;
using namespace Aws;

const Aws::String AWS_ACCESS_KEY_ID = "<your ACCESS KEY>";
const Aws::String AWS_SECRET_ACCESS_KEY = "<your SECRET ACCESS KEY>";

bool Uploaddata(string bucketName, string objname)
{


       Aws::String bcktName = Aws::String (bucketName.c_str());
       Aws::String obj = Aws::String (objname.c_str());

       Aws::Client::ClientConfiguration config;
       config.scheme = Aws::Http::Scheme::HTTPS;
       config.connectTimeoutMs = 5000000;
       config.requestTimeoutMs = 6000000;
       config.region = Aws::Region::EU_WEST_1;

       Aws::S3::S3Client client(Aws::Auth::AWSCredentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), config);


       Aws::S3::Model::PutObjectRequest request;

       request.SetBucket(bcktName);
       request.SetKey(obj);

       auto requestStream = Aws::MakeShared<Aws::FStream>("PutObjectInputStream", obj.c_str(), std::ios_base::in | std::ios_base::binary);
       request.SetBody(requestStream);

       auto outcome = client.PutObject(request);
       if (!outcome.IsSuccess()) {
               std::cout <<"UploadObject error = "<< outcome.GetError().GetMessage().c_str() << std::endl;
               return false;
       }
}


int main(){

       char filename[9][128];
       std::string name_bucket = "<your bucket>";
       std::vector<std::thread> threads;

       Aws::SDKOptions options;
       Aws::InitAPI(options);
       {
           //creating thread to test upload
           for (int j = 1; j<=9; j++)
           {
                   snprintf(filename[j],128,"centos%d.txt",j);
                   cout << "count = " << j << endl;
                   threads.push_back(std::thread(Uploaddata,name_bucket,filename[j]));

           }
          for (auto& th : threads) th.join();
          sleep(120);
       }
       Aws::ShutdownAPI(options);
}