python-3.x - How to Create a tar file containing all the files in a directory

Question

I have been trying to figure out how to generate a tar file of a directory of files. I have this code

tar = tarfile.open('/tmp/' + newDate + '.tar', 'w')
for fname in get_matching_s3_keys(bucket=agtBucket, prefix=key, suffix='.log'):
    print(fname)
    file_obj = s3object.Object(agtBucket, fname)
    file_content = file_obj.get()['Body'].read()
    tar.add(file_content)
tar.close()

But I get this error when I try to add file_content to tar

"errorMessage": "a bytes-like object is required, not 'str'"

I hope someone can please help me correct what I have wrong.

标签： python-3.xamazon-web-servicesaws-lambda

Answer 1

Here is answer :

import boto3 
import tarfile
import os.path

s3Client = boto3.client('s3')
s3object = boto3.resource('s3')

def lambda_handler(event, context):
    agtBucket = "angularbuildbucket"
    key=""
    tar = tarfile.open('/tmp/example.tar', 'w')
    source_dir="/tmp/"
    for fname in get_matching_s3_keys(bucket=agtBucket, prefix=key, suffix='.log'):
        print(fname)
        file_obj = s3object.Object(agtBucket, fname)
        #file_content = file_obj.get()['Body'].read()
        #tar.add(file_content)
        s3object.Bucket(agtBucket).download_file(fname, '/tmp/'+fname)
        tar.add(source_dir, arcname=os.path.basename(source_dir))
    tar.close()
    s3object.meta.client.upload_file(source_dir+"example.tar", agtBucket, 'example.tar')
    
    
    
def get_matching_s3_keys(bucket, prefix='', suffix=''):
    """
    Generate the keys in an S3 bucket.

    :param bucket: Name of the S3 bucket.
    :param prefix: Only fetch keys that start with this prefix (optional).
    :param suffix: Only fetch keys that end with this suffix (optional).
    """
    kwargs = {'Bucket': bucket, 'Prefix': prefix}
    while True:
        resp = s3Client.list_objects_v2(**kwargs)
        for obj in resp['Contents']:
            key = obj['Key']
            if key.endswith(suffix):
                yield key

        try:
            kwargs['ContinuationToken'] = resp['NextContinuationToken']
        except KeyError:
            break

You are getting this error because tar.add() expect object as input to add into buffer for compressing objects. But while doing file_obj.get()['Body'].read() getting content of file in string format.