google-cloud-platform - 提高 AWS S3 文件到 Google Cloud Storage 的传输速度

Question

试图从亚马逊 s3 下载一个巨大的数据集。文件大小约为 500Gigs，我有一个 python 脚本可以将它下载到我的 GCP 虚拟机上。但是，初始下载速度约为 30-50mb/s，然后下降到 1mbps。有没有办法提高下载速度？

from tqdm.auto import tqdm
import requests
import errno
import tarfile
import os

def download(url, save_dir):
    filename = url.rsplit('/', 1)[1]

    if not os.path.exists(os.path.dirname(save_dir)):
        try:
            os.makedirs(save_dir)
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

    with open(f'{save_dir}/{filename}', 'wb+') as f:
        response = requests.get(url, stream=True)
        total = int(response.headers.get('content-length'))

        if total is None:
            f.write(response.content)
        else:
            with tqdm(total=total, unit='B', unit_scale=True, desc=filename) as pbar:
                for data in tqdm(response.iter_content(chunk_size=1024)):
                    f.write(data)
                    pbar.update(1024)


TRAIN_CSV = 'https://s3.amazonaws.com/google-landmark/metadata/train.csv'
TRAIN_ATTRIBUTION_CSV = 'https://s3.amazonaws.com/google-landmark/metadata/train_attribution.csv'
TAR_URLS = [f'https://s3.amazonaws.com/google-landmark/train/images_{ ("00" + str(n))[-3:] }.tar' for n in range(0,500)]

SAVE_DIR = 'train'
NUM_TARS_TO_DL = 1
UNTAR = False

print('DOWNLOADING train.csv')
download(TRAIN_CSV, SAVE_DIR)

print('\nDOWNLOADING train_attribution.csv')
download(TRAIN_ATTRIBUTION_CSV, SAVE_DIR)

print('\nDOWNLOADING image tar files')
for url in tqdm(TAR_URLS[:NUM_TARS_TO_DL]):
    download(url, SAVE_DIR)

if UNTAR:
    print('\n UNTARRING image tar files')
    for filename in tqdm(os.listdir(SAVE_DIR)):
        if filename.endswith('.tar'):
            img_tar = tarfile.open(f'{SAVE_DIR}/{filename}')
            img_tar.extractall(path=SAVE_DIR)

标签： google-cloud-platformcloud

Answer 1

我意识到如果我使用 pythons 多核包并使用每个核心下载，它会显着增加下载时间。