python-3.x - 对大量设备并行请求 api

Question

我正在编写一个脚本来针对大量 ID（500 万）并行运行 api，我希望在文件中输出以下是我的脚本，它适用于少量 ID，但我想获得一个解决方案来运行大量ID：

import requests
import os
import urllib
import json



class XRAYUtils:

    @classmethod
    def get_New_Account_Info(cls,Id:str):
        master_dict = {}
        list = []
        try:
            url = "https://search-api" + billingId + "?xyz"

            auth_t = 'JwSD62FiayGDFx4K8'
            headers = {
                'authorization': "yyyyy" + auth_t,
                'content-type': "application/json",
            }
            try:
                init_res = requests.get(url, headers=headers, allow_redirects=False)
                response = init_res.json()
            # below code is to parse result json
            for item in response['data']['items']:
                    master_dict = item
                if not master_dict: # if None
                    return {'m': bId}
                else:
                    for dictn in master_dict['devices']:
                        return {'x1Eac': dictn['ac'], 'x1DId': dictn['x1Id'], 'x1Mod': dictn['x1Mod'], 'x1Status': dictn['x1Status']}
            except Exception as ce:
                print(ce)
        except Exception as e:
            print(e)


if __name__ == '__main__':
    r = XRAYUtils()
    from multiprocessing import Pool
    mac_list = [line.rstrip('\n') for line in open('/mnt/backup/ICAM2_Mac.txt')] # has 5M IDs
    # mac_list = ['4657618' ,'58686735',....]
    pl = Pool()
    result = pl.map(r.get_New_Account_Info, mac_list)
    with open('/mnt/backup/result.txt', 'w') as fout:
        json.dump(result, fout)

标签： python-3.xpython-requestspython-3.6urllib