python - uwsgi进程占用大量内存

Question

我正在使用Django+ uwsgi，以下是一些主要设置

uwsgi

[uwsgi]
pythonpath=/usr/local/server
chdir=/home/server
env=DJANGO_SETTINGS_MODULE=conf.settings
module=server.wsgi
master=True
pidfile=logs/server.pid
vacuum=True
max-requests=1000
enable-threads=true
processes = 4
threads=8
listen=1024
daemonize=logs/wsgi.log
http=0.0.0.0:16020
buffer-size=32000

当我尝试从服务器获取 excel 文件时，一个uwsgi进程的内存正在快速增长，几秒钟后，浏览器Nginx 504超时但内存仍在增长。

该文件是由我的数据库中的行数据30M生成的。450k

top -p 866

PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
866 soe       20   0 7059m 5.3g 5740 S 100.8 33.9   4:17.34 uwsgi --ini /home/smb/work/soe_server

Main logic codes

from openpyxl.writer.excel import save_virtual_workbook
from django.http import HttpResponse
...

class ExcelTableObj(object):
    def __init__(self, file_name=None):
        if file_name:
            self.file_name = file_name
            self.wb = load_workbook(file_name)
        else:
            self.wb = Workbook()

    def create_new_sheet(self, title='Sheet1'):
        new_ws = self.wb.create_sheet(title=title)

    def write_to_sheet(self, sheetname, datas, filename):
        ws = self.wb[sheetname]
        for data in datas:
            ws.append(data)
        self.wb.save(filename)

    def update_sheet_name(self, sheetname):
        ws = self.wb.active
        ws.title = sheetname

    def append_data_to_sheet(self, sheetname, data):
        ws = self.wb[sheetname]
        ws.append(data)

    def save_file(self, file_name):
        self.wb.save(file_name)
        self.wb.close()

    def get_upload_file_data(self, name=None):
        if name:
            ws = self.wb.get_sheet_by_name(name)
        else:
            ws = self.wb.worksheets[0]
        rows = ws.max_row
        cols = ws.max_column
        file_data = []   
        fields = []   
        for i in range(1, cols+1):
            cell = ws.cell(row=1, column=i)
            if cell.value:
                fields.append(cell.value.lower().strip())
        for row in range(2, rows + 1):
            row_data = {}
            for j in range(len(fields)):
                value = ws.cell(row=row, column=j+1).value
                if value:
                    row_data[fields[j]] = str(value).strip()
            if row_data:
                file_data.append(row_data)
        return file_data

    def get_sheet_maxrow(self, name):
        ws = self.wb.get_sheet_by_name(name)
        rows = ws.max_row
        return rows

def _get_download_data(datas):
    for data in queryset :
    ...
    item = [str(data.account_id),
                    ILLEGAL_CHARACTERS_RE.sub(r'', data.account_name) if data.account_name else data.account_name,
                    type, fb_aac_conf.FB_ACCOUNT_STATUS[data.account_status],
                    data.submitter, data.submit_time, data.confirmor, data.confirm_time,
                    fb_aac_conf.BATCH_STATUS[data.status], data.reason, data.entity_name, data.payment_name,
                    data.sale, data.ae_note, urgent
                    ]
    yield item

queryset = MyModel.objects.filter(...) # about `450k` rows

datas = _get_download_data(queryset)

excel = ExcelTableObj()
excel.update_sheet_name(sheetname)
excel.append_data_to_sheet(sheetname, title)
excel.write_to_sheet(sheetname, datas, filename)
excel.save_file(filename)

response = HttpResponse(save_virtual_workbook(excel.wb),
                                    content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
            response['Content-Disposition'] = 'attachment; filename={}'.format(filename)

为什么内存增长如此之快，我该如何避免这个问题？

谢谢

标签： pythondjangoopenpyxluwsgi