python - C_Python 不释放缓冲内存
问题描述
我正在为python(Python C API)编写C代码,我注意到python没有释放文件的内存,我想知道问题是否出在我的代码中。
我想尽可能简化,但我希望不会遗漏任何细节。
该文件是带有缓冲区的二进制文件,前 4 个字节是缓冲区大小,然后是缓冲区。
二进制文件(big_file.comp):
du ~/Desktop/TEST_FILES/big_file.comp
4175416 ~/Desktop/TEST_FILES/big_file.comp
python代码(test.py):
#!/usr/bin/env python3
from struct import unpack_from
from psutil import Process
from os import getpid
import decomplib
def file_handler(file_name):
with open(file_name, 'rb') as reader:
while True:
next_4_bytes = reader.read(4)
if next_4_bytes == b'':
break
next_size, *_ = unpack_from("I", next_4_bytes)
buffer = reader.read(next_size)
yield buffer, next_size
def main():
args = _parse_args()
decompress = decomplib.Decompress()
for buf, buf_size in file_handler(args.file):
for msg in decompress.decompress_buffer(buf, buf_size):
print(msg)
if __name__ == "__main__":
pid = getpid()
ps = Process(pid)
main()
print(ps.memory_info())
一些 C 代码简化:
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
uint32_t arr_size;
} DecompressObject;
static int Decompress_init(DecompressObject *self, PyObject *args, PyObject *kwds){
return 0;
}
static PyObject* Decompress_handle_buffer(DecompressObject* self, PyObject* args){
uint32_t buf_size = 0;
uint8_t *buf = NULL;
// get buffer and buffer length from python function
if(!PyArg_ParseTuple(args, "y*i", &buf, &buf_size)){
PyErr_SetString(PyExc_Exception, "Failed to parse function arguments");
return NULL;
}
self->arr_size = 10;
Py_XINCREF(self);
return (PyObject *) self;
}
static PyObject* Decompress_next(DecompressObject *self, PyObject *Py_UNUSED(ignored)){
static uint32_t seq_index = 0;
if (seq_index < self->arr_size) {
seq_index++;
Py_RETURN_NONE;
}
seq_index = 0;
return NULL;
}
static void Decompress_dealloc(DecompressObject *self){
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyMethodDef Decompress_methods[] = {
{"decompress_buffer", (PyCFunction) Decompress_handle_buffer, METH_VARARGS, "Decompress a buffer to asc data."},
{NULL} /* Sentinel */
};
static PyTypeObject DecompressType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "decomplib.Decompress",
.tp_doc = "Decompress object",
.tp_basicsize = sizeof(DecompressObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.tp_alloc = PyType_GenericAlloc,
.tp_new = PyType_GenericNew,
.tp_iter = PyObject_SelfIter,
.tp_init = (initproc) Decompress_init,
.tp_dealloc = (destructor) Decompress_dealloc,
.tp_iternext = (iternextfunc) Decompress_next,
.tp_methods = Decompress_methods,
};
static PyModuleDef Decompressmodule = {
PyModuleDef_HEAD_INIT,
.m_name = "decomplib",
.m_doc = "Decompress an compressed file.",
.m_size = -1,
};
PyMODINIT_FUNC PyInit_decomplib(void){
PyObject *d;
if (PyType_Ready(&DecompressType) < 0)
return NULL;
d = PyModule_Create(&Decompressmodule);
if (d == NULL)
return NULL;
Py_INCREF(&DecompressType);
if (PyModule_AddObject(d, "Decompress", (PyObject *) &DecompressType) < 0) {
Py_DECREF(&DecompressType);
Py_DECREF(d);
return NULL;
}
return d;
}
结果,我得到了以下输出:
./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=4349915136, vms=4412583936, shared=6270976, text=2867200, lib=0, data=4344135680, dirty=0)
在玩耍时,我注意到如果我在函数中更改对C
函数的第二个参数Decompress_handle_buffer
的调用to ,Python 会清理内存......PyArg_ParseTuple
"y*i"
"Si"
./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=22577152, vms=84869120, shared=6361088, text=2867200, lib=0, data=16420864, dirty=0)
但是,未正确读取缓冲区。
有任何想法吗?!
额外信息:
解决方案
y*
uint8_t
与您正在使用它不对应。如文档中所述,它填充了Py_buffer
您应该提供的结构。
您实际上需要提供一个Py_buffer,当您完成它时,您需要使用PyBuffer_Release
.
推荐阅读
- mysql - 如何写一个删除命令来替换三个sql?
- android - 在不同的机器上运行时,不变违规反应本机
- laravel - 服务器上的 Laravel 与 ip
- android - 如何在android中设置进度条0
- python - hello_world() 缺少 1 个必需的位置参数:“请求”
- jsf - 如何在选中元素的情况下呈现 selectManyCheckbox?
- python - Python lambda 函数是否有助于减少执行时间?
- java - 异常处理模板“searchresult”
- java - PKIX 路径构建失败:
- scenekit - [addressX] 处的 SCNParticleSystem 弱变量持有 [addressY]