python - 使用列表作为输入在 Python 中运行进程时出错
问题描述
我试图让这段代码工作：
代码
进程运行方法
def process(tetrahedrons, startOfWork, endOfWork, RelativeResult):
print("Start of the process", mp.current_process().name)
result = False
for c in range(startOfWork,endOfWork):
for j in range(c+1,len(tetrahedrons)):
if tetrahedrons[c].IsInterpenetrated(tetrahedrons[j]) == 1:
result = True
print("Relative result = ",result, end="\n\n")
RelativeResult.put(result)
并行搜索方法
@staticmethod
def parallelInterpenetrationSearchWithProcess(tetrahedrons):
'''
parallel verification of the existence of an interpenetration between tetrahedra
'''
time_start = datetime.datetime.now()
N_CORE = mp.cpu_count()
RelativeResult = mp.Queue()
Workers = []
dimList = len(tetrahedrons)
QuantityForWorker = int(dimList / N_CORE)
Rest = dimList % N_CORE
StartWork = 0
if QuantityForWorker == 0:
iterations = Rest -1
else:
iterations = N_CORE
for i in range(iterations):
EndWork = StartWork + QuantityForWorker
if i < Rest -1:
EndWork = EndWork + 1
IdWork = i
Workers.append(mp.Process(target= process, args=(tetrahedrons, StartWork, EndWork, RelativeResult,)))
Workers[IdWork].start()
StartWork = EndWork
for worker in Workers:
worker.join()
while not(RelativeResult.empty()):
if RelativeResult.get():
print("Parallel search took %f seconds" % ( timeelapsed(time_start) ))
return True
print("Parallel search took %f seconds" % ( timeelapsed(time_start) ))
return False
这段代码的作用：并行搜索的静态方法将“四面体”对象列表作为输入，并检查这些元素之间是否验证了某个属性，这种情况将是搜索两个四面体之间的交集。
并行搜索方法由 main 方法调用，该方法构建四面体列表，然后将其传递给并行搜索方法以验证上述属性。我尝试使用包含 100 个元素的四面体列表测试此代码，它工作正常，但是当我尝试使用更高的输入“特别是 160000 个元素”运行它时，我收到以下错误：
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
所以我认为问题可能在于当前参数的使用，因为要将包含 160000 个元素的列表传递给一个进程，后者会将这个列表复制到内存中，并对 4 或 8 个进程执行此过程，将不是最大的。深信这个解决方案，然后我做了这样一种方式，即传递列表的一部分，每个进程只检查一个元素，但这种方法也给出了负面结果。所以我尝试在 multiprocessing.Manager() 的帮助下在进程之间共享列表。确切地：
@staticmethod
def parallelInterpenetrationSearchWithProcess(tetrahedrons):
'''
parallel verification of the existence of an interpenetration between tetrahedra
'''
time_start = datetime.datetime.now()
N_CORE = mp.cpu_count()
RelativeResult = mp.Queue()
Workers = []
trList = mp.Manager().list(tetrahedrons)
dimList = len(tetrahedrons)
QuantityForWorker = int(dimList / N_CORE)
Rest = dimList % N_CORE
StartWork = 0
if QuantityForWorker == 0:
iterations = Rest -1
else:
iterations = N_CORE
for i in range(iterations):
EndWork = StartWork + QuantityForWorker
if i < Rest -1:
EndWork = EndWork + 1
IdWork = i
Workers.append(mp.Process(target= process, args=(trList, StartWork, EndWork, RelativeResult,)))
Workers[IdWork].start()
StartWork = EndWork
for worker in Workers:
worker.join()
while not(RelativeResult.empty()):
if RelativeResult.get():
print("Parallel search took %f seconds" % ( timeelapsed(time_start) ))
return True
print("Parallel search took %f seconds" % ( timeelapsed(time_start) ))
return False
但是这种方法给了我以下错误：
Traceback (most recent call last):
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "c:\Users\85612\.vscode\extensions\ms-python.python-2020.11.371526539\pythonFiles\lib\python\debugpy\__main__.py", line 45, in <module>
cli.main()
File "c:\Users\85612\.vscode\extensions\ms-python.python-2020.11.371526539\pythonFiles\lib\python\debugpy/..\debugpy\server\cli.py", line 430, in main
run()
File "c:\Users\85612\.vscode\extensions\ms-python.python-2020.11.371526539\pythonFiles\lib\python\debugpy/..\debugpy\server\cli.py", line 267, in run_file
runpy.run_path(options.target, run_name=compat.force_str("__main__"))
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\runpy.py", line 263, in run_path
pkg_name=pkg_name, script_name=fname)
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "c:\Users\85612\Documents\GMarco2\tet\main.py", line 19, in <module>
out = mesh.mesh.parallelInterpenetrationSearchWithProcess(trList)
File "c:\Users\85612\Documents\GMarco2\tet\mesh\mesh.py", line 264, in parallelInterpenetrationSearchWithProcess
trList = mp.Manager().list(tetrahedrons)
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\multiprocessing\managers.py", line 701, in temp
token, exp = self._create(typeid, *args, **kwds)
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\multiprocessing\managers.py", line 586, in _create
id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\multiprocessing\managers.py", line 78, in dispatch
c.send((id, methodname, args, kwds))
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\multiprocessing\connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "C:\Users\85612\AppData\Local\Programs\Python\Python37\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
RecursionError: maximum recursion depth exceeded while pickling an object
在第一种出现错误的情况下：EOFError: Ran out of input似乎试图从文件中读取。我不知道这条消息是否是由于 Python 是一种解释语言。我已经阅读了很多关于它的不同解决方案的问题，但没有一个可以帮助我。但是，我注意到在许多解决方案中都提到了Pickle 。据我了解，Pickle 本质上是将 Python 对象的层次结构转换为字节流。我是 python 新手，所有这些信息都造成了一些混乱。
问题
总之，我想知道我的问题可能是什么，如果我在某些事情上做错了，或者我是否遗漏了一些能够使我的代码工作的难题。如果可能的话，也欢迎简要解释 Pickle 在我的代码中所扮演的角色。提前致谢。
浏览有关 pickle 的在线文档，我注意到该模块用于多处理信息传递。所以我的疑问是我的课程被 pickle 管理得很差，或者更确切地说他们不能被 pickle。我正在尝试验证此声明是否正确，但是在使我的代码易于消化以进行泡菜时，我遇到了很多困难。
这是我的四面体课
def handedness(vertices):
"""
returns the handedness of a list of vertices
"""
# compute handedness
c = np.cross(vertices[1].coords-vertices[0].coords , vertices[2].coords-vertices[0].coords)
h = np.inner( c, vertices[3].coords-vertices[0].coords )
return h
class tetrahedron:
"""
a tetrahedron
"""
def __init__(self, vertices):
"""
initialize a tetrahedron
"""
# compute handedness
h = handedness(vertices)
# vertex indices with correct right-handed orientation
if h > 0:
self.vertices = vertices
else:
s = "old h = %f" % (h)
# swap last two vertices
self.vertices = [vertices[0], vertices[1], vertices[3], vertices[2]]
vertices = self.vertices
h = handedness(vertices)
s = s + " new h = %f" % (h)
print(s)
这是我的顶点类
class vertex:
"""
a vertex
"""
def __init__(self, coords, idx):
"""
initialize a vertex
"""
# vertex coordinates
self.coords = numpy.array(coords)
# incident triangles
self.triangles = []
# id
self.idx = idx
解决方案
推荐阅读
- python - 如何从python中的字符串中提取国家
- android - 我的 Firebase 无法上传到实时数据库？安卓科特林
- python - 如何从多边形几何中获取坐标数组
- sql - 从（本身左连接其他表）更新表
- visual-studio-code - VSCode - 如何删除文件夹内文件夹的正斜杠？
- kotlin - Klaxon 解析 null
- pandas - 如何清理此数据框？
- java - Java，为什么不自引用类分配无限内存？
- python - 在 Python 中同时执行 json.load() 和 json.dump()
- javascript - 滚动已经开始后，Javascript在touchmove上停止滚动