python - pathos 多进程无法写入共享内存
问题描述
我正在尝试更改代码(来自此示例)以使用 pathos 多进程而不是多进程。示例代码运行良好。用多进程替换多进程无法更改存储在共享内存中的数组并引发断言错误。下面是我修改的代码。
import ctypes
import logging
import multiprocess as mp
from contextlib import closing
import numpy as np
info = mp.get_logger().info
def main():
logger = mp.log_to_stderr()
logger.setLevel(logging.INFO)
# create shared array
N, M = 100, 11
shared_arr = mp.Array(ctypes.c_double, N)
arr = tonumpyarray(shared_arr)
# fill with random values
arr[:] = np.random.uniform(size=N)
print(arr)
arr_orig = arr.copy()
# write to arr from different processes
with closing(mp.Pool(initializer=init, initargs=(shared_arr,))) as p:
# many processes access the same slice
stop_f = N // 10
p.map_async(f, [slice(stop_f)]*M)
# many processes access different slices of the same array
assert M % 2 # odd
step = N // 10
p.map_async(g, [slice(i, i + step) for i in range(stop_f, N, step)])
p.join()
print(arr)
assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig)
def init(shared_arr_):
global shared_arr
shared_arr = shared_arr_ # must be inherited, not passed as an argument
def tonumpyarray(mp_arr):
return np.frombuffer(mp_arr.get_obj())
def f(i):
"""synchronized."""
with shared_arr.get_lock(): # synchronize access
g(i)
def g(i):
"""no synchronization."""
info("start %s" % (i,))
arr = tonumpyarray(shared_arr)
arr[i] = -1 * arr[i]
info("end %s" % (i,))
if __name__ == '__main__':
mp.freeze_support()
main()
这是演示不良行为的脚本输出(共享数组未更改)。
[INFO/MainProcess] allocating a new mmap of length 4096
[7.53982236e-01 8.91817533e-01 3.42952411e-01 4.81562566e-01
8.11524911e-01 9.39701276e-01 2.49035240e-01 9.81663723e-01
8.08235711e-01 1.42681227e-01 3.45497140e-01 8.90562288e-01
4.35241873e-01 4.60578813e-01 3.46264261e-01 4.93318426e-01
3.22399114e-01 7.83147144e-01 9.47066783e-02 5.18534236e-01
8.51086344e-01 6.27441844e-01 1.06383034e-01 2.38705882e-01
7.97136179e-01 7.86004956e-01 8.66374084e-01 5.17888509e-01
8.17624705e-01 9.49644794e-01 6.31860201e-01 5.36854341e-01
3.04143232e-01 3.72530617e-01 2.30401250e-01 8.07846412e-02
9.07925558e-01 2.83202034e-01 2.29512224e-01 1.10851201e-01
7.72062029e-01 3.60300517e-01 5.54688922e-04 5.65488687e-01
5.62561777e-01 6.34814941e-01 5.96339057e-01 9.79598924e-01
2.09899979e-01 7.15104955e-01 6.55292670e-01 1.68393973e-01
9.72467624e-01 8.35882816e-01 7.43016913e-01 4.94503316e-01
4.64440104e-01 9.19258107e-01 8.77918280e-01 3.50650473e-02
1.78977134e-02 8.37892866e-01 3.97096237e-01 3.98471290e-01
1.50104963e-01 6.85465476e-01 8.44218457e-01 5.62325961e-01
1.10385518e-01 3.60357701e-01 3.57917840e-01 8.63808991e-01
8.39141018e-01 1.79879196e-01 8.02565416e-01 6.78710916e-01
9.40750906e-01 6.95072638e-01 4.07945520e-01 8.04220531e-01
9.36671866e-01 8.04467890e-01 2.71010169e-01 6.12931552e-02
9.13496687e-01 9.94252838e-03 6.29349539e-01 7.82656394e-02
6.58311027e-01 9.18628747e-01 9.21899148e-01 6.34954946e-03
9.87928873e-01 2.35952899e-01 6.93470029e-01 9.41543773e-01
2.27509027e-01 7.76581113e-01 5.76203661e-01 7.70004310e-01]
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[7.53982236e-01 8.91817533e-01 3.42952411e-01 4.81562566e-01
8.11524911e-01 9.39701276e-01 2.49035240e-01 9.81663723e-01
8.08235711e-01 1.42681227e-01 3.45497140e-01 8.90562288e-01
4.35241873e-01 4.60578813e-01 3.46264261e-01 4.93318426e-01
3.22399114e-01 7.83147144e-01 9.47066783e-02 5.18534236e-01
8.51086344e-01 6.27441844e-01 1.06383034e-01 2.38705882e-01
7.97136179e-01 7.86004956e-01 8.66374084e-01 5.17888509e-01
8.17624705e-01 9.49644794e-01 6.31860201e-01 5.36854341e-01
3.04143232e-01 3.72530617e-01 2.30401250e-01 8.07846412e-02
9.07925558e-01 2.83202034e-01 2.29512224e-01 1.10851201e-01
7.72062029e-01 3.60300517e-01 5.54688922e-04 5.65488687e-01
5.62561777e-01 6.34814941e-01 5.96339057e-01 9.79598924e-01
2.09899979e-01 7.15104955e-01 6.55292670e-01 1.68393973e-01
9.72467624e-01 8.35882816e-01 7.43016913e-01 4.94503316e-01
4.64440104e-01 9.19258107e-01 8.77918280e-01 3.50650473e-02
1.78977134e-02 8.37892866e-01 3.97096237e-01 3.98471290e-01
1.50104963e-01 6.85465476e-01 8.44218457e-01 5.62325961e-01
1.10385518e-01 3.60357701e-01 3.57917840e-01 8.63808991e-01
8.39141018e-01 1.79879196e-01 8.02565416e-01 6.78710916e-01
9.40750906e-01 6.95072638e-01 4.07945520e-01 8.04220531e-01
9.36671866e-01 8.04467890e-01 2.71010169e-01 6.12931552e-02
9.13496687e-01 9.94252838e-03 6.29349539e-01 7.82656394e-02
6.58311027e-01 9.18628747e-01 9.21899148e-01 6.34954946e-03
9.87928873e-01 2.35952899e-01 6.93470029e-01 9.41543773e-01
2.27509027e-01 7.76581113e-01 5.76203661e-01 7.70004310e-01]
这是完整的追溯。
Traceback (most recent call last):
File "C:\Users\Fernando\AppData\Local\Continuum\anaconda3\envs\fountaintx\lib\site-packages\IPython\core\interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-2-4f9235b548aa>", line 1, in <module>
runfile('C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py', wdir='C:/Users/Fernando/.PyCharm2019.2/config/scratches')
File "C:\Program Files\JetBrains\PyCharm Professional Edition with Anaconda plugin 2019.2.4\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
pydev_imports.execfile(filename, global_vars, local_vars) # execute the script
File "C:\Program Files\JetBrains\PyCharm Professional Edition with Anaconda plugin 2019.2.4\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py", line 60, in <module>
main()
File "C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py", line 37, in main
assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig)
AssertionError
解决方案
推荐阅读
- reactive-programming - 反应式单声道如何将订阅上下文传播到`doOnSubscribe`和`doFinally`
- jquery - jQuery 和 CSS:侧边栏应该在 flexbox 内容之上推出,而不影响主要区域
- c++ - 摆脱“未声明的标识符”错误
- html - 为什么我的 HTML 表单显示奇怪的符号?
- c# - C#程序返回通过Active Directory连接的开机机器
- python - 在python中添加列表的意外结果
- javascript - 为什么我们在 javascript 中的 .split 函数之前放置 ''
- javascript - 如何检测用户是否滑动浏览器后退导航
- vue.js - Nuxt 2.12.2:使用新的 fetch 方法填充存储
- python - 按每周间隔创建时间序列熊猫数据框的多个子集