python - 如何找出“无法腌制 '_io.BufferedReader' 对象”错误
问题描述
我正在尝试通过多处理下载插图。这几天我一直在为多处理而苦苦挣扎。请帮我解决这个问题
这里我的代码如下
from urllib.request import urlopen
from bs4 import BeautifulSoup
import urllib
import multiprocessing
import sys
from multiprocessing import Lock
import time
import pickle
path = 'D:/my_folder/illust'
num = 0
b = 0
tf = False
def get_links(numb):
time.sleep(1)
url = 'https://anime-pictures.net/pictures/view_posts/' + str(numb) + '?lang=en'
html = urlopen(url)
bsObject = BeautifulSoup(html, "html.parser")
urls = bsObject.find_all("span", class_="img_block_big")
print(len(urls))
return urls
def get_content(urls):
global b
imgSiteUrl = urls.find("a")["href"]
print(imgSiteUrl)
url_ = 'https://anime-pictures.net' + imgSiteUrl
html_ = urlopen(url_)
bsObject_ = BeautifulSoup(html_, "html.parser")
urls_ = bsObject_.find("div", id="big_preview_cont")
imgSiteUrl_ = urls_.find("a")["href"]
imgurl = 'https://anime-pictures.net' + imgSiteUrl_
b += 1
print(b)
print(imgurl)
#full_url = path + "/" + "test" + str(imgSiteUrl_.split('/')[3].split('-')[0]) + ".jpg"
#urllib.request.urlretrieve(imgurl, full_url)
if __name__ == '__main__':
while True:
start_time = time.time()
sys.setrecursionlimit(20000)
pool = multiprocessing.Pool(processes=8)
pool.map(get_content, get_links(0), chunksize=1)
num += 1
print(num)
print('time' + str(time.time() - start_time))
当我运行这个程序时,我在下面得到这个错误
Traceback (most recent call last):
File "D:/my_flolder/python/test.py", line 48, in <module>
pool.map(get_content, get_links(0), chunksize=1)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.1776.0_x64__qbz5n2kfra8p0\lib\multiprocessing\pool.py", line 364, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.1776.0_x64__qbz5n2kfra8p0\lib\multiprocessing\pool.py", line 771, in get
raise self._value
multiprocessing.pool.MaybeEncodingError: Error sending result: '<multiprocessing.pool.ExceptionWithTraceback object at 0x000001DA4B722100>'. Reason: 'TypeError("cannot pickle '_io.BufferedReader' object")'
错误是什么意思,我该如何解决?
解决方案
推荐阅读
- python - 使用 zmq 创建客户端时,`tempfile.NamedTemporaryFile` 的目的是什么?
- mysql - mysql shell 无法指定数据库,显示用法
- javascript - ionic3 angular4:如何仅使复选框可单击而不是整行?
- jquery - 为什么输入显示半文本而ajax响应具有完整字符串
- c# - 无法从一种模型转换为另一种模型
- javascript - 我应该忽略 React 警告:输入元素不应该从不受控切换到受控吗?
- javascript - 使用jquery append时onload不起作用
- python - 即使在表单 html 中添加 csrf_token 标记后,Django CSRF 验证也失败
- postgresql - 如何在 postgres 中的插入触发器函数期间引用 data_table
- angular - Jasmine fakeAsync 滴答不等待承诺被解决