1 import requests 2 import re 3 # from bs4 import BeautifulSoup 4 from urllib import request 5 # import threading 6 import gevent 7 from gevent import monkey 8 9 monkey.patch_all() 10 11 def get_html_text(url): 12 try: 13 hd = {'User-Agent':'Mozilla/5.0'} #添加伪装浏览器头部信息 14 r = requests.get(url, timeout=10, headers = hd) 15 r.raise_for_status() 16 r.encoding = r.apparent_encoding 17 print(len(r.text)) 18 return r.text 19 except Exception as result: 20 print('错误类型：', result) 21 22 23 def html_text_parser(img_list, html): 24 25 # 下面是修改的重点部分，采用了正则表达式，没有采用bs4 26 # 技术大神可以研究一下斗鱼这个网页的源代码，真正的图片信息都存储在后面，不是传统的# # html,我还没怎么接触过前端的知识，不知怎么使用bs4,所以使用了正则表达式。 27 28 img_pat = r'"rs\w+":"(.*?g)"' 29 links = re.compile(img_pat, re.S).findall(html) 30 print(len(links)) 31 print(links) 32 for link in links: 33 if link: 34 img_list.append(link) 35 return img_list 36 37 38 39 40 def get_douyu_img(Img_list): 41 for i,j in enumerate(Img_list): 42 # name = j.split('.')[-1] 43 try: #异常捕获，如果链接不能访问，退出当前一次循环，进入下一次循环 44 r = request.urlopen(j) 45 ima_content = r.read() 46 path = str(i) 47 with open(path, 'wb') as f: 48 f.write(ima_content) 49 except: 50 continue 51 def main(): 52 url = 'https://www.douyu.com/g_yz' 53 html = get_html_text(url) 54 img_list = list() 55 Img_list = html_text_parser(img_list, html) 56 # print(Img_list) 57 #t1 = threading.Thread(target=get_html_text, args=(url,)) 58 #t2 = threading.Thread(target=html_text_parser, args=(img_list,html)) 59 #t3 = threading.Thread(target=get_douyu_img, args=(Img_list,)) 60 #t1.start() 61 #t2.start() 62 #t3.start() 63 gevent.joinall([ 64 gevent.spawn(get_html_text, url), 65 gevent.spawn(html_text_parser, img_list, html), 66 gevent.spawn(get_douyu_img, Img_list) 67 ]) 68 69 70 if __name__ == '__main__': 71 main()