1 import requests 2 from bs4 import BeautifulSoup as bs 3 import re 4 import os 5 import socket 6 import time 7 import threading 8 9 10 def url_open(url): 11 socket.setdefaulttimeout(20) 12 headers = { 13 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER'} 14 res = requests.get(url, headers=headers) 15 16 return res 17 18 19 def save(url): # 传入每个子网页链接 20 res = url_open(url) 21 res.encoding = 'gbk' 22 soup = bs(res.text, 'lxml') 23 title = soup.find('title').text.split('-')[0] # 标题 24 #os.mkdir(title) 25 # os.chdir(title) 26 temp = soup.find_all('tr', class_='tr3') 27 img = re.findall(r'data-src="(.*?jpg)" type', str(temp)) 28 29 30 imglist = [] 31 32 for each in img: 33 imglist.append(each) 34 for each in imglist: 35 filename = each.split('/')[-1] 36 img = url_open(each) 37 print('saving...+%s'%filename) 38 39 with open(title+filename, 'wb')as f: 40 f.write(img.content) 41 #os.chdir('..') 42 43 44 45 46 if __name__ == '__main__': 47 os.makedirs('1024', exist_ok=True) 48 os.chdir('1024') 49 url = 'https://cl.e7s.win/thread0806.php?fid=16&search=&page=1' #默认爬取第一个页面，毕竟要注意身体，需要多个页面的话，自己加个for循环也不是什么难事~ 50 urlhead = 'https://cl.e7s.win/' #页面解析出来的连接前面需要加上这个头才能打开，根据多年经验这个头是会变的，如果哪天不能用了自己看下是不是这个头变了 51 res = url_open(url) 52 res.encoding = 'gbk' 53 54 '''找到页面中的所有子网页''' 55 soup = bs(res.text, 'lxml') 56 temp = soup.find_all('td', class_="tal") 57 link = [] 58 for each in temp: 59 link.append(urlhead + each.h3.a.get('href')) 60 # del link[0:10] 61 62 downloads = [] 63 for each in link: 64 print(each) 65 66 down = threading.Thread(target=save, args=[each]) 67 downloads.append(down) 68 69 down.start() 70 for each in downloads: 71 each.join() 72 print('Done')