python - 抓取页面返回 200,稍后检查该页面返回 403
问题描述
我正在使用 Scrapy 抓取一些网站以发布招聘信息。如果网站上的页面符合我的要求,我会将指向该页面的链接存储在数据库中。那里没有问题。我还创建了一个脚本,它遍历数据库中的每个链接并 ping URL。如果它返回 404,它将被删除。我遇到的问题是某些网站在我进行删除检查时返回 403 错误。奇怪的是,它们都允许抓取,但它们阻止了检查。这是我用来进行删除检查的脚本:
from pymongo import MongoClient
import requests
import urllib3
from operator import itemgetter
import random
import time
client = MongoClient("path-to-mongo")
db = client["mongoDB"]
col = db['mongoCollection']
openings = list(col.find())
sorted_openings = sorted(openings, key=itemgetter('Company'))
del_counter = 0
user_agents = ["Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)",
"Mozilla/5.0 CK={} (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)"]
headers = {"User-Agent": user_agents[random.randint(0,11)]}
counter = 0
del_counter = 0
passed_counter = 0
deleted_links = []
passed_links = []
forbidden = []
for item in sorted_openings:
try:
if requests.get(item['Link'], allow_redirects=False, verify=False, headers=headers).status_code == 200:
print(str(requests.get(item['Link'])) + ' ' + item['Link'])
counter += 1
print(counter)
elif requests.get(item['Link'], allow_redirects=False, verify=False, headers=headers).status_code == 304:
print(requests.get(item['Link']))
counter += 1
print(counter)
elif requests.get(item['Link'], allow_redirects=False, verify=False, headers=headers).status_code == 403:
forbidden.append(item['Link'])
print(requests.get(item['Link']))
counter += 1
print(counter)
else:
db.openings.remove(item)
deleted_links.append(item['Link'])
del_counter += 1
counter += 1
print('Deleted ' + item['Link'])
print(counter)
except:
pass
passed_links.append(item['Link'])
passed_counter += 1
counter += 1
print('Passed link ' + item['Link'])
print(counter)
解决方案
您在每个条件下发送请求,发送一个请求并将结果存储在一个值中,然后检查条件。
from pymongo import MongoClient
import requests
import urllib3
from operator import itemgetter
import random
import time
client = MongoClient("path-to-mongo")
db = client["mongoDB"]
col = db['mongoCollection']
openings = list(col.find())
sorted_openings = sorted(openings, key=itemgetter('Company'))
del_counter = 0
user_agents = ["Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)",
"Mozilla/5.0 CK={} (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)"]
headers = {"User-Agent": user_agents[random.randint(0,11)]}
counter = 0
del_counter = 0
passed_counter = 0
deleted_links = []
passed_links = []
forbidden = []
for item in sorted_openings:
try:
response = requests.get(item['Link'], allow_redirects=False, verify=False, headers=headers)
if response.status_code == 200:
print(str(response) + ' ' + item['Link'])
counter += 1
print(counter)
elif response.status_code == 304:
print(response)
counter += 1
print(counter)
elif response.status_code == 403:
forbidden.append(item['Link'])
print(response))
counter += 1
print(counter)
else:
db.openings.remove(item)
deleted_links.append(item['Link'])
del_counter += 1
counter += 1
print('Deleted ' + item['Link'])
print(counter)
except:
passed_links.append(item['Link'])
passed_counter += 1
counter += 1
print('Passed link ' + item['Link'])
print(counter)
推荐阅读
- c++ - 在 xcode 10 上找不到 stdlibc++ 标头错误
- c# - 在 C# 中提高 MySQL INSERT SELECT 的性能
- javascript - Meteor 应用程序在启动时崩溃,并在 fusioncharts 模块上找不到 Window 错误
- intellij-idea - 如何在 Intellij IDEA 中使用 TeaVM 调试器
- html - 在 Reactstrap/Bootstrap CardImg 上设置最大高度不保持纵横比 - 如何解决?
- python - 如何使用 pyPDF2 从 PDF 格式的多页简历中提取文本数据?
- codenameone - 代号一已购买证书,但无法继续
- javascript - Jquery在选择元素上添加一个空行
- css - 避免中间标签干扰 display:table
- android - 无法为 org.gradle.api.internal.artifacts.dsl.dependencies.DefaultDependencyHandler 类型的对象获取未知属性“FCM_VERSION”