python - 使用 Python Selenium 在内存中下载文件,而不是在磁盘中
问题描述
我有一堆脚本可以进行网页抓取、下载文件并使用 pandas 读取它们。这个过程必须部署在一个新的架构中,在这种架构中下载磁盘上的文件是不合适的,而是最好将文件保存在内存中并从那里用 pandas 读取它。出于演示目的,我在这里留下一个从随机网站下载 excel 文件的网络抓取脚本:
import time
import pandas as pd
from io import StringIO, BytesIO
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from datetime import date, timedelta
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
pathDriver = #Path to chromedriver
driver = webdriver.Chrome(executable_path=pathDriver)
url = 'https://file-examples.com/index.php/sample-documents-download/sample-xls-download/'
driver.get(url)
time.sleep(1)
file_link = driver.find_element_by_xpath('//*[@id="table-files"]/tbody/tr[1]/td[5]/a[1]')
file_link.click()
此脚本有效地将文件下载到我的下载文件夹中。我尝试的是在方法之前和之后放置一个StringIO()
或BytesIO()
流click()
并读取与此类似的对象:
file_object = StringIO()
df = pd.read_excel(file_object.read())
但是 file_object 没有捕获文件,甚至文件仍然下载到我的磁盘中。
有什么建议吗?
解决方案
您的问题可以通过添加selenium add_experimental_option来完成。我还重新设计了您的代码以循环遍历表以提取 href 以将它们传递给 StringIO。使用此代码不会将文件下载到我的本地系统。
如果我错过了什么,请告诉我。
import pandas as pd
from time import sleep
from io import StringIO
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
capabilities = DesiredCapabilities().CHROME
chrome_options = Options()
chrome_options.add_argument("--incognito")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")
prefs = {
'profile.default_content_setting_values':
{
'automatic_downloads': 0
},
'profile.content_settings.exceptions':
{
'automatic_downloads': 0
}
}
chrome_options.add_experimental_option('prefs', prefs)
capabilities.update(chrome_options.to_capabilities())
driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)
url_main = 'https://file-examples.com/index.php/sample-documents-download/sample-xls-download/'
driver.get(url_main)
elements = driver.find_elements_by_xpath('//*[@id="table-files"]//td/a')
for element in elements:
if str(element.get_attribute("href")).endswith('.xls'):
file_object = StringIO(element.get_attribute("href"))
xls_file = file_object.read()
df = pd.read_excel(xls_file)
print(df.to_string(index=False))
First Name Last Name Gender Country Age Date Id
1 Dulce Abril Female United States 32 15/10/2017 1562
2 Mara Hashimoto Female Great Britain 25 16/08/2016 1582
3 Philip Gent Male France 36 21/05/2015 2587
4 Kathleen Hanner Female United States 25 15/10/2017 3549
5 Nereida Magwood Female United States 58 16/08/2016 2468
6 Gaston Brumm Male United States 24 21/05/2015 2554
7 Etta Hurn Female Great Britain 56 15/10/2017 3598
8 Earlean Melgar Female United States 27 16/08/2016 2456
9 Vincenza Weiland Female United States 40 21/05/2015 6548
sleep(360)
这是一个使用评论中提到的 RAMDISK 的示例。此选项不使用selenium add_experimental_option或 StringIO。
import fs
import pandas as pd
from time import sleep
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--incognito")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")
driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)
url_main = 'https://file-examples.com/index.php/sample-documents-download/sample-xls-download/'
driver.get(url_main)
urls_to_process = []
elements = driver.find_elements_by_xpath('//*[@id="table-files"]//td/a')
# Create RAMDISK
mem_fs = fs.open_fs('mem://')
mem_fs.makedir('hidden_dir')
for element in elements:
if str(element.get_attribute("href")).endswith('.xls'):
with mem_fs.open('hidden_dir/file1.csv', 'w') as in_file:
in_file.write(element.get_attribute("href"))
in_file.close()
with mem_fs.open('hidden_dir/file1.csv', 'r') as out_file:
df = pd.read_excel(out_file.read())
print(df.to_string(index=False))
# same output as above
sleep(360)
推荐阅读
- javascript - For...in 循环导致潜在的内存不足崩溃
- c# - 如何使用 asp.net c# 在 azure 门户中使用 SendGrid Api Key 发送邮件
- javascript - Webpack-merge 和 Object.assign() 有什么区别?
- c# - AutoQuery 无法查询嵌套对象
- firebase - 我可以从与 Firebase 分析相关联的 BigQuery 中获取“ad_impression”和“ad_click”吗?
- javascript - HTML 输入类型文本不允许数字
- powershell - 按字母顺序排序,小写在大写之前?
- javascript - 音频播放并在按钮单击时停止
- c# - C# 控制台应用程序作为 Windows 窗体
- java - 当前平台不支持 ICON_IMAGE 功能