时间戳

首页 > 解决方案 > Python优化脚本？

python - Python优化脚本？

问题描述

我为 Pinterest 制作了一个可以正常运行但在良好规格硬件上非常慢的刮板。

逻辑

问题 脚本运行良好，但速度很慢，有时会冻结。

代码 - 声明

import time, argparse, pandas, datetime
from selenium import webdriver

# global lists for data storage
image_links = []
image_titles = []
image_descriptions = []
pin_links = []
pin_Board_Names = []
pin_Author_Names = []
scrape_timestamps = []

代码 - makeCSV

def makeCsv(filename):
    # csv head title along with data list for each
    data={"Time Stamps":scrape_timestamps, "Author Names":pin_Author_Names, "Pins Board":pin_Board_Names, "Pin Links":pin_links, "Image URLs":image_links, "Image Titles":image_titles, "Image Descriptions":image_descriptions}

    # create pandas dataframe from above dictionary
    dataframe = pandas.DataFrame.from_dict(data, orient='index')

    # transpose the dataframe - lists data as rows instead of columns in csv
    dataframe = dataframe.transpose()

    # save csv file
    dataframe.to_csv(filename, sep=',')

代码 - 抓取/抓取

def grab(url, filename):
    driver = webdriver.PhantomJS(executable_path='phantomjs.exe')
    print("\n\nGhost Driver Invoked")
    driver.implicitly_wait(10) # if element not found, wait for (seconds) before next operation
    driver.get(url) # grab the url

    time.sleep(3) # seconds

    # Scrolling till the end of page
    print("Started Scroling ...")
    last_height = driver.execute_script("return document.body.scrollHeight")
    i = 0
    while True:
        if i == 2:
            break
        else:
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(5) # seconds
            new_height = driver.execute_script("return document.body.scrollHeight")
            print("Scrolled: " + str(new_height))
            if new_height == last_height:
                break
            last_height = new_height
        i += 1

    time.sleep(3) # seconds

    # Get all pins , number of pins collected
    total_pins = []
    try:
        total_pins = driver.find_elements_by_class_name("Grid__Item")
    except:
        print("Unable to load pins")
    print("Total Pins: " + str(len(total_pins)))

    # get number of 'see more' buttons collected - for error checking
    moreButtons = driver.find_elements_by_xpath('//button[@data-test-id="seemoretoggle"]')
    print("Dynamic Elements: " + str(len(moreButtons)))
    print("Display: Dynamic Elements ...")

    # clicking all 'See More' buttons
    i = 0
    while i <= (len(moreButtons) - 1):
        moreButtons[i].click()
        print("Clicked: " + str(i+1))
        i += 1

    # Pin Image URLs
    i = 1
    while i < len(total_pins):
        try:

            temp_xpath = "/html/body/div/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[1]/a/img"
            temp = driver.find_element_by_xpath(temp_xpath).get_attribute("src")
            image_links.append(temp)
            scrape_timestamps.append(str(datetime.datetime.now())) # save timestamps of scrape
            i += 1
        except:
            i += 1
            pass
    print("Total Images: " + str(i))

    # Image Titles
    imageTitle_Elements = driver.find_elements_by_class_name("PinAttributionTitle__title")
    i = 1
    for imageTitle_Element in imageTitle_Elements:
        image_titles.append(imageTitle_Element.text)
        i += 1
    print("Total Titles: " + str(i))

    # Image Descriptions
    imageDesc_Elements = driver.find_elements_by_class_name("PinDescription__desc")
    i = 1
    for imageDesc_Element in imageDesc_Elements:
        image_descriptions.append(imageDesc_Element.text)
        i += 1
    print("Total Descriptions: " + str(i))

    # Pin Page Link
    i = 1
    while i < len(total_pins):
        temp_xpath = "/html/body/div/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[1]/a"
        temp = driver.find_element_by_xpath(temp_xpath).get_attribute("href")
        print("Pin Link:" + str(i))
        pin_links.append(temp)
        i += 1
    print("Total PinLinks: " + str(i))

    # Pin Board Names
    print("Extracting Board Names ... ")
    i = 1
    successful = False # for checking success of try | else not working
    while i <= len(total_pins):
        try:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div[2]/h4/a[1]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Board_Names.append(temp.text)
            print("Board_No: " + str(i) + " > " + temp.text)
            successful = True
        except:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div/h4/a[1]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Board_Names.append(temp.text)
            print("Board_No: " + str(i) + " > " + temp.text)
            successful = True
        if successful == False:
            print("Board_No: " + str(i) + " not found!")
        i += 1

    # Pin Author Names
    print("Extracting Author Names ... ")
    i = 1
    successful = False
    while i <= len(total_pins):
        try:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div[2]/h4/a[2]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Author_Names.append(temp.text)
            print("Author_No: " + str(i) + " > " + temp.text)
            successful = True
        except:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div/h4/a[2]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Author_Names.append(temp.text)
            print("Author_No: " + str(i) + " > " + temp.text)
            successful = True
        if successful == False:
            print("Author_No: " + str(i) + " not found!")
        i += 1

    # generate csv from collected items
    makeCsv(filename)

    # quit driver
    driver.quit()

代码 - 主要

def main():
    # start argument parser
    parser = argparse.ArgumentParser()

    # custom arguments
    parser.add_argument("-url", "--url", help="Enter the URL")
    parser.add_argument("-fname", "--fname", help="Enter Filename")

    # parse arguments
    args = parser.parse_args()

    # all web driver functions happen here:
    grab(args.url, args.fname)

if __name__ == "__main__":
    main()

慢速输出

>>> C:\Users\da74\Desktop\sites\pinscraper\ps>python -W ignore scrape.py --url https://www.pinterest.com/vijayakumargana/classic-kuala-lumpur/ --fname classic_kl.csv


Ghost Driver Invoked
Started Scroling ...
Scrolled: 11124
Scrolled: 18388
Total Pins: 126
Dynamic Elements: 125
Display: Dynamic Elements ...
Clicked: 1
Clicked: 2
Clicked: 3
Clicked: 4
Clicked: 5
Clicked: 6
Clicked: 7
Clicked: 8
Clicked: 9
Clicked: 10
Clicked: 11
Clicked: 12
Clicked: 13
Clicked: 14
Clicked: 15
Clicked: 16
Clicked: 17
Clicked: 18
Clicked: 19
Clicked: 20
Clicked: 21
Clicked: 22
Clicked: 23
Clicked: 24
Clicked: 25
Clicked: 26
Clicked: 27
Clicked: 28
Clicked: 29
Clicked: 30
Clicked: 31
Clicked: 32
Clicked: 33
Clicked: 34
Clicked: 35
Clicked: 36
Clicked: 37
Clicked: 38
Clicked: 39
Clicked: 40
Clicked: 41
Clicked: 42
Clicked: 43
Clicked: 44
Clicked: 45
Clicked: 46
Clicked: 47
Clicked: 48
Clicked: 49
Clicked: 50
Clicked: 51
Clicked: 52
Clicked: 53
Clicked: 54
Clicked: 55
Clicked: 56
Clicked: 57
Clicked: 58
Clicked: 59
Clicked: 60
Clicked: 61
Clicked: 62
Clicked: 63
Clicked: 64
Clicked: 65
Clicked: 66
Clicked: 67
Clicked: 68
Clicked: 69
Clicked: 70
Clicked: 71
Clicked: 72
Clicked: 73
Clicked: 74
Clicked: 75
Clicked: 76
Clicked: 77
Clicked: 78
Clicked: 79
Clicked: 80
Clicked: 81
Clicked: 82
Clicked: 83
Clicked: 84
Clicked: 85
Clicked: 86
Clicked: 87
Clicked: 88
Clicked: 89
Clicked: 90
Clicked: 91
Clicked: 92
Clicked: 93
Clicked: 94
Clicked: 95
Clicked: 96
Clicked: 97
Clicked: 98
Clicked: 99
Clicked: 100
Clicked: 101
Clicked: 102
Clicked: 103
Clicked: 104
Clicked: 105
Clicked: 106
Clicked: 107
Clicked: 108
Clicked: 109
Clicked: 110
Clicked: 111
Clicked: 112
Clicked: 113
Clicked: 114
Clicked: 115
Clicked: 116
Clicked: 117
Clicked: 118
Clicked: 119
Clicked: 120
Clicked: 121
Clicked: 122
Clicked: 123
Clicked: 124
Clicked: 125

单击所有 js 按钮后，脚本会冻结。即使它正在单击按钮，它也会减慢很多。使用 Python3.6，Windows 10 x64 Professional，8GB RAM，2TB HDD。

如果有人可以提供帮助并建议优化它的方法并阻止它在太多东西或更大的页面时冻结？谢谢你。

标签： pythonpython-3.xseleniumoptimizationphantomjs

解决方案

推荐阅读