python - Python CSV Writer 仅写入处理的最后一个抓取的项目

问题描述

所以我的刮刀只将最后两个项目从它处理的最后一页发送到 csv。我不知道我在哪里做错了它打印输出非常好。可能是经验丰富的眼睛能够提供帮助。

下面的代码：

from requests_html import HTMLSession
import csv
import time


 def get_links(url):
    _request = _session.get(url)
    items = _request.html.find('li.product-grid-view.product.sale')
    links = []
    for item in items:
         links.append(item.find('a', first=True).attrs['href'])

   # print(len(links))

    return links


 def get_product(link):
     _request = _session.get(link)

      title = _request.html.find('h2', first=True).full_text
      price = _request.html.find('span.woocommerce-Price-amount.amount bdi')[1].full_text
      sku = _request.html.find('span.sku', first=True).full_text
      categories = _request.html.find('span.posted_in', first=True).full_text.replace('Categories:', "").strip()
      brand = _request.html.find('span.posted_in')[1].full_text.replace('Brand:', "").strip()
      #print(brand)

       product = {
         'Title': title,
         'Price': price,
         'SKU': sku,
         'Categories': categories,
         'Brand': brand
       }

    #print(product)
     return product


if __name__ == '__main__':
    for page in range(1, 4):

        url = 'https://www.thebassplace.com/product-category/basses/4-string/'
    
        if page == 1:
           parse_url = url
        else:
            parse_url = f'https://www.thebassplace.com/product-category/basses/4-string/page/{page}/'

       _session = HTMLSession()

        links = get_links(parse_url)
        results = []

        for link in links:
            results.append(get_product(link))
            time.sleep(1)
            #print(len(results))


with open('on_sale_bass.csv', 'w', newline='', encoding='utf-8') as csv_file:
    
    writer = csv.DictWriter(csv_file, fieldnames=results[0].keys())
    writer.writeheader()

    for row in results:
        writer.writerow(row)

当我尝试附加记录时，记录是用 csv 编写的，但每次页面迭代都会重复标题。

标签： pythonpython-3.xcsvexport-to-csvpython-requests-html

问题出在语句中results = []，在范围循环内。results您在循环的每次迭代中都清空了range(1, 4)。因此，你得到的只是上一次迭代带来的东西。

请注意，我做了_sessionas global，但在这种情况下，我认为（请随时更正）在函数之间传递它是合理的。现在，试试这个。

from requests_html import HTMLSession
import csv
import time


def get_links(url):
    global _session
    _request = _session.get(url)
    items = _request.html.find('li.product-grid-view.product.sale')
    links = []
    for item in items:
        links.append(item.find('a', first=True).attrs['href'])
    return links


def get_product(link):
    global _session
    _request = _session.get(link)
    title = _request.html.find('h2', first=True).full_text
    price = _request.html.find('span.woocommerce-Price-amount.amount bdi')[1].full_text
    sku = _request.html.find('span.sku', first=True).full_text
    categories = _request.html.find('span.posted_in', first=True).full_text.replace('Categories:', "").strip()
    brand = _request.html.find('span.posted_in')[1].full_text.replace('Brand:', "").strip()
    product = {
        'Title': title,
        'Price': price,
        'SKU': sku,
        'Categories': categories,
        'Brand': brand
    }
    return product


if __name__ == '__main__':
    results = []
    for page in range(1, 4):
        url = 'https://www.thebassplace.com/product-category/basses/4-string/'
        if page == 1:
            parse_url = url
        else:
            parse_url = f'https://www.thebassplace.com/product-category/basses/4-string/page/{page}/'
    
        _session = HTMLSession()
        links = get_links(parse_url)

        for link in links:
            product = get_product(link)
            results.append(product)
            #time.sleep(1)
            
    with open('on_sale_bass.csv', 'w', newline='', encoding='utf-8') as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=results[0].keys())
        writer.writeheader()
        for row in results:
            writer.writerow(row)

我得到的一个例子：

python - Python CSV Writer 仅写入处理的最后一个抓取的项目

问题描述

解决方案

推荐阅读