python - 字典中的某些字段未写入 csv 文件

Question

我已经从duckduckgo.com 抓取了结果并将结果存储在标题、链接、描述链接和描述被打印但标题没有被打印

我已经用 print(title) 打印了标题，它给出了输出

class DuckduckgoScraper(web_scraping):
    def scrape(self,search_Term):
        self.filename = search_Term
        self.url = 'https://duckduckgo.com/html?q='+search_Term
        r = requests.get(self.url,headers=USER_AGENT)
        soup = BeautifulSoup(r.content,'html5lib')
        result_block = soup.find_all(class_ = 'result__body')
        for result in result_block:
            link = result.find('a', attrs={'class':'result__a'}, href=True)
            title = result.find('h2')
            description = result.find(attrs={'class':'result__snippet'})
            if link and title:
                link = link['href']
                title = title.get_text()
                if description:
                    description = description.get_text()
                    with open(self.filename+'.csv', 'a', encoding='utf-8',newline='') as csv_file:
                        file_is_empty = os.stat(self.filename+'.csv').st_size==0
                        fieldname = ['title','link','description']
                        writer = csv.DictWriter(csv_file,fieldnames=fieldname)
                        if file_is_empty:
                            writer.writeheader()
                        writer.writerow({'title':title,'link':link,'description':description})

它没有给出任何错误

标签： pythoncsvweb-scrapingbeautifulsoupduckduckgo

Answer 1

您在每行迭代中打开 ng 并将其写入 csv 文件。.writerows()取而代之的是，将行存储在列表中，并在最后用函数一次写入它们。

注意：.strip()对行的每个项目执行此操作很有用，否则 Excel/LibreOffice/...在打开文件时可能会感到困惑。

import os
import csv
import requests
from bs4 import BeautifulSoup

USER_AGENT = {'User-Agent':'Mozilla/5.0'}

def scrape(search_Term):
    filename = search_Term
    url = 'https://duckduckgo.com/html?q='+search_Term
    r = requests.get(url,headers=USER_AGENT)
    soup = BeautifulSoup(r.content,'html5lib')
    result_block = soup.find_all(class_ = 'result__body')
    for result in result_block:
        link = result.find('a', attrs={'class':'result__a'}, href=True)
        title = result.find('h2')
        description = result.find(attrs={'class':'result__snippet'})

        rows = []
        if link and title:
            link = link['href']
            title = title.get_text()
            if description:
                description = description.get_text()
                rows.append({'title':title.strip(), 'link':link.strip(), 'description':description.strip()})
                # print(title.strip(), link.strip())
                # print(description.strip())
                # print('*'* 80)

        with open(filename+'.csv', 'a', encoding='utf-8',newline='') as csv_file:
            file_is_empty = os.stat(filename+'.csv').st_size==0
            fieldname = ['title','link','description']
            writer = csv.DictWriter(csv_file,fieldnames=fieldname)
            if file_is_empty:
                writer.writeheader()
            writer.writerows(rows)

scrape('tree')

这会创建tree.csv. 在 LibreOffice 中，它看起来像这样：