python - Skip Over Item if element doesn't exist on page

Question

I have a script that loops through multiple pages of a website and I want to skip over or add a blank space for the item that might not be on certain pages. For example, there are some pages that do not contain a description about the book. When I run into one of those pages I get an attribute error. My script below loops through the first two pages with no problem, but when it hits the third page it stops. Here is the traceback

item['description'] = about.h2.nextSibling.nextSibling.nextSibling.text File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/beautifulsoup4-4.6.0-py3.6.egg/bs4/element.py", line 737, in __getattr__ AttributeError: 'NavigableString' object has no attribute 'text'

How can I fix this? Here is my script:

from bs4 import BeautifulSoup as soup
import requests
import json

base_url = "https://open.umn.edu/opentextbooks/"

data = []
n = 30
for i in range(4, n+1):
    response = requests.get(base_url + "BookDetail.aspx?bookId=" + str(i))

    #html parsing
    page_soup = soup(response.content, "html5lib")

    #grabs info for each textbook
    containers = page_soup.findAll("div",{"class":"LongDescription"})
    author = page_soup.select("p")
    about = page_soup.find("div",{"id":"AboutBook"})

    for container in containers:
        item = {}
        item['type'] = "Textbook"
        item['title'] = container.find("div",{"class":"twothird"}).h1.text
        item['author'] = author[3].get_text(separator=', ')
        if item['author'] == " ":
            item['author'] = "University of Minnesota Libraries Publishing"
        item['link'] = "https://open.umn.edu/opentextbooks/BookDetail.aspx?bookId=" + str(i)
        if not container.find(string="Publisher: "):
            item['publisher_url'] = item['publisher'] = ""
        else:
            item['publisher'] = container.find(text="Publisher: ").nextSibling.text
            item['publisher_url'] = container.find(text="Publisher: ").nextSibling['href']
        item['source'] = "Open Textbook Library"
        if not about.h2.nextSibling.nextSibling.nextSibling:
            item['description'] = ""
        else:
            item['description'] = about.h2.nextSibling.nextSibling.nextSibling.text
        item['base_url'] = "https://open.umn.edu/opentextbooks/"
        if container.find("p",{"class":"Badge-Condition"}).a:
            item['license'] = container.find("p",{"class":"Badge-Condition"}).a.text
        if container.find("img",{"class":"ctl00_maincontent_imgLicence"}):
            item['license'] = ''
        if container.find("p",{"class":"Badge-Condition"}).a:
            item['license_url'] = container.find("p",{"class":"Badge-Condition"}).a["href"]
        if container.find("img",{"class":"ctl00_maincontent_imgLicence"}):
            item['license_url'] = ''
        if container.find("div",{"class":"twothird"}).p:
            item['review'] = container.find("div",{"class":"twothird"}).p.text
        else:
            item['review'] = ''
        if item['review'].startswith('('):
            item['review'] = item['review'].replace('(', '')
        if item['review'].endswith(' reviews)'):
            item['review'] = item['review'].replace(' reviews)', '')
        if item['review'] > str(0):
            item['review'] = "Reviewed Resource"
        else:
            item['review'] = ''
        item['image_url'] = "https://open.umn.edu/opentextbooks/" + container.img["src"]
        data.append(item) # add the item to the list

    with open("./json/otl-1.json", "w") as writeJSON:
        json.dump(data, writeJSON, ensure_ascii=False)

标签： pythonjsonpython-3.xbeautifulsoup

Answer 1

我不建议用 解析描述item['description'] = about.h2.nextSibling.nextSibling.nextSibling.text，这太具体了。我想出了这段代码：

from bs4 import BeautifulSoup as soup
import requests
import json
from pprint import pprint

base_url = "https://open.umn.edu/opentextbooks/"

data = []
n = 30
for i in range(4, n+1):
    response = requests.get(base_url + "BookDetail.aspx?bookId=" + str(i))

    page_soup = soup(response.content, "lxml")

    data = {}
    title, author, description =  page_soup.select('h1')[0].text, \
                                  page_soup.select('h1 ~ p')[3].get_text(', '), \
                                  '\n'.join(p.text.strip() for p in page_soup.select('div#AboutBook > p') if p.text.strip())

    data['type'] = "Textbook"
    data['title'] = title
    data['author'] = author if author.strip() else "University of Minnesota Libraries Publishing"
    data['link'] = "https://open.umn.edu/opentextbooks/BookDetail.aspx?bookId=" + str(i)
    data['source'] = "Open Textbook Library"
    data['description'] = description

    pprint(data)

    # with open("./json/otl-1.json", "w") as writeJSON:
    #     json.dump(data, writeJSON, ensure_ascii=False)

印刷：

{'author': 'University of Minnesota Libraries Publishing',
 'description': 'This book is intended for an undergraduate or MBA level '
                'Financial Accounting course. It covers the standard topics in '
                'a standard sequence, utilizing the Socratic method of asking '
                'and answering questions.',
 'link': 'https://open.umn.edu/opentextbooks/BookDetail.aspx?bookId=4',
 'source': 'Open Textbook Library',
 'title': 'Financial Accounting',
 'type': 'Textbook'}

...and so on (for each book)