python - 网页抓取页表Python
问题描述
我正在尝试从网站上抓取表格,但我对 Python 不是很熟悉。网址在这里 我想以 data.frame 格式获取表格,以便继续我的分析。
这是我测试过的代码:
from lxml import html
import requests
from bs4 import BeautifulSoup
from urllib.request import urlopen, Request
url="https://www.wunderground.com/history/daily/gr/almyros/LGBL/date/2017-4-4"
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}
req = Request(url=url1, headers=headers)
html = urlopen(req).read()
print(html)
soup = BeautifulSoup(html)
type(soup)
print(soup)
p=soup.find_all('thead')
k=soup.find_all('tbody')
print(k)
我不知道如何获得每日观察表
解决方案
试试这个:
import pandas as pd
from pandas.io.json import json_normalize
import re
import requests
url="https://www.wunderground.com/history/daily/gr/almyros/LGBL/date/2017-4-4"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'}
with requests.Session() as s:
response = s.get(url, headers=headers)
p = re.compile(r'point\?apiKey=([a-zA-z0-9]*)&', re.MULTILINE)
apiKey = re.findall(p, response.text)[0]
url = 'https://api.weather.com/v1/location/LGBL:9:GR/observations/historical.json'
payload = {
'apiKey': apiKey,
'startDate': '20170404',
'endDate': '20170404'}
jsonData = s.get(url, headers=headers, params=payload).json()
df = json_normalize(jsonData['observations'])
输出:
print (df.to_string())
blunt_phrase class clds day_ind dewPt expire_time_gmt feels_like gust heat_index icon_extd key max_temp min_temp obs_id obs_name precip_hrly precip_total pressure pressure_desc pressure_tend primary_swell_direction primary_swell_height primary_swell_period primary_wave_height primary_wave_period qualifier qualifier_svrty rh secondary_swell_direction secondary_swell_height secondary_swell_period snow_hrly temp terse_phrase uv_desc uv_index valid_time_gmt vis water_temp wc wdir wdir_cardinal wspd wx_icon wx_phrase
0 None observation OVC N 45 1491263400 44 None 46 2600 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 46 None Low 0 1491256200 6 None 44 260 W 5 26 Cloudy
1 None observation OVC N 45 1491267000 44 None 46 2600 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 46 None Low 0 1491259800 6 None 44 250 WSW 6 26 Cloudy
2 None observation OVC N 43 1491270600 40 None 45 2600 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 45 None Low 0 1491263400 6 None 40 250 WSW 8 26 Cloudy
3 None observation OVC N 41 1491274200 39 None 43 2600 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 43 None Low 0 1491267000 6 None 39 250 WSW 6 26 Cloudy
4 None observation OVC N 43 1491277800 39 None 43 2600 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 100 None None None None 43 None Low 0 1491270600 6 None 39 250 WSW 7 26 Cloudy
5 None observation SCT N 39 1491281400 36 None 41 2900 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 93 None None None None 41 None Low 0 1491274200 6 None 36 250 WSW 8 29 Partly Cloudy
6 None observation FEW N 39 1491283200 36 None 41 3300 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 41 None Low 0 1491276000 6 None 36 260 W 8 33 Fair
7 None observation FEW D 39 1491285000 36 None 41 3400 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 41 None Low 0 1491277800 6 None 36 250 WSW 7 34 Fair
8 None observation FEW D 39 1491286800 37 None 41 3400 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 41 None Low 1 1491279600 6 None 37 250 WSW 6 34 Fair
9 None observation FEW D 39 1491288600 36 None 41 3400 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 41 None Low 1 1491281400 6 None 36 250 WSW 7 34 Fair
10 None observation FEW D 41 1491290400 39 None 43 3400 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 93 None None None None 43 None Low 1 1491283200 6 None 39 260 W 6 34 Fair
11 None observation FEW D 45 1491292200 46 None 48 3400 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 87 None None None None 48 None Low 1 1491285000 6 None 46 240 WSW 6 34 Fair
12 None observation FEW D 46 1491294000 50 None 50 3400 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 87 None None None None 50 None Low 1 1491286800 6 None 50 270 W 2 34 Fair
13 None observation FEW D 46 1491295800 54 None 54 3400 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 77 None None None None 54 None Low 2 1491288600 6 None 54 250 WSW 2 34 Fair
14 None observation FEW D 48 1491297600 57 None 57 3400 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 72 None None None None 57 None Moderate 3 1491290400 6 None 57 200 SSW 2 34 Fair
15 None observation FEW D 48 1491299400 57 None 57 3400 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 72 None None None None 57 None Moderate 3 1491292200 6 None 57 90 E 3 34 Fair
16 None observation FEW D 48 1491301200 57 None 57 3400 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 72 None None None None 57 None Moderate 4 1491294000 6 None 57 100 E 3 34 Fair
17 None observation FEW D 48 1491303000 59 None 59 3400 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 67 None None None None 59 None Moderate 5 1491295800 6 None 59 80 E 3 34 Fair
18 None observation SCT D 48 1491304800 59 None 59 3000 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 67 None None None None 59 None Moderate 4 1491297600 6 None 59 80 E 3 30 Partly Cloudy
19 None observation SCT D 48 1491306600 61 None 61 3000 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 63 None None None None 61 None Moderate 5 1491299400 6 None 61 70 ENE 6 30 Partly Cloudy
20 None observation SCT D 50 1491308400 61 None 61 3000 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 68 None None None None 61 None Moderate 5 1491301200 6 None 61 60 ENE 5 30 Partly Cloudy
21 None observation SCT D 48 1491310200 61 None 61 3000 LGBL None None LGBL Volos/Nea Anghia None None 30.00 None None None None None None None None None 63 None None None None 61 None Moderate 5 1491303000 6 None 61 80 E 6 30 Partly Cloudy
22 None observation SCT D 50 1491312000 61 None 61 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 68 None None None None 61 None Moderate 5 1491304800 6 None 61 70 ENE 6 30 Partly Cloudy
23 None observation SCT D 50 1491313800 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 63 None None None None 63 None Moderate 4 1491306600 6 None 63 70 ENE 6 30 Partly Cloudy
24 None observation SCT D 50 1491315600 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 63 None None None None 63 None Moderate 4 1491308400 6 None 63 70 ENE 6 30 Partly Cloudy
25 None observation SCT D 50 1491317400 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 63 None None None None 63 None Moderate 3 1491310200 6 None 63 100 E 7 30 Partly Cloudy
26 None observation SCT D 52 1491319200 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 68 None None None None 63 None Moderate 3 1491312000 6 None 63 80 E 7 30 Partly Cloudy
27 None observation SCT D 52 1491321000 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 68 None None None None 63 None Low 2 1491313800 6 None 63 90 E 7 30 Partly Cloudy
28 None observation SCT D 52 1491322800 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 68 None None None None 63 None Low 2 1491315600 6 None 63 90 E 8 30 Partly Cloudy
29 None observation SCT D 52 1491324600 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 68 None None None None 63 None Low 1 1491317400 6 None 63 70 ENE 8 30 Partly Cloudy
30 None observation SCT D 50 1491326400 63 None 63 3000 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 63 None None None None 63 None Low 1 1491319200 6 None 63 100 E 10 30 Partly Cloudy
31 None observation FEW D 52 1491328200 61 None 61 3400 LGBL None None LGBL Volos/Nea Anghia None None 29.94 None None None None None None None None None 72 None None None None 61 None Low 1 1491321000 6 None 61 110 ESE 14 34 Fair
32 None observation FEW D 54 1491330000 59 None 59 3400 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 82 None None None None 59 None Low 1 1491322800 6 None 59 110 ESE 10 34 Fair
33 None observation FEW D 54 1491331800 59 None 59 3400 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 82 None None None None 59 None Low 1 1491324600 6 None 59 120 ESE 12 34 Fair
34 None observation FEW N 54 1491333600 57 None 57 3300 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 88 None None None None 57 None Low 0 1491326400 6 None 57 120 ESE 10 33 Fair
35 None observation FEW N 54 1491335400 57 None 57 3300 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 88 None None None None 57 None Low 0 1491328200 6 None 57 110 ESE 13 33 Fair
36 None observation FEW N 54 1491339000 55 None 55 3300 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 94 None None None None 55 None Low 0 1491331800 6 None 55 90 E 5 33 Fair
37 None observation FEW N 52 1491342600 55 None 55 3300 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 88 None None None None 55 None Low 0 1491335400 6 None 55 130 SE 5 33 Fair
38 None observation FEW N 48 1491346200 50 None 50 3300 LGBL None None LGBL Volos/Nea Anghia None None 29.97 None None None None None None None None None 94 None None None None 50 None Low 0 1491339000 6 None 50 270 W 2 33 Fair