python3爬虫 -----华东交大校园新闻爬取与数据分析

 1 import requests
 2 import requests.exceptions
 3 import re
 4 import json
 5 
 6 #请求头，防止防爬虫的网页
 7 headers={
 8 "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
 9 }
10 
11 #获取一张网页上的内容
12 def get_one_page(url):
13     try:
14         res = requests.get(url, headers=headers)
15         if res.status_code == 200:
16             return res.text
17         return None
18     except Exception:
19         return None
20 
21 #根据网页上的内容，再析取新闻标题
22 def parse_one_page(html):
23     pattern=re.compile('<td align="left".*?<a href.*?>(.*?)</a>.*?</td>',re.S)
24     items=re.findall(pattern,html)
25     return items
26     # for item in items:
27     #     yield {
28     #          "title":item.split()
29     #     }
30 
31 
32 #写入文件
33 def write_to_file(content):
34     with open('news_ecjtu.txt','a',encoding='utf-8') as f:
35         f.write(json.dumps(content,ensure_ascii=False)+'\n')
36         f.close()
37 
38 
39 def main(page):
40     if(page):
41         page+=1
42         url='http://xw.ecjtu.jx.cn/1083/list'+str(page)+'.htm'
43     else:
44         url='http://xw.ecjtu.jx.cn/1083/list.htm'
45     html=get_one_page(url)
46 
47     for item in parse_one_page(html):
48        write_to_file(item)
49 
50 
51 if __name__ == '__main__':
52     for i in range(10): #582
53         main(i)

 1 # -*- coding :utf-8 -*-
 2 # CountVectorizer是通过fit_transform函数将文本中的词语转换为词频矩阵，
 3 # 矩阵元素weight[i][j] 表示j词在第i个文本下的词频，即各个词语出现的次数；
 4 # 通过get_feature_names()
 5 # 可看到所有文本的关键字，通过toarray()可看到词频矩阵的结果。
 6 # TfidfTransformer也有个fit_transform函数，它的作用是计算tf-idf值。
 7 
 8 
 9 import jieba
10 import jieba.posseg
11 import os
12 import sys
13 from sklearn import feature_extraction
14 from sklearn.feature_extraction.text import TfidfTransformer
15 from sklearn.feature_extraction.text import CountVectorizer
16 import sys
17 
18 
19 from numpy import *
20 
21 def read_from_file(file_name):
22     with open(file_name,'r',encoding='utf-8') as f:
23         words=f.read()
24     return words
25 
26 
27 def main(path1,name,path2):
28 
29     with open(path1+name+".txt",'r',encoding="utf-8") as f:
30         f_list=f.read()
31         dataList=f_list.split('\n')
32         data=[]
33         words=read_from_file("D:\\PyCharm_Project\\My_project\\Dictionary\\chineseStop_word")
34         for line in dataList:
35             data.append(" ".join(jieba.cut(line)))
36 
37 
38 
39     freqWord=CountVectorizer(stop_words='english') #将所得的词转换为词频矩阵
40 
41     # 第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵
42     tfidf=TfidfTransformer().fit_transform(freqWord.fit_transform(data))
43 
44     words=freqWord.get_feature_names() #获得词袋模型中所有的词语
45     weight=tfidf.toarray()
46     tfidfDict={}
47 
48     for i in range(len(weight)):
49         for j in range(len(words)):
50             getWord=words[j]
51             getValue=weight[i][j]
52             if getValue !=0:
53                 if getWord in tfidfDict.keys():
54                     tfidfDict[getWord]+=float(getValue)
55                 else:
56                     tfidfDict.update({getWord:getValue})
57     sortted_tfidf=sorted(tfidfDict.items(),key=lambda d:d[1],reverse=True)
58 
59     print('Start Kmeans:')
60     from sklearn.cluster import KMeans
61     clf = KMeans(n_clusters=10)
62     s = clf.fit(weight)
63     print(s)
64 
65     # 20个中心点
66     print(clf.cluster_centers_)
67 
68     # 每个样本所属的簇
69     print(clf.labels_)
70     i = 1
71     while i <= len(clf.labels_):
72         print(i, clf.labels_[i - 1])
73         i = i + 1
74 
75     # 用来评估簇的个数是否合适，距离越小说明簇分的越好，选取临界点的簇个数
76     print(clf.inertia_)
77     with open(path2+name+'result.csv',"w",encoding="utf-8") as f:
78         for i in sortted_tfidf:
79             f.write(i[0]+'\t'+str(i[1])+'\n')
80 
81 
82 if __name__ == '__main__':
83     # names={'story','story2'}
84     # for name in names:
85     #     main(name)
86     path1="D:\\PyCharm_Project\\My_project\\news\\"
87     name="news_ecjtu"
88     path2="D:\\PyCharm_Project\\My_project\\parseResult\\"
89     main(path1,name,path2)

  1 from sklearn.feature_extraction.text import TfidfVectorizer
  2 import jieba
  3 
  4 with open("news_ecjtu.txt", 'r', encoding="utf-8") as f:
  5     f_list = f.read()
  6     dataList = f_list.split('\n')
  7     data = []
  8     for line in dataList:
  9         data.append(" ".join(jieba.cut(line)))
 10 
 11 
 12 tfidf=TfidfVectorizer(token_pattern=r"(?u)\b\w\w+\b")
 13 weight=tfidf.fit_transform(data).toarray()
 14 word=tfidf.get_feature_names()
 15 
 16 dict1 = {}
 17 f = open('D:\\PyCharm_Project\\My_project\\Dictionary\\provinces.txt', 'r', encoding="utf-8")
 18 i = 0
 19 for line in f.readlines():
 20     tmp = line.strip()
 21     dict1[tmp] = i
 22 
 23 dict2 = {}
 24 f = open('D:\\PyCharm_Project\\My_project\\Dictionary\\Universities.txt', 'r', encoding="utf-8")
 25 i=0
 26 for line in f.readlines():
 27     tmp = line.strip()
 28     dict2[tmp] = i
 29 
 30 
 31 for key,value in tfidf.vocabulary_.items():
 32     if key in dict1.keys():
 33         dict1[key] = value
 34     if key in dict2.keys():
 35         dict2[key]=value
 36 
 37 print(dict1)
 38 print(dict2)
 39 
 40 
 41 
 42 from pyecharts import GeoLines, Style
 43 import  echarts_china_cities_pypkg
 44 import echarts_china_counties_pypkg
 45 import echarts_china_provinces_pypkg
 46 
 47 style = Style(
 48     title_top="#fff",
 49     title_pos = "center",
 50     width=1200,
 51     height=600,
 52     background_color="#404a59"
 53 )
 54 
 55 style_geo = style.add(
 56     is_label_show=True,
 57     line_curve=0.2,
 58     line_opacity=0.6,
 59     legend_text_color="#eee",
 60     legend_pos="right",
 61     geo_effect_symbol="plane",
 62     geo_effect_symbolsize=15,
 63     label_color=['#a6c84c', '#ffa022', '#46bee9'],
 64     label_pos="right",
 65     label_formatter="{b}",
 66     label_text_color="#eee",
 67 )
 68 
 69 data_ganzhou = [
 70     ['赣州', '上海'],
 71     ['赣州', '北京'],
 72 ]
 73 
 74 
 75 
 76 
 77 data_nanchang = [
 78     ['南昌','赣州'],
 79     ['南昌','北京'],
 80     ['南昌','深圳'],
 81     ['南昌','广州'],
 82 ]
 83 
 84 
 85 
 86 for key,value in dict1.items():
 87     if key=="南昌":
 88         continue
 89     else:
 90         if value !=0:
 91             tmp = ['南昌']
 92             tmp.append(key)
 93             data_nanchang.append(tmp)
 94 
 95 print(data_nanchang)
 96 
 97 geolines = GeoLines("各省份联系图 ", **style.init_style)
 98 geolines.add("赣州", data_ganzhou,**style_geo)
 99 geolines.add("南昌", data_nanchang,**style_geo)
100 
101 geolines.render()