From 1e93ed693ee5522b01b5e7e13ddca1451c8d5294 Mon Sep 17 00:00:00 2001
From: YYL469 <2049360881@qq.com>
Date: Tue, 2 Jul 2024 19:02:18 +0800
Subject: [PATCH 1/3] =?UTF-8?q?=E3=80=90spiderNav.py=E3=80=91=E5=AE=9E?=
=?UTF-8?q?=E7=8E=B0=E5=AF=BC=E8=88=AA=E6=A0=8F=E7=88=AC=E5=8F=96?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
spider/spiderNav.py | 53 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 53 insertions(+)
create mode 100644 spider/spiderNav.py
diff --git a/spider/spiderNav.py b/spider/spiderNav.py
new file mode 100644
index 0000000..fbcdf7d
--- /dev/null
+++ b/spider/spiderNav.py
@@ -0,0 +1,53 @@
+import requests
+import csv
+import numpy as np
+import os
+
+def init():
+ if not os.path.exists('./navData.csv'):
+ with open('./navData.csv','w',encoding='utf-8',newline='') as csvFile:
+ writer = csv.writer(csvFile)
+ writer.writerow([
+ 'typeName',
+ 'gid',
+ 'containerid'
+ ])
+
+def writerRow(row):
+ with open('./navData.csv', 'a', encoding='utf-8', newline='') as csvFile:
+ writer = csv.writer(csvFile)
+ writer.writerow(row)
+
+def get_data(url):
+ headers = {
+ 'Cookie':'SINAGLOBAL=2555941826014.1074.1676801766625; ULV=1719829459275:6:1:2:4660996305989.918.1719827559898:1719743122299; UOR=,,www.baidu.com; XSRF-TOKEN=VtLXviYSIs8lor7sz4iGyigL; SUB=_2A25LhvU9DeRhGeFH6FIX-S3MyD2IHXVo-gj1rDV8PUJbkNAGLRXMkW1Ne2nhI3Gle25QJK0Z99J3trq_NZn6YKJ-; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WW3Mv8V5EupQbbKh.vaZIwU5JpX5KzhUgL.FoM4e05c1Ke7e022dJLoIp7LxKML1KBLBKnLxKqL1hnLBoM41hz41hqReKqN; WBPSESS=Dt2hbAUaXfkVprjyrAZT_LRaDLsnxG-kIbeYwnBb5OUKZiwfVr_UrcYfWuqG-4ZVDM5HeU3HXkDNK_thfRfdS9Ao6ezT30jDksv-CpaVmlTAqGUHjJ7PYkH5aCK4HLxmRq14ZalmQNwzfWMPa4y0VNRLuYdg7L1s49ymNq_5v5vusoz0r4ki6u-MHGraF0fbUTgX14x0kHayEwOoxfLI-w==; SCF=AqmJWo31oFV5itnRgWNU1-wHQTL6PmkBLf3gDuqpdqAIfaWguDTMre6Oxjf5Uzs74JAh2r0DdV1sJ1g6m-wJ5NQ.; _s_tentry=-; Apache=4660996305989.918.1719827559898; PC_TOKEN=7955a7ab1f; appkey=; geetest_token=602cd4e3a7ed1898808f8adfe1a2048b; ALF=1722421868',
+ 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0'
+ }
+ params = {
+ 'is_new_segment':1,
+ 'fetch_hot':1
+ }
+ response = requests.get(url,headers=headers,params=params)
+ if response.status_code == 200:
+ return response.json()
+ else:
+ return None
+
+def parse_json(response):
+ navList = np.append(response['groups'][3]['group'],response['groups'][4]['group'])
+ for nav in navList:
+ navName = nav['title']
+ gid = nav['gid']
+ containerid = nav['containerid']
+ writerRow([
+ navName,
+ gid,
+ containerid
+ ])
+
+
+if __name__ == '__main__':
+ init()
+ url = 'https://weibo.com/ajax/feed/allGroups'
+ response = get_data(url)
+ parse_json(response)
\ No newline at end of file
From f4f9c098d46906d7e4238357678b1965289a8733 Mon Sep 17 00:00:00 2001
From: YYL469 <2049360881@qq.com>
Date: Tue, 2 Jul 2024 19:37:10 +0800
Subject: [PATCH 2/3] =?UTF-8?q?=E3=80=90spiderContent.py=E3=80=91=E5=AE=9E?=
=?UTF-8?q?=E7=8E=B0=E6=96=87=E7=AB=A0=E5=86=85=E5=AE=B9=E7=88=AC=E5=8F=96?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
spider/spiderContent.py | 123 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 123 insertions(+)
create mode 100644 spider/spiderContent.py
diff --git a/spider/spiderContent.py b/spider/spiderContent.py
new file mode 100644
index 0000000..6da3a57
--- /dev/null
+++ b/spider/spiderContent.py
@@ -0,0 +1,123 @@
+import time
+import requests
+import csv
+import os
+from datetime import datetime
+
+def init():
+ if not os.path.exists('./articleData.csv'):
+ with open('./articleData.csv','w',encoding='utf-8',newline='') as csvFile:
+ writer = csv.writer(csvFile)
+ writer.writerow([
+ 'id',
+ 'likeNum',
+ 'commentsLen',
+ 'reposts_count',
+ 'region',
+ 'content',
+ 'contentLen',
+ 'created_at',
+ 'type',
+ 'detailUrl',# followBtnCode>uid + mblogid
+ 'authorAvatar',
+ 'authorName',
+ 'authorDetail',
+ 'isVip' # v_plus
+ ])
+
+def writerRow(row):
+ with open('./articleData.csv', 'a', encoding='utf-8', newline='') as csvFile:
+ writer = csv.writer(csvFile)
+ writer.writerow(row)
+
+def get_data(url,params):
+ headers = {
+ 'Cookie':'SINAGLOBAL=2555941826014.1074.1676801766625; ULV=1719829459275:6:1:2:4660996305989.918.1719827559898:1719743122299; UOR=,,www.baidu.com; XSRF-TOKEN=VtLXviYSIs8lor7sz4iGyigL; SUB=_2A25LhvU9DeRhGeFH6FIX-S3MyD2IHXVo-gj1rDV8PUJbkNAGLRXMkW1Ne2nhI3Gle25QJK0Z99J3trq_NZn6YKJ-; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WW3Mv8V5EupQbbKh.vaZIwU5JpX5KzhUgL.FoM4e05c1Ke7e022dJLoIp7LxKML1KBLBKnLxKqL1hnLBoM41hz41hqReKqN; WBPSESS=Dt2hbAUaXfkVprjyrAZT_LRaDLsnxG-kIbeYwnBb5OUKZiwfVr_UrcYfWuqG-4ZVDM5HeU3HXkDNK_thfRfdS9Ao6ezT30jDksv-CpaVmlTAqGUHjJ7PYkH5aCK4HLxmRq14ZalmQNwzfWMPa4y0VNRLuYdg7L1s49ymNq_5v5vusoz0r4ki6u-MHGraF0fbUTgX14x0kHayEwOoxfLI-w==; SCF=AqmJWo31oFV5itnRgWNU1-wHQTL6PmkBLf3gDuqpdqAIfaWguDTMre6Oxjf5Uzs74JAh2r0DdV1sJ1g6m-wJ5NQ.; _s_tentry=-; Apache=4660996305989.918.1719827559898; PC_TOKEN=7955a7ab1f; appkey=; geetest_token=602cd4e3a7ed1898808f8adfe1a2048b; ALF=1722421868',
+ 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0'
+ }
+ response = requests.get(url,headers=headers,params=params)
+ if response.status_code == 200:
+ return response.json()['statuses']
+ else:
+ return None
+
+def getAllTypeList():
+ typeList = []
+ with open('./navData.csv','r',encoding='utf-8') as reader:
+ readerCsv = csv.reader(reader)
+ next(reader)
+ for nav in readerCsv:
+ typeList.append(nav)
+ return typeList
+
+def parse_json(response,type):
+ for artice in response:
+ id = artice['id']
+ likeNum = artice['attitudes_count']
+ commentsLen = artice['comments_count']
+ reposts_count = artice['reposts_count']
+ try:
+ region = artice['region_name'].replace('发布于 ', '')
+ except:
+ region = '无'
+ content = artice['text_raw']
+ contentLen = artice['textLength']
+ created_at = datetime.strptime(artice['created_at'],'%a %b %d %H:%M:%S %z %Y').strftime('%Y-%m-%d')
+ type = type
+ try:
+ detailUrl = 'https://weibo.com/' + str(artice['id']) + '/' + str(artice['mblogid'])
+ except:
+ detailUrl = '无'
+ authorAvatar = artice['user']['avatar_large']
+ authorName = artice['user']['screen_name']
+ authorDetail = 'https://weibo.com/u/' + str(artice['user']['id'])
+ isVip = artice['user']['v_plus']
+ writerRow([
+ id,
+ likeNum,
+ commentsLen,
+ reposts_count,
+ region,
+ content,
+ contentLen,
+ created_at,
+ type,
+ detailUrl,
+ authorAvatar,
+ authorName,
+ authorDetail,
+ isVip
+ ])
+
+def start(typeNum=1,pageNum=1):
+ articleUrl = 'https://weibo.com/ajax/feed/hottimeline'
+ init()
+ typeList = getAllTypeList()
+ typeNumCount = 0
+ for type in typeList:
+ if typeNumCount > typeNum:return
+ time.sleep(2)
+ for page in range(0,pageNum):
+ print('正在爬取的类型：%s 中的第%s页文章数据' % (type[0],page + 1))
+ time.sleep(1)
+ parmas = {
+ 'group_id':type[1],
+ 'containerid':type[2],
+ 'max_id':page,
+ 'count':10,
+ 'extparam':'discover|new_feed'
+ }
+ response = get_data(articleUrl,parmas)
+ parse_json(response,type[0])
+ typeNumCount += 1
+
+if __name__ == '__main__':
+ start()
+
+
+
+
+
+
+
+
From 3ee5a70fa646ae0c1e94aa95b7fbd78326881101 Mon Sep 17 00:00:00 2001
From: redhong-xy <13591594+redhong-xy@user.noreply.gitee.com>
Date: Tue, 2 Jul 2024 20:41:09 +0800
Subject: [PATCH 3/3] hong
---
.idea/.gitignore | 5 +++++
.idea/Weibo_PublicOpinion_AnalysisSystem.iml | 19 +++++++++++++++++++
.idea/inspectionProfiles/Project_Default.xml | 17 +++++++++++++++++
.../inspectionProfiles/profiles_settings.xml | 6 ++++++
.idea/misc.xml | 7 +++++++
.idea/modules.xml | 8 ++++++++
.idea/vcs.xml | 6 ++++++
model/cipingTotal.py | 0
model/sentiment.marshal.3.py | 0
9 files changed, 68 insertions(+)
create mode 100644 .idea/.gitignore
create mode 100644 .idea/Weibo_PublicOpinion_AnalysisSystem.iml
create mode 100644 .idea/inspectionProfiles/Project_Default.xml
create mode 100644 .idea/inspectionProfiles/profiles_settings.xml
create mode 100644 .idea/misc.xml
create mode 100644 .idea/modules.xml
create mode 100644 .idea/vcs.xml
create mode 100644 model/cipingTotal.py
create mode 100644 model/sentiment.marshal.3.py
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..b58b603
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,5 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
diff --git a/.idea/Weibo_PublicOpinion_AnalysisSystem.iml b/.idea/Weibo_PublicOpinion_AnalysisSystem.iml
new file mode 100644
index 0000000..a4e9024
--- /dev/null
+++ b/.idea/Weibo_PublicOpinion_AnalysisSystem.iml
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000..b8ef0fa
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,17 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..31a993c
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..b4049e4
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..35eb1dd
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/model/cipingTotal.py b/model/cipingTotal.py
new file mode 100644
index 0000000..e69de29
diff --git a/model/sentiment.marshal.3.py b/model/sentiment.marshal.3.py
new file mode 100644
index 0000000..e69de29