修改spider文件夹结构,优化数据爬取结构,实现模块化
This commit is contained in:
+7
-8
@@ -1,14 +1,13 @@
|
||||
from spiderContent import start as spiderContentStart
|
||||
from spiderComments import start as spiderCommentsStart
|
||||
from spiderData import spiderData
|
||||
from saveData import save_to_sql as saveData
|
||||
|
||||
def main():
|
||||
print('正在爬取文章数据')
|
||||
spiderContentStart(1,1)
|
||||
print('正在爬取文章评论数据')
|
||||
spiderCommentsStart()
|
||||
print('正在存储数据')
|
||||
saveData()
|
||||
try:
|
||||
spiderData()
|
||||
saveData()
|
||||
print("爬取数据更新")
|
||||
except:
|
||||
print("爬取数据失败")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user