juanboy
35 lines
1.4 KiB
Python
35 lines
1.4 KiB
Python
import os
|
|
from sqlalchemy import create_engine
|
|
import pandas as pd
|
|
from spiderDataPackage.settings import articleAddr,commentsAddr
|
|
from model.topicDefine import *
|
|
|
|
engine = create_engine('mysql+pymysql://XiaoXueQi:XiaoXueQi@47.92.235.6/Weibo_PublicOpinion_AnalysisSystem?charset=utf8mb4')
|
|
|
|
def saveData():
|
|
try:
|
|
oldArticle = pd.read_sql('select * from article',engine)
|
|
newArticle = pd.read_csv(articleAddr)
|
|
oldComment = pd.read_sql('select * from comments',engine)
|
|
newComment = pd.read_csv(commentsAddr)
|
|
|
|
mergeArticle = pd.concat([newArticle,oldArticle],join='inner')
|
|
mergeComment = pd.concat([newComment,oldComment],join='inner')
|
|
|
|
mergeArticle.drop_duplicates(subset='id',keep='last',inplace=True)
|
|
mergeComment.drop_duplicates(subset='content',keep='last',inplace=True)
|
|
|
|
mergeArticle.to_sql('article', con=engine, if_exists='replace', index=False)
|
|
mergeComment.to_sql('comments', con=engine, if_exists='replace', index=False)
|
|
except:
|
|
newArticle = pd.read_csv(articleAddr)
|
|
newComment = pd.read_csv(commentsAddr)
|
|
newArticle.to_sql('article',con=engine,if_exists='replace',index=False)
|
|
newComment.to_sql('comments',con=engine,if_exists='replace',index=False)
|
|
|
|
os.remove(articleAddr)
|
|
os.remove(commentsAddr)
|
|
update_data()
|
|
|
|
if __name__ == '__main__':
|
|
saveData() |