Local sentiment analysis upload.

2025-08-23 15:55:07 +08:00
parent 4e33224633
commit f448ddd466
65 changed files with 1563359 additions and 11 deletions
@@ -0,0 +1,78 @@
+# 微博情感分析 - 基于BertChinese的微调模型
+
+本模块使用HuggingFace上的预训练微博情感分析模型进行情感分析。
+
+## 模型信息
+
+- **模型名称**: wsqstar/GISchat-weibo-100k-fine-tuned-bert  
+- **模型类型**: BERT中文情感分类模型
+- **训练数据**: 10万条微博数据
+- **输出**: 二分类（正面/负面情感）
+
+## 使用方法
+
+### 方法1: 直接模型调用 (推荐)
+```bash
+python predict.py
+```
+
+### 方法2: Pipeline方式
+```bash
+python predict_pipeline.py
+```
+
+## 快速开始
+
+1. 确保已安装依赖：
+```bash
+pip install transformers torch
+```
+
+2. 运行预测程序：
+```bash
+python predict.py
+```
+
+3. 输入微博文本进行分析：
+```
+请输入微博内容: 今天天气真好，心情特别棒！
+预测结果: 正面情感 (置信度: 0.9234)
+```
+
+## 代码示例
+
+```python
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
+
+# 加载模型
+model_name = "wsqstar/GISchat-weibo-100k-fine-tuned-bert"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForSequenceClassification.from_pretrained(model_name)
+
+# 预测
+text = "今天心情很好"
+inputs = tokenizer(text, return_tensors="pt")
+outputs = model(**inputs)
+prediction = torch.argmax(outputs.logits, dim=1).item()
+print("正面情感" if prediction == 1 else "负面情感")
+```
+
+## 文件说明
+
+- `predict.py`: 主预测程序，使用直接模型调用
+- `predict_pipeline.py`: 使用pipeline方式的预测程序  
+- `README.md`: 使用说明
+
+## 模型存储
+
+- 首次运行时会自动下载模型到当前目录的 `model` 文件夹
+- 后续运行会直接从本地加载，无需重复下载
+- 模型大小约400MB，首次下载需要网络连接
+
+## 注意事项
+
+- 首次运行时会自动下载模型，需要网络连接
+- 模型会保存到当前目录，方便后续使用
+- 支持GPU加速，会自动检测可用设备
+- 如需清理模型文件，删除 `model` 文件夹即可
@@ -0,0 +1,90 @@
+import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline
+import re
+
+def preprocess_text(text):
+    return text
+
+def main():
+    print("正在加载微博情感分析模型...")
+    
+    # 使用HuggingFace预训练模型
+    model_name = "wsqstar/GISchat-weibo-100k-fine-tuned-bert"
+    local_model_path = "./model"
+    
+    try:
+        # 检查本地是否已有模型
+        import os
+        if os.path.exists(local_model_path):
+            print("从本地加载模型...")
+            tokenizer = AutoTokenizer.from_pretrained(local_model_path)
+            model = AutoModelForSequenceClassification.from_pretrained(local_model_path)
+        else:
+            print("首次使用，正在下载模型到本地...")
+            # 下载并保存到本地
+            tokenizer = AutoTokenizer.from_pretrained(model_name)
+            model = AutoModelForSequenceClassification.from_pretrained(model_name)
+            
+            # 保存到本地
+            tokenizer.save_pretrained(local_model_path)
+            model.save_pretrained(local_model_path)
+            print(f"模型已保存到: {local_model_path}")
+        
+        # 设置设备
+        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        model.to(device)
+        model.eval()
+        print(f"模型加载成功! 使用设备: {device}")
+        
+    except Exception as e:
+        print(f"模型加载失败: {e}")
+        print("请检查网络连接或使用pipeline方式")
+        return
+    
+    print("\n============= 微博情感分析 =============")
+    print("输入微博内容进行分析 (输入 'q' 退出):")
+    
+    while True:
+        text = input("\n请输入微博内容: ")
+        if text.lower() == 'q':
+            break
+        
+        if not text.strip():
+            print("输入不能为空，请重新输入")
+            continue
+        
+        try:
+            # 预处理文本
+            processed_text = preprocess_text(text)
+            
+            # 分词编码
+            inputs = tokenizer(
+                processed_text,
+                max_length=512,
+                padding=True,
+                truncation=True,
+                return_tensors='pt'
+            )
+            
+            # 转移到设备
+            inputs = {k: v.to(device) for k, v in inputs.items()}
+            
+            # 预测
+            with torch.no_grad():
+                outputs = model(**inputs)
+                logits = outputs.logits
+                probabilities = torch.softmax(logits, dim=1)
+                prediction = torch.argmax(probabilities, dim=1).item()
+            
+            # 输出结果
+            confidence = probabilities[0][prediction].item()
+            label = "正面情感" if prediction == 1 else "负面情感"
+            
+            print(f"预测结果: {label} (置信度: {confidence:.4f})")
+            
+        except Exception as e:
+            print(f"预测时发生错误: {e}")
+            continue
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,101 @@
+from transformers import pipeline
+import re
+
+def preprocess_text(text):
+    """简单的文本预处理"""
+    text = re.sub(r"\{%.+?%\}", " ", text)           # 去除 {%xxx%}
+    text = re.sub(r"@.+?( |$)", " ", text)           # 去除 @xxx
+    text = re.sub(r"【.+?】", " ", text)              # 去除 【xx】
+    text = re.sub(r"\u200b", " ", text)              # 去除特殊字符
+    # 删除表情符号
+    text = re.sub(r'[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF\U00002600-\U000027BF\U0001f900-\U0001f9ff\U0001f018-\U0001f270\U0000231a-\U0000231b\U0000238d-\U0000238d\U000024c2-\U0001f251]+', '', text)
+    text = re.sub(r"\s+", " ", text)                 # 多个空格合并
+    return text.strip()
+
+def main():
+    print("正在加载微博情感分析模型...")
+    
+    # 使用pipeline方式 - 更简单
+    model_name = "wsqstar/GISchat-weibo-100k-fine-tuned-bert"
+    local_model_path = "./model"
+    
+    try:
+        # 检查本地是否已有模型
+        import os
+        if os.path.exists(local_model_path):
+            print("从本地加载模型...")
+            classifier = pipeline(
+                "text-classification", 
+                model=local_model_path,
+                return_all_scores=True
+            )
+        else:
+            print("首次使用，正在下载模型到本地...")
+            # 先下载模型
+            from transformers import AutoTokenizer, AutoModelForSequenceClassification
+            tokenizer = AutoTokenizer.from_pretrained(model_name)
+            model = AutoModelForSequenceClassification.from_pretrained(model_name)
+            
+            # 保存到本地
+            tokenizer.save_pretrained(local_model_path)
+            model.save_pretrained(local_model_path)
+            print(f"模型已保存到: {local_model_path}")
+            
+            # 使用本地模型创建pipeline
+            classifier = pipeline(
+                "text-classification", 
+                model=local_model_path,
+                return_all_scores=True
+            )
+        print("模型加载成功!")
+        
+    except Exception as e:
+        print(f"模型加载失败: {e}")
+        print("请检查网络连接")
+        return
+    
+    print("\n============= 微博情感分析 (Pipeline版) =============")
+    print("输入微博内容进行分析 (输入 'q' 退出):")
+    
+    while True:
+        text = input("\n请输入微博内容: ")
+        if text.lower() == 'q':
+            break
+        
+        if not text.strip():
+            print("输入不能为空，请重新输入")
+            continue
+        
+        try:
+            # 预处理文本
+            processed_text = preprocess_text(text)
+            
+            # 预测
+            outputs = classifier(processed_text)
+            
+            # 解析结果
+            positive_score = None
+            negative_score = None
+            
+            for output in outputs[0]:
+                if output['label'] == 'LABEL_1':  # 正面
+                    positive_score = output['score']
+                elif output['label'] == 'LABEL_0':  # 负面
+                    negative_score = output['score']
+            
+            # 确定预测结果
+            if positive_score > negative_score:
+                label = "正面情感"
+                confidence = positive_score
+            else:
+                label = "负面情感"
+                confidence = negative_score
+            
+            print(f"预测结果: {label} (置信度: {confidence:.4f})")
+            
+        except Exception as e:
+            print(f"预测时发生错误: {e}")
+            continue
+
+if __name__ == "__main__":
+    main()