panda/bettafish-company
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1331a5b9839b6290fd21ca2c67ef201eb43ff2fa
bettafish-company/WeiboSentiment_MachineLearning/5.bert.ipynb
T

697 lines
24 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"cell_id": 39
},
"source": [
"### 加载数据集"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"cell_id": 1
},
"outputs": [],
"source": [
"from utils import load_corpus_bert\n",
"\n",
"TRAIN_PATH = \"./data/weibo2018/train.txt\"\n",
"TEST_PATH = \"./data/weibo2018/test.txt\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"cell_id": 3
},
"outputs": [],
"source": [
"# 分别加载训练集和测试集\n",
"train_data = load_corpus_bert(TRAIN_PATH)\n",
"test_data = load_corpus_bert(TEST_PATH)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"cell_id": 4
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>label</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>“书中自有黄金屋，书中自有颜如玉”。沿着岁月的长河跋涉，或是风光旖旎，或是姹紫嫣红，万千...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>这是英超被黑的最惨的一次[二哈][二哈]十几年来，中国只有孙继海，董方卓，郑智，李铁登陆过英...</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>中国远洋海运集团副总经理俞曾港4月21日在 上表示，中央企业“走出去”是要站在更高的平台参...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>看《流星花园》其实也还好啦，现在的观念以及时尚眼光都不一样了，或许十几年之后的人看我们的现在...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>汉武帝的罪己诏的真实性尽管存在着争议，然而“轮台罪己诏”作为中国历史上第一份皇帝自我批评的文...</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" text label\n",
"0 “书中自有黄金屋，书中自有颜如玉”。沿着岁月的长河跋涉，或是风光旖旎，或是姹紫嫣红，万千... 1\n",
"1 这是英超被黑的最惨的一次[二哈][二哈]十几年来，中国只有孙继海，董方卓，郑智，李铁登陆过英... 0\n",
"2 中国远洋海运集团副总经理俞曾港4月21日在 上表示，中央企业“走出去”是要站在更高的平台参... 1\n",
"3 看《流星花园》其实也还好啦，现在的观念以及时尚眼光都不一样了，或许十几年之后的人看我们的现在... 1\n",
"4 汉武帝的罪己诏的真实性尽管存在着争议，然而“轮台罪己诏”作为中国历史上第一份皇帝自我批评的文... 1"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import pandas as pd\n",
"\n",
"df_train = pd.DataFrame(train_data, columns=[\"text\", \"label\"])\n",
"df_test = pd.DataFrame(test_data, columns=[\"text\", \"label\"])\n",
"df_train.head()"
]
},
{
"cell_type": "markdown",
"metadata": {
"cell_id": 41
},
"source": [
"### 加载Bert"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"cell_id": 5
},
"outputs": [],
"source": [
"import os\n",
"from transformers import BertTokenizer, BertModel\n",
"\n",
"os.environ[\"KMP_DUPLICATE_LIB_OK\"] = \"TRUE\" # 在我的电脑上不加这一句, bert模型会报错\n",
"MODEL_PATH = \"./model/chinese_wwm_pytorch\" # 下载地址见 https://github.com/ymcui/Chinese-BERT-wwm"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"cell_id": 6
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Some weights of the model checkpoint at ./model/chinese_wwm_pytorch were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight']\n",
"- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
]
}
],
"source": [
"# 加载\n",
"tokenizer = BertTokenizer.from_pretrained(MODEL_PATH) # 分词器\n",
"bert = BertModel.from_pretrained(MODEL_PATH) # 模型"
]
},
{
"cell_type": "markdown",
"metadata": {
"cell_id": 43
},
"source": [
"### 神经网络"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"cell_id": 7
},
"outputs": [],
"source": [
"import torch\n",
"from torch import nn\n",
"from torch.utils.data import Dataset, DataLoader\n",
"\n",
"device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\""
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"cell_id": 8
},
"outputs": [],
"source": [
"# 超参数\n",
"learning_rate = 1e-3\n",
"input_size = 768\n",
"num_epoches = 10\n",
"batch_size = 100\n",
"decay_rate = 0.9"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"cell_id": 9
},
"outputs": [],
"source": [
"# 数据集\n",
"class MyDataset(Dataset):\n",
" def __init__(self, df):\n",
" self.data = df[\"text\"].tolist()\n",
" self.label = df[\"label\"].tolist()\n",
"\n",
" def __getitem__(self, index):\n",
" data = self.data[index]\n",
" label = self.label[index]\n",
" return data, label\n",
"\n",
" def __len__(self):\n",
" return len(self.label)\n",
"\n",
"# 训练集\n",
"train_data = MyDataset(df_train)\n",
"train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
"\n",
"# 测试集\n",
"test_data = MyDataset(df_test)\n",
"test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=True)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"cell_id": 10
},
"outputs": [],
"source": [
"# 网络结构\n",
"class Net(nn.Module):\n",
" def __init__(self, input_size):\n",
" super(Net, self).__init__()\n",
" self.fc = nn.Linear(input_size, 1)\n",
" self.sigmoid = nn.Sigmoid()\n",
"\n",
" def forward(self, x):\n",
" out = self.fc(x)\n",
" out = self.sigmoid(out)\n",
" return out\n",
"\n",
"net = Net(input_size).to(device)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"cell_id": 34
},
"outputs": [],
"source": [
"from sklearn import metrics\n",
"\n",
"# 测试集效果检验\n",
"def test():\n",
" y_pred, y_true = [], []\n",
"\n",
" with torch.no_grad():\n",
" for words, labels in test_loader:\n",
" tokens = tokenizer(words, padding=True)\n",
" input_ids = torch.tensor(tokens[\"input_ids\"]).to(device)\n",
" attention_mask = torch.tensor(tokens[\"attention_mask\"]).to(device)\n",
" last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
" bert_output = last_hidden_states[0][:, 0]\n",
" outputs = net(bert_output) # 前向传播\n",
" outputs = outputs.view(-1) # 将输出展平\n",
" y_pred.append(outputs)\n",
" y_true.append(labels)\n",
"\n",
" y_prob = torch.cat(y_pred)\n",
" y_true = torch.cat(y_true)\n",
" y_pred = y_prob.clone()\n",
" y_pred[y_pred > 0.5] = 1\n",
" y_pred[y_pred <= 0.5] = 0\n",
" \n",
" print(metrics.classification_report(y_true, y_pred))\n",
" print(\"准确率:\", metrics.accuracy_score(y_true, y_pred))\n",
" print(\"AUC:\", metrics.roc_auc_score(y_true, y_prob) )"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"cell_id": 11
},
"outputs": [],
"source": [
"# 定义损失函数和优化器\n",
"criterion = nn.BCELoss()\n",
"optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)\n",
"scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=decay_rate)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"cell_id": 14,
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch:1, step:10, loss:0.6710587739944458\n",
"epoch:1, step:20, loss:0.6176288723945618\n",
"epoch:1, step:30, loss:0.578593909740448\n",
"epoch:1, step:40, loss:0.5502474308013916\n",
"epoch:1, step:50, loss:0.5323082804679871\n",
"epoch:1, step:60, loss:0.515110194683075\n",
"epoch:1, step:70, loss:0.5127577185630798\n",
"epoch:1, step:80, loss:0.48992329835891724\n",
"epoch:1, step:90, loss:0.4868148863315582\n",
"epoch:1, step:100, loss:0.49194520711898804\n",
" precision recall f1-score support\n",
"\n",
" 0 0.74 0.74 0.74 155\n",
" 1 0.88 0.88 0.88 345\n",
"\n",
" accuracy 0.84 500\n",
" macro avg 0.81 0.81 0.81 500\n",
"weighted avg 0.84 0.84 0.84 500\n",
"\n",
"准确率: 0.84\n",
"AUC: 0.9027582982702197\n",
"saved model: ./model/bert_dnn_1.model\n",
"epoch:2, step:10, loss:0.46188774704933167\n",
"epoch:2, step:20, loss:0.4335215985774994\n",
"epoch:2, step:30, loss:0.4540901184082031\n",
"epoch:2, step:40, loss:0.4392821788787842\n",
"epoch:2, step:50, loss:0.47116056084632874\n",
"epoch:2, step:60, loss:0.4669877886772156\n",
"epoch:2, step:70, loss:0.4401330053806305\n",
"epoch:2, step:80, loss:0.4518135190010071\n",
"epoch:2, step:90, loss:0.4567466676235199\n",
"epoch:2, step:100, loss:0.4663034975528717\n",
" precision recall f1-score support\n",
"\n",
" 0 0.72 0.83 0.77 155\n",
" 1 0.92 0.85 0.88 345\n",
"\n",
" accuracy 0.85 500\n",
" macro avg 0.82 0.84 0.83 500\n",
"weighted avg 0.86 0.85 0.85 500\n",
"\n",
"准确率: 0.846\n",
"AUC: 0.9149322113136981\n",
"saved model: ./model/bert_dnn_2.model\n",
"epoch:3, step:10, loss:0.42892661690711975\n",
"epoch:3, step:20, loss:0.4225884974002838\n",
"epoch:3, step:30, loss:0.415252685546875\n",
"epoch:3, step:40, loss:0.43130287528038025\n",
"epoch:3, step:50, loss:0.42938193678855896\n",
"epoch:3, step:60, loss:0.4340507388114929\n",
"epoch:3, step:70, loss:0.4466826319694519\n",
"epoch:3, step:80, loss:0.45244288444519043\n",
"epoch:3, step:90, loss:0.41808539628982544\n",
"epoch:3, step:100, loss:0.44330015778541565\n",
" precision recall f1-score support\n",
"\n",
" 0 0.73 0.83 0.77 155\n",
" 1 0.92 0.86 0.89 345\n",
"\n",
" accuracy 0.85 500\n",
" macro avg 0.82 0.84 0.83 500\n",
"weighted avg 0.86 0.85 0.85 500\n",
"\n",
"准确率: 0.85\n",
"AUC: 0.9206545114539505\n",
"saved model: ./model/bert_dnn_3.model\n",
"epoch:4, step:10, loss:0.39769938588142395\n",
"epoch:4, step:20, loss:0.4465697407722473\n",
"epoch:4, step:30, loss:0.4216257929801941\n",
"epoch:4, step:40, loss:0.41328248381614685\n",
"epoch:4, step:50, loss:0.41364049911499023\n",
"epoch:4, step:60, loss:0.4332212507724762\n",
"epoch:4, step:70, loss:0.4280005395412445\n",
"epoch:4, step:80, loss:0.41606149077415466\n",
"epoch:4, step:90, loss:0.43310579657554626\n",
"epoch:4, step:100, loss:0.4076871871948242\n",
" precision recall f1-score support\n",
"\n",
" 0 0.76 0.80 0.78 155\n",
" 1 0.91 0.88 0.90 345\n",
"\n",
" accuracy 0.86 500\n",
" macro avg 0.83 0.84 0.84 500\n",
"weighted avg 0.86 0.86 0.86 500\n",
"\n",
"准确率: 0.858\n",
"AUC: 0.9222814399251986\n",
"saved model: ./model/bert_dnn_4.model\n",
"epoch:5, step:10, loss:0.39923620223999023\n",
"epoch:5, step:20, loss:0.4110904633998871\n",
"epoch:5, step:30, loss:0.4446052610874176\n",
"epoch:5, step:40, loss:0.4050986170768738\n",
"epoch:5, step:50, loss:0.41362982988357544\n",
"epoch:5, step:60, loss:0.3961515724658966\n",
"epoch:5, step:80, loss:0.43208274245262146\n",
"epoch:5, step:90, loss:0.4123595356941223\n",
"epoch:5, step:100, loss:0.4114747643470764\n",
" precision recall f1-score support\n",
"\n",
" 0 0.75 0.81 0.78 155\n",
" 1 0.91 0.88 0.90 345\n",
"\n",
" accuracy 0.86 500\n",
" macro avg 0.83 0.85 0.84 500\n",
"weighted avg 0.86 0.86 0.86 500\n",
"\n",
"准确率: 0.86\n",
"AUC: 0.9251238896680692\n",
"saved model: ./model/bert_dnn_5.model\n",
"epoch:6, step:10, loss:0.4047953188419342\n",
"epoch:6, step:20, loss:0.41434162855148315\n",
"epoch:6, step:30, loss:0.4052816927433014\n",
"epoch:6, step:40, loss:0.3726503849029541\n",
"epoch:6, step:50, loss:0.4252064824104309\n",
"epoch:6, step:60, loss:0.411870539188385\n",
"epoch:6, step:70, loss:0.43613123893737793\n",
"epoch:6, step:80, loss:0.4038943350315094\n",
"epoch:6, step:90, loss:0.40738430619239807\n",
"epoch:6, step:100, loss:0.41697797179222107\n",
" precision recall f1-score support\n",
"\n",
" 0 0.76 0.82 0.79 155\n",
" 1 0.92 0.88 0.90 345\n",
"\n",
" accuracy 0.86 500\n",
" macro avg 0.84 0.85 0.84 500\n",
"weighted avg 0.87 0.86 0.87 500\n",
"\n",
"准确率: 0.864\n",
"AUC: 0.9266947171575501\n",
"saved model: ./model/bert_dnn_6.model\n",
"epoch:7, step:10, loss:0.4255238175392151\n",
"epoch:7, step:20, loss:0.3951468765735626\n",
"epoch:7, step:30, loss:0.41892367601394653\n",
"epoch:7, step:40, loss:0.40587490797042847\n",
"epoch:7, step:50, loss:0.3918803036212921\n",
"epoch:7, step:60, loss:0.43665409088134766\n",
"epoch:7, step:70, loss:0.4085603654384613\n",
"epoch:7, step:80, loss:0.3877314627170563\n",
"epoch:7, step:90, loss:0.3680875301361084\n",
"epoch:7, step:100, loss:0.4211949408054352\n",
" precision recall f1-score support\n",
"\n",
" 0 0.74 0.85 0.79 155\n",
" 1 0.93 0.87 0.90 345\n",
"\n",
" accuracy 0.86 500\n",
" macro avg 0.83 0.86 0.84 500\n",
"weighted avg 0.87 0.86 0.86 500\n",
"\n",
"准确率: 0.86\n",
"AUC: 0.9282094436652641\n",
"saved model: ./model/bert_dnn_7.model\n",
"epoch:8, step:10, loss:0.3657851815223694\n",
"epoch:8, step:20, loss:0.3944622576236725\n",
"epoch:8, step:30, loss:0.40657711029052734\n",
"epoch:8, step:40, loss:0.3935934901237488\n",
"epoch:8, step:50, loss:0.4171984791755676\n",
"epoch:8, step:60, loss:0.4169773459434509\n",
"epoch:8, step:70, loss:0.4021885395050049\n",
"epoch:8, step:80, loss:0.4106557369232178\n",
"epoch:8, step:100, loss:0.4116268754005432\n",
" precision recall f1-score support\n",
"\n",
" 0 0.80 0.78 0.79 155\n",
" 1 0.90 0.91 0.91 345\n",
"\n",
" accuracy 0.87 500\n",
" macro avg 0.85 0.85 0.85 500\n",
"weighted avg 0.87 0.87 0.87 500\n",
"\n",
"准确率: 0.87\n",
"AUC: 0.9288078541374474\n",
"saved model: ./model/bert_dnn_8.model\n",
"epoch:9, step:10, loss:0.4415532052516937\n",
"epoch:9, step:20, loss:0.4093624949455261\n",
"epoch:9, step:30, loss:0.3825526833534241\n",
"epoch:9, step:40, loss:0.3692132532596588\n",
"epoch:9, step:50, loss:0.39409342408180237\n",
"epoch:9, step:60, loss:0.40440621972084045\n",
"epoch:9, step:70, loss:0.3859332203865051\n",
"epoch:9, step:80, loss:0.40987101197242737\n",
"epoch:9, step:90, loss:0.4061252176761627\n",
"epoch:9, step:100, loss:0.4131951332092285\n",
" precision recall f1-score support\n",
"\n",
" 0 0.75 0.84 0.79 155\n",
" 1 0.92 0.88 0.90 345\n",
"\n",
" accuracy 0.86 500\n",
" macro avg 0.84 0.86 0.85 500\n",
"weighted avg 0.87 0.86 0.87 500\n",
"\n",
"准确率: 0.864\n",
"AUC: 0.9293501636278635\n",
"saved model: ./model/bert_dnn_9.model\n",
"epoch:10, step:10, loss:0.40611615777015686\n",
"epoch:10, step:20, loss:0.42403316497802734\n",
"epoch:10, step:30, loss:0.3972412943840027\n",
"epoch:10, step:40, loss:0.4144269526004791\n",
"epoch:10, step:50, loss:0.37967294454574585\n",
"epoch:10, step:60, loss:0.3992181420326233\n",
"epoch:10, step:70, loss:0.3896545469760895\n",
"epoch:10, step:80, loss:0.39779797196388245\n",
"epoch:10, step:90, loss:0.38316115736961365\n",
"epoch:10, step:100, loss:0.4042983055114746\n",
" precision recall f1-score support\n",
"\n",
" 0 0.76 0.82 0.79 155\n",
" 1 0.92 0.88 0.90 345\n",
"\n",
" accuracy 0.86 500\n",
" macro avg 0.84 0.85 0.84 500\n",
"weighted avg 0.87 0.86 0.86 500\n",
"\n",
"准确率: 0.862\n",
"AUC: 0.9303973819541842\n",
"saved model: ./model/bert_dnn_10.model\n"
]
}
],
"source": [
"# 迭代训练\n",
"for epoch in range(num_epoches):\n",
" total_loss = 0\n",
" for i, (words, labels) in enumerate(train_loader):\n",
" tokens = tokenizer(words, padding=True)\n",
" input_ids = torch.tensor(tokens[\"input_ids\"]).to(device)\n",
" attention_mask = torch.tensor(tokens[\"attention_mask\"]).to(device)\n",
" labels = labels.float().to(device)\n",
" with torch.no_grad():\n",
" last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
" bert_output = last_hidden_states[0][:, 0]\n",
" optimizer.zero_grad() # 梯度清零\n",
" outputs = net(bert_output) # 前向传播\n",
" logits = outputs.view(-1) # 将输出展平\n",
" loss = criterion(logits, labels) # loss计算\n",
" total_loss += loss\n",
" loss.backward() # 反向传播，计算梯度\n",
" optimizer.step() # 梯度更新\n",
" if (i+1) % 10 == 0:\n",
" print(\"epoch:{}, step:{}, loss:{}\".format(epoch+1, i+1, total_loss/10))\n",
" total_loss = 0\n",
" \n",
" # learning_rate decay\n",
" scheduler.step()\n",
" \n",
" # test\n",
" test()\n",
" \n",
" # save model\n",
" model_path = \"./model/bert_dnn_{}.model\".format(epoch+1)\n",
" torch.save(net, model_path)\n",
" print(\"saved model: \", model_path)"
]
},
{
"cell_type": "markdown",
"metadata": {
"cell_id": 23
},
"source": [
"### 手动输入句子，判断情感倾向（1正/0负）"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"cell_id": 38
},
"outputs": [],
"source": [
"net = torch.load(\"./model/bert_dnn_8.model\") # 训练过程中的巅峰时刻"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"cell_id": 37
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[0.9007],\n",
" [0.2211]], grad_fn=<SigmoidBackward>)"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s = [\"华丽繁荣的城市、充满回忆的小镇、郁郁葱葱的山谷...\", \"突然就觉得人间不值得\"]\n",
"tokens = tokenizer(s, padding=True)\n",
"input_ids = torch.tensor(tokens[\"input_ids\"])\n",
"attention_mask = torch.tensor(tokens[\"attention_mask\"])\n",
"last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
"bert_output = last_hidden_states[0][:, 0]\n",
"outputs = net(bert_output)\n",
"outputs"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"cell_id": 27,
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[0.9735],\n",
" [0.9882]], grad_fn=<SigmoidBackward>)"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s = [\"今天天气真好\", \"今天天气特别特别棒\"]\n",
"tokens = tokenizer(s, padding=True)\n",
"input_ids = torch.tensor(tokens[\"input_ids\"])\n",
"attention_mask = torch.tensor(tokens[\"attention_mask\"])\n",
"last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
"bert_output = last_hidden_states[0][:, 0]\n",
"outputs = net(bert_output)\n",
"outputs"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cell_id": 32
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
},
"max_cell_id": 45
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink