preprocess_char.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess training data to BERT's format. \n",
    "### we will get train.tsv and dev.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import ended...\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "random.seed = 16\n",
    "import pandas as pd\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "import jieba\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import re\n",
    "print(\"import ended...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read csv started...\n",
      "training.shape: (105000, 22)\n",
      "data_validation.shape: (15000, 22)\n",
      "data_test_old.shape: (15000, 22)\n",
      "data_test.shape: (200000, 22)\n",
      "read csv ended...\n"
     ]
    }
   ],
   "source": [
    "print(\"read csv started...\")\n",
    "data = pd.read_csv(\"ai_challenger_sentiment_analysis_trainingset_20180816/sentiment_analysis_trainingset.csv\")\n",
    "data_validation = pd.read_csv(\"ai_challenger_sentiment_analysis_validationset_20180816/sentiment_analysis_validationset.csv\")\n",
    "data_test_old = pd.read_csv(\"ai_challenger_sentiment_analysis_testa_20180816/sentiment_analysis_testa.csv\")\n",
    "data_test = pd.read_csv(\"ai_challenger_sentimetn_analysis_testb_20180816/sentiment_analysis_testb.csv\")\n",
    "\n",
    "print(\"training.shape:\",data.shape)\n",
    "print(\"data_validation.shape:\",data_validation.shape)\n",
    "print(\"data_test_old.shape:\",data_test_old.shape)\n",
    "print(\"data_test.shape:\",data_test.shape)\n",
    "print(\"read csv ended...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(data.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   id                                            content  \\\n",
      "0   0  \"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，...   \n",
      "\n",
      "   location_traffic_convenience  location_distance_from_business_district  \\\n",
      "0                            -2                                        -2   \n",
      "\n",
      "   location_easy_to_find  service_wait_time  service_waiters_attitude  \\\n",
      "0                     -2                 -2                         1   \n",
      "\n",
      "   service_parking_convenience  service_serving_speed  price_level  \\\n",
      "0                           -2                     -2           -2   \n",
      "\n",
      "                ...                 environment_decoration  environment_noise  \\\n",
      "0               ...                                     -2                 -2   \n",
      "\n",
      "   environment_space  environment_cleaness  dish_portion  dish_taste  \\\n",
      "0                 -2                    -2            -2          -2   \n",
      "\n",
      "   dish_look  dish_recommendation  others_overall_experience  \\\n",
      "0          1                   -2                          1   \n",
      "\n",
      "   others_willing_to_consume_again  \n",
      "0                               -2  \n",
      "\n",
      "[1 rows x 22 columns]\n"
     ]
    }
   ],
   "source": [
    "print(data.head(n=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(105000, 22)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # id,content,location_traffic_convenience,location_distance_from_business_district,location_easy_to_find,service_wait_time,service_waiters_attitude,service_parking_convenience,service_serving_speed,price_level,price_cost_effective,price_discount,environment_decoration,environment_noise,environment_space,environment_cleaness,dish_portion,dish_taste,dish_look,dish_recommendation,others_overall_experience,others_willing_to_consume_again\n",
    "#  0,\"\"\"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，大众点评给了我这个土老冒一个见识的机会。看介绍棒棒糖是用德国糖做的，不会很甜，中间的照片是糯米的，能食用，真是太高端大气上档次了，还可以买蝴蝶结扎口，送人可以买礼盒。我是先打的卖家电话，加了微信，给卖家传的照片。等了几天，卖家就告诉我可以取货了，去大官屯那取的。虽然连卖家的面都没见到，但是还是谢谢卖家送我这么可爱的东西，太喜欢了，这哪舍得吃啊。\"\"\",-2,-2,-2,-2,1,-2,-2,-2,-2,1,-2,-2,-2,-2,-2,-2,1,-2,1,-2\n",
    "\n",
    "#print(data.head(n=1))\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id_: 0\n",
      "content: \"吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，大众点评给了我这个土老冒一个见识的机会。看介绍棒棒糖是用德国糖做的，不会很甜，中间的照片是糯米的，能食用，真是太高端大气上档次了，还可以买蝴蝶结扎口，送人可以买礼盒。我是先打的卖家电话，加了微信，给卖家传的照片。等了几天，卖家就告诉我可以取货了，去大官屯那取的。虽然连卖家的面都没见到，但是还是谢谢卖家送我这么可爱的东西，太喜欢了，这哪舍得吃啊。\"\n",
      "label: location_traffic_convenience                -2\n",
      "location_distance_from_business_district    -2\n",
      "location_easy_to_find                       -2\n",
      "service_wait_time                           -2\n",
      "service_waiters_attitude                     1\n",
      "service_parking_convenience                 -2\n",
      "service_serving_speed                       -2\n",
      "price_level                                 -2\n",
      "price_cost_effective                        -2\n",
      "price_discount                               1\n",
      "environment_decoration                      -2\n",
      "environment_noise                           -2\n",
      "environment_space                           -2\n",
      "environment_cleaness                        -2\n",
      "dish_portion                                -2\n",
      "dish_taste                                  -2\n",
      "dish_look                                    1\n",
      "dish_recommendation                         -2\n",
      "others_overall_experience                    1\n",
      "others_willing_to_consume_again             -2\n",
      "Name: 0, dtype: object\n",
      "===============================================================\n",
      "id_: 1\n",
      "content: \"第三次参加大众点评网霸王餐的活动。这家店给人整体感觉一般。首先环境只能算中等，其次霸王餐提供的菜品也不是很多，当然商家为了避免参加霸王餐吃不饱的现象，给每桌都提供了至少六份主食，我们那桌都提供了两份年糕，第一次吃火锅会在桌上有这么多的主食了。整体来说这家火锅店没有什么特别有特色的，不过每份菜品分量还是比较足的，这点要肯定！至于价格，因为没有看菜单不了解，不过我看大众有这家店的团购代金券，相当于7折，应该价位不会很高的！最后还是要感谢商家提供霸王餐，祝生意兴隆，财源广进\"\n",
      "label: location_traffic_convenience                -2\n",
      "location_distance_from_business_district    -2\n",
      "location_easy_to_find                       -2\n",
      "service_wait_time                           -2\n",
      "service_waiters_attitude                    -2\n",
      "service_parking_convenience                 -2\n",
      "service_serving_speed                       -2\n",
      "price_level                                  0\n",
      "price_cost_effective                        -2\n",
      "price_discount                               1\n",
      "environment_decoration                       0\n",
      "environment_noise                            0\n",
      "environment_space                            0\n",
      "environment_cleaness                         0\n",
      "dish_portion                                 1\n",
      "dish_taste                                  -2\n",
      "dish_look                                   -2\n",
      "dish_recommendation                         -2\n",
      "others_overall_experience                    1\n",
      "others_willing_to_consume_again             -2\n",
      "Name: 1, dtype: object\n",
      "===============================================================\n",
      "id_: 2\n",
      "content: \"4人同行 点了10个小吃\n",
      "榴莲酥 榴莲味道不足 松软 奶味浓\n",
      "虾饺 好吃 两颗大虾仁\n",
      "皮蛋粥 皮蛋多 但是一般 挺稠的\n",
      "奶黄包 很好吃 真的是蛋黄和奶 而且真的是流沙\n",
      "叉烧包 面香\n",
      "鲜虾烧卖 好吃 外面的黄色皮看着让人特别有食欲\n",
      "云吞面 云吞分量足 但是汤头不是很好喝 而且云吞的馅儿不知为何感觉不是很新鲜\n",
      "鲍汁腐皮卷 没怎么吃 味道倒是不错\n",
      "排骨 味道不错 不算很腻 但是油确实微多\n",
      "鲜虾锅贴 确实今天吃了很多虾 这个很酥脆，里头的虾也很好吃\n",
      "\n",
      "刚好有优惠券，所以4个人花了100不到【这个优惠券只能在1层用，5层用不了】\n",
      "\n",
      "原价大概人均50\n",
      "\n",
      "服务一般，上菜速度倒是很快，人挺多，坐在沙发上感觉很舒服\"\n",
      "label: location_traffic_convenience                -2\n",
      "location_distance_from_business_district    -2\n",
      "location_easy_to_find                       -2\n",
      "service_wait_time                           -2\n",
      "service_waiters_attitude                     0\n",
      "service_parking_convenience                 -2\n",
      "service_serving_speed                        1\n",
      "price_level                                  0\n",
      "price_cost_effective                        -2\n",
      "price_discount                               1\n",
      "environment_decoration                      -2\n",
      "environment_noise                           -2\n",
      "environment_space                            1\n",
      "environment_cleaness                        -2\n",
      "dish_portion                                 0\n",
      "dish_taste                                   1\n",
      "dish_look                                   -2\n",
      "dish_recommendation                         -2\n",
      "others_overall_experience                    0\n",
      "others_willing_to_consume_again             -2\n",
      "Name: 2, dtype: object\n",
      "===============================================================\n",
      "id_: 3\n",
      "content: \"之前评价了莫名其妙被删 果断继续差评！ 换了菜单 价格更低 开始砸牌子 但套餐还是有150的 点了两份套餐 上菜顺序是沙拉 汤 餐前面包 餐中饮料 另一份汤 甜点 牛排！ 火大 上个菜上成这样 你见过谁先吃冰激凌再吃牛排的啊！ 啊？！ 两份牛排要的七分 上来一份全熟 一份不及五分熟  你厨师哪里请来的啊大街上随便拉个人过来都烤的比你好吧？！ 套餐的价格又没变 凭什么跟以前差距这么大！ 自己砸自己牌子！ 火大 负分！ 起码五角场这家不会再去了！ 再见！\"\n",
      "label: location_traffic_convenience                -2\n",
      "location_distance_from_business_district    -2\n",
      "location_easy_to_find                       -2\n",
      "service_wait_time                           -2\n",
      "service_waiters_attitude                    -2\n",
      "service_parking_convenience                 -2\n",
      "service_serving_speed                       -2\n",
      "price_level                                  0\n",
      "price_cost_effective                        -2\n",
      "price_discount                              -2\n",
      "environment_decoration                      -2\n",
      "environment_noise                           -2\n",
      "environment_space                           -2\n",
      "environment_cleaness                        -2\n",
      "dish_portion                                -2\n",
      "dish_taste                                  -1\n",
      "dish_look                                   -2\n",
      "dish_recommendation                         -2\n",
      "others_overall_experience                   -1\n",
      "others_willing_to_consume_again             -1\n",
      "Name: 3, dtype: object\n",
      "===============================================================\n",
      "id_: 4\n",
      "content: \"出乎意料地惊艳，椰子鸡清热降火，美容养颜，大大满足了爱吃火锅怕上火星人。椰子冻是帅帅的老板原创，不加吉利丁片，而是在专业机器发酵，只加淡奶油和椰汁，还是每天限量；鸡肉很嫩，是不吃饲料的嫩鸡；蟹柳太好吃了，日本进口，当刺身直接蘸酱油吃比在锅里煮了还好吃，可见食材的新鲜，再也不会想吃普通火锅店冰镇的蟹肉棒了；蔬菜很新鲜，老板说是在大兴有蔬菜园；煲仔饭也是高于一般水平，老板说遗憾的是商场只能用电炉，不能用传统做法。\n",
      "最后，安利一下老板，原来是西餐厨师，看得出是对食物有要求的人，一万个赞！！！\"\n",
      "label: location_traffic_convenience                -2\n",
      "location_distance_from_business_district    -2\n",
      "location_easy_to_find                       -2\n",
      "service_wait_time                           -2\n",
      "service_waiters_attitude                    -2\n",
      "service_parking_convenience                 -2\n",
      "service_serving_speed                       -2\n",
      "price_level                                 -2\n",
      "price_cost_effective                        -2\n",
      "price_discount                              -2\n",
      "environment_decoration                      -2\n",
      "environment_noise                           -2\n",
      "environment_space                           -2\n",
      "environment_cleaness                        -2\n",
      "dish_portion                                -2\n",
      "dish_taste                                   1\n",
      "dish_look                                    1\n",
      "dish_recommendation                         -2\n",
      "others_overall_experience                    1\n",
      "others_willing_to_consume_again             -2\n",
      "Name: 4, dtype: object\n",
      "===============================================================\n",
      "id_: 5\n",
      "content: \"烤鸭使用的蘸酱很地道，觉得烤鸭好不好吃在很大程度上都取决于这个酱料是否正宗。个人感觉他们家的蘸酱还是很正宗的，看到大厅的LED显示器上面宣传的是使用的北京老字号六必居的正宗酱料，不知道是否属实，4元一份，也就是一小蝶，但一个人如果正常使用的话也差不多够了，但对于像我这种比较好这一口的来说就有点不够了，后面又加了一份。那个配菜中的面饼也有点不够，吃到后面面饼吃完了，但烤鸭还剩下很多，就直接蘸酱吃了，不过味道也很好吃。一只烤鸭的分量还是很多的，估计足够两三个人吃了。然后再陪一两个小炒时蔬，一顿饭也算是稳妥。服务很好，不过我们去的那一次好像店堂里面的顾客不多，很大的一个大厅里面也就三五桌客人有点不是很相称。\"\n",
      "label: location_traffic_convenience                -2\n",
      "location_distance_from_business_district    -2\n",
      "location_easy_to_find                       -2\n",
      "service_wait_time                           -2\n",
      "service_waiters_attitude                     1\n",
      "service_parking_convenience                 -2\n",
      "service_serving_speed                       -2\n",
      "price_level                                  0\n",
      "price_cost_effective                        -2\n",
      "price_discount                              -2\n",
      "environment_decoration                      -2\n",
      "environment_noise                           -2\n",
      "environment_space                            1\n",
      "environment_cleaness                        -2\n",
      "dish_portion                                 0\n",
      "dish_taste                                   1\n",
      "dish_look                                   -2\n",
      "dish_recommendation                         -2\n",
      "others_overall_experience                    1\n",
      "others_willing_to_consume_again             -2\n",
      "Name: 5, dtype: object\n",
      "===============================================================\n"
     ]
    }
   ],
   "source": [
    "# print some line to have a look\n",
    "for index, row in data.iterrows():\n",
    "    id_=row['id']\n",
    "    content=row['content']\n",
    "    label=row[2:]\n",
    "    print(\"id_:\",id_)\n",
    "    print(\"content:\",content)\n",
    "    print(\"label:\",label)\n",
    "    print(\"===============================================================\")\n",
    "    if index==5: \n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sentiment_analysis_labels(row):\n",
    "    # 1)location\n",
    "    location_traffic_convenience = row['location_traffic_convenience']               \n",
    "    location_distance_from_business_district= row['location_distance_from_business_district'] \n",
    "    location_easy_to_find  = row['location_easy_to_find']  \n",
    "    # 2)service\n",
    "    service_wait_time  = row['service_wait_time']                         \n",
    "    service_waiters_attitude = row['service_waiters_attitude']                     \n",
    "    service_parking_convenience = row['service_parking_convenience']                 \n",
    "    service_serving_speed  = row['service_serving_speed']  \n",
    "    # 3)price\n",
    "    price_level         = row['price_level']                        \n",
    "    price_cost_effective = row['price_cost_effective']                          \n",
    "    price_discount  = row['price_discount']     \n",
    "    # 4)environment\n",
    "    environment_decoration  = row['environment_decoration']                   \n",
    "    environment_noise   = row['environment_noise']                        \n",
    "    environment_space   = row['environment_space']                         \n",
    "    environment_cleaness     = row['environment_cleaness']  \n",
    "    # 5)dish\n",
    "    dish_portion   = row['dish_portion']                              \n",
    "    dish_taste =row['dish_taste']                                   \n",
    "    dish_look  = row['dish_look']                                  \n",
    "    dish_recommendation = row['dish_recommendation']   \n",
    "    # 6)other\n",
    "    others_overall_experience  = row['others_overall_experience']                    \n",
    "    others_willing_to_consume_again   = row['others_willing_to_consume_again']   \n",
    "    \n",
    "    label_list=[]\n",
    "    label_list=[location_traffic_convenience,location_distance_from_business_district,location_easy_to_find, # location\n",
    "                  service_wait_time,service_waiters_attitude,service_parking_convenience,service_serving_speed, # service\n",
    "               price_level,price_cost_effective,price_discount, # price\n",
    "               environment_decoration,environment_noise,environment_space,environment_cleaness, # environment\n",
    "               dish_portion,dish_taste,dish_look,dish_recommendation, # dish\n",
    "               others_overall_experience,others_willing_to_consume_again] # other\n",
    "    label_list=[str(i)+\"_\"+str(label_list[i]) for i  in range(len(label_list))]\n",
    "    return label_list "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label_list_one_hot: [1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "def convet_to_one_hot(label_list,num_classes=80):\n",
    "    new_label_list=[0 for i in range(num_classes)]\n",
    "    for label in label_list:\n",
    "        new_label_list[label]=1\n",
    "    return new_label_list\n",
    "\n",
    "label_list=[0, 4, 8, 12, 19, 20, 24, 28, 32, 39, 43, 47, 51, 55, 59, 62, 64, 68, 75, 76]\n",
    "label_list_one_hot=convet_to_one_hot(label_list)\n",
    "print(\"label_list_one_hot:\",label_list_one_hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id_: 0\n",
      "content: ['吼', '吼', '吼', '，', '萌', '死', '人', '的', '棒', '棒', '糖', '，', '中', '了', '大', '众', '点', '评', '的', '霸', '王', '餐', '，', '太', '可', '爱', '了', '。', '一', '直', '就', '好', '奇', '这', '个', '棒', '棒', '糖', '是', '怎', '么', '个', '东', '西', '，', '大', '众', '点', '评', '给', '了', '我', '这', '个', '土', '老', '冒', '一', '个', '见', '识', '的', '机', '会', '。', '看', '介', '绍', '棒', '棒', '糖', '是', '用', '德', '国', '糖', '做', '的', '，', '不', '会', '很', '甜', '，', '中', '间', '的', '照', '片', '是', '糯', '米', '的', '，', '能', '食', '用', '，', '真', '是', '太', '高', '端', '大', '气', '上', '档', '次', '了', '，', '还', '可', '以', '买', '蝴', '蝶', '结', '扎', '口', '，', '送', '人', '可', '以', '买', '礼', '盒', '。', '我', '是', '先', '打', '的', '卖', '家', '电', '话', '，', '加', '了', '微', '信', '，', '给', '卖', '家', '传', '的', '照', '片', '。', '等', '了', '几', '天', '，', '卖', '家', '就', '告', '诉', '我', '可', '以', '取', '货', '了', '，', '去', '大', '官', '屯', '那', '取', '的', '。', '虽', '然', '连', '卖', '家', '的', '面', '都', '没', '见', '到', '，', '但', '是', '还', '是', '谢', '谢', '卖', '家', '送', '我', '这', '么', '可', '爱', '的', '东', '西', '，', '太', '喜', '欢', '了', '，', '这', '哪', '舍', '得', '吃', '啊', '。']\n",
      "sentiment_label_list: ['0_-2', '1_-2', '2_-2', '3_-2', '4_1', '5_-2', '6_-2', '7_-2', '8_-2', '9_1', '10_-2', '11_-2', '12_-2', '13_-2', '14_-2', '15_-2', '16_1', '17_-2', '18_1', '19_-2'] ;length: 20\n"
     ]
    }
   ],
   "source": [
    " # print first row\n",
    " for index, row in data.iterrows():\n",
    "    id_=row['id']\n",
    "    #content=[x for x in jieba.lcut(row['content']) if x.strip() and x!=\"\\\"\"]\n",
    "    content_raw=row['content']\n",
    "    content=[content_raw[i] for i in range(len(content_raw)) if content_raw[i].strip() and content_raw[i]!=\"\\\"\" ]\n",
    "    sentiment_label_list=get_sentiment_analysis_labels(row)\n",
    "    print(\"id_:\",id_)\n",
    "    print(\"content:\",content)\n",
    "    print(\"sentiment_label_list:\",sentiment_label_list,\";length:\",len(sentiment_label_list))\n",
    "    if index==0: \n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_traininig_small.shape: (105000, 22)\n",
      "data_validation_small.shape: (15000, 22)\n",
      "data_test_small.shape: (15000, 22)\n",
      "data_test2_small.shape: (200000, 22)\n"
     ]
    }
   ],
   "source": [
    "data_traininig_small=data.sample(frac=1.0)\n",
    "data_validation_small=data_validation.sample(frac=1.0)\n",
    "data_test_small=data_test_old #[0:1000]\n",
    "data_test2_small=data_test #[0:1000]\n",
    "\n",
    "print(\"data_traininig_small.shape:\",data_traininig_small.shape)\n",
    "print(\"data_validation_small.shape:\",data_validation_small.shape)\n",
    "print(\"data_test_small.shape:\",data_test_small.shape)\n",
    "print(\"data_test2_small.shape:\",data_test2_small.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_big==>: (335000, 22)\n"
     ]
    }
   ],
   "source": [
    "data_big=pd.concat([data_traininig_small,data_validation_small,data_test_small,data_test2_small])#(data_traininig_small, data_test_small, on='key')\n",
    "print(\"data_big==>:\",data_big.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_example_for_length: 33500\n",
      "avg_length: 336.1846865671642\n",
      "input_length_dict: {50: 3, 100: 5, 150: 18, 200: 952, 250: 11691, 300: 6506, 350: 4422, 400: 2670, 500: 3060, 600: 1854, 700: 867, 1000: 1452}\n",
      "input_length_dict_percentage: {50: 8.955223880597016e-05, 100: 0.00014925373134328358, 150: 0.0005373134328358209, 200: 0.028417910447761194, 250: 0.3489850746268657, 300: 0.1942089552238806, 350: 0.132, 400: 0.07970149253731343, 500: 0.09134328358208955, 600: 0.05534328358208955, 700: 0.02588059701492537, 1000: 0.04334328358208955}\n",
      "max_sequence_length: 600\n"
     ]
    }
   ],
   "source": [
    "# compute length distribution of inputs and set max_sequence_length; \n",
    "# choose a mininum value of max_sequence_length that let 90% of inputs' s length is less of it. actually it cover 93%.\n",
    "total_length=0\n",
    "input_length_list=[50,100,150,200,250,300,350,400,500,600,700,1000]\n",
    "input_length_dict={x:0 for x in input_length_list }\n",
    "data_big_part=data_big.sample(frac=0.1)\n",
    "\n",
    "num_example_for_length,_=data_big_part.shape\n",
    "print(\"num_example_for_length:\",num_example_for_length)\n",
    "for index, row in data_big_part.iterrows(): # data_traininig_small\n",
    "    id_=row['id']\n",
    "    #input_list=[x for x in jieba.lcut(row['content']) if x.strip() and x!=\"\\\"\"]\n",
    "    content=row['content']\n",
    "    content=[content[i] for i in range(len(content)) if content[i].strip() and content[i]!=\"\\\"\" ]\n",
    "    length_input=len(content)\n",
    "    \n",
    "    fixed_len=1000\n",
    "    for length in input_length_list:\n",
    "        if length_input<length:\n",
    "            fixed_len=length\n",
    "            break\n",
    "    input_length_dict[fixed_len]=input_length_dict[fixed_len]+1\n",
    "    total_length+=length_input\n",
    "    #c_inputs.update(input_list)\n",
    "    \n",
    "avg_length=float(total_length) /float(num_example_for_length)\n",
    "print(\"avg_length:\",avg_length)\n",
    "print(\"input_length_dict:\",input_length_dict)\n",
    "\n",
    "input_length_dict_percentage={}\n",
    "for k,v in input_length_dict.items():\n",
    "    v=v/num_example_for_length\n",
    "    input_length_dict_percentage[k]=v\n",
    "print(\"input_length_dict_percentage:\",input_length_dict_percentage)\n",
    "\n",
    "# conclusion: most length is between 150-250. average length is 216. if we use max length 350, then we can cover about:  91% of inputs.\n",
    "# choose a mininum value of max_sequence_length that let 90% of inputs' s length is less of it.\n",
    "max_sequence_length=0\n",
    "accumulate_percentage=0\n",
    "for fixed_length,percentage in input_length_dict_percentage.items():\n",
    "    accumulate_percentage+=percentage\n",
    "    if accumulate_percentage>0.9:\n",
    "        max_sequence_length=fixed_length\n",
    "        print(\"max_sequence_length:\",max_sequence_length)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label2index: {'0_-2': 0, '0_-1': 1, '0_0': 2, '0_1': 3, '1_-2': 4, '1_-1': 5, '1_0': 6, '1_1': 7, '2_-2': 8, '2_-1': 9, '2_0': 10, '2_1': 11, '3_-2': 12, '3_-1': 13, '3_0': 14, '3_1': 15, '4_-2': 16, '4_-1': 17, '4_0': 18, '4_1': 19, '5_-2': 20, '5_-1': 21, '5_0': 22, '5_1': 23, '6_-2': 24, '6_-1': 25, '6_0': 26, '6_1': 27, '7_-2': 28, '7_-1': 29, '7_0': 30, '7_1': 31, '8_-2': 32, '8_-1': 33, '8_0': 34, '8_1': 35, '9_-2': 36, '9_-1': 37, '9_0': 38, '9_1': 39, '10_-2': 40, '10_-1': 41, '10_0': 42, '10_1': 43, '11_-2': 44, '11_-1': 45, '11_0': 46, '11_1': 47, '12_-2': 48, '12_-1': 49, '12_0': 50, '12_1': 51, '13_-2': 52, '13_-1': 53, '13_0': 54, '13_1': 55, '14_-2': 56, '14_-1': 57, '14_0': 58, '14_1': 59, '15_-2': 60, '15_-1': 61, '15_0': 62, '15_1': 63, '16_-2': 64, '16_-1': 65, '16_0': 66, '16_1': 67, '17_-2': 68, '17_-1': 69, '17_0': 70, '17_1': 71, '18_-2': 72, '18_-1': 73, '18_0': 74, '18_1': 75, '19_-2': 76, '19_-1': 77, '19_0': 78, '19_1': 79}\n"
     ]
    }
   ],
   "source": [
    " # create dict of label to index\n",
    "value_list=[-2,-1,0,1]\n",
    "num_aspect=20\n",
    "label2index={}\n",
    "count_label=0\n",
    "for i in range(num_aspect):\n",
    "    for value in value_list:\n",
    "        label2index[str(i)+\"_\"+str(value)]=count_label\n",
    "        count_label+=1\n",
    "print(\"label2index:\",label2index)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate training data by following data format of bert\n",
    "\n",
    "def process_data(data,target_file):\n",
    "    target_object=open(target_file,'w')\n",
    "    print(data.shape)\n",
    "    for index, row in data.iterrows():\n",
    "        sentiment_label_list=get_sentiment_analysis_labels(row) \n",
    "        content=row['content'].replace(\"\\n\",\"。\").replace(\"\\r\",\"。\")\n",
    "        content=[content[i] for i in range(len(content)) if content[i].strip() and content[i]!=\"\\\"\" ]\n",
    "        content=\"\".join(content)\n",
    "        content=re.sub('''\"''','',content)\n",
    "        strings=\",\".join(sentiment_label_list)+\"\\t\"+content+\"\\n\"\n",
    "        target_object.write(strings)\n",
    "        if index%50000==0:print(strings)\n",
    "    target_object.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15000, 22)\n",
      "0_-2,1_-2,2_-2,3_0,4_-2,5_-2,6_-2,7_1,8_-2,9_-2,10_-2,11_-2,12_-2,13_-2,14_-2,15_0,16_-2,17_-2,18_1,19_0\t哎，想当年来佘山的时候，啥都没有，三品香算镇上最大看起来最像样的饭店了。菜品多，有点太多，感觉啥都有，杂都不足以形容。随便点些，居然口味什么的都好还可以，价钱自然是便宜当震惊。元宝虾和椒盐九肚鱼都不错吃。不过近来几次么，味道明显没以前好了。冷餐里面一个凉拌海带丝还可以，酸酸甜甜的。镇上也有了些别的大点的饭店，所以不是每次必来了。对了，这家的生意一如既往的超级好，不定位基本吃不到。不过佘山这边的人吃晚饭很早的，所以稍微晚点去就很空了。\n",
      "\n",
      "process_data.valid.ended. 15000\n"
     ]
    }
   ],
   "source": [
    "target_file_valid='./data/dev.tsv'\n",
    "process_data(data_validation_small,target_file_valid)\n",
    "print(\"process_data.valid.ended.\",len(data_validation_small))\n",
    "#X_test,_=process_data(data_test_small,data_type='test')\n",
    "#print(\"process_data.ended...\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(105000, 22)\n",
      "0_1,1_1,2_-2,3_-2,4_1,5_1,6_-2,7_-2,8_-2,9_1,10_1,11_1,12_1,13_1,14_1,15_1,16_-2,17_-2,18_1,19_1\t非常喜欢去新开的有特色的店品尝，有幸被大众点评网同城聚会抽到黄金替补免费品鉴的机会，我感到十分幸运！红不让台湾特色餐厅位于李沧万达金街1号入口扶梯旁，交通便利，停车方便。店面小资，适合举行小型派对。走近店门，服务员立刻开门迎接并且热情的打招呼。进入店面，环顾四周，鹅黄的灯光下透露出温馨慵懒的气氛，店家匠心独居，屋内的摆设典雅质朴。我们大家围坐在一起开始品尝台湾特色美食，前菜是鸡排沙拉，鸡排外焦里嫩，沙拉酸爽开胃陪的饮品是本店招牌—寒天奶栋，口感丝化。在我们的玩闹嘻戏中，硬菜短上来了——招牌全垒打拼盘，样子可爱，味道独特，被大家迅雷不及掩耳盗铃之势一扫而光。紧接着——等一个人小火锅，西班牙海鲜焗饭，黑椒牛柳焗饭海鲜焗烤嫩鸡排上桌了，食材琳琅满目，令人目不暇接。人多力量大，盘子渐渐见底了，新菜品——普罗旺斯田园披萨，台湾卤肉饭，秘制鸡腿饭，招牌巨蛋烧，阳光手打面，超级霸大热狗震撼来袭。。这里特别提一下普罗旺斯田园披萨，顶上配料五颜六色，芝士浓郁，饼胚酥脆，搭配沙拉和洋葱圈口味绝佳。还有在卤肉饭，鸡腿饭，手打面扮演重要角色的酱料，里面居然搭配了小鱼干，补钙又美味。。环境搭配合理，既有闺密情侣说悄悄话的私密空间，又有三五好友畅饮阔聊的长桌吧台，气氛轻松惬意，互不打扰。。红不让台湾特色餐厅绝对是小资的你餐饮休闲，畅聊发呆的绝佳选择，千万不要错过！\n",
      "\n",
      "0_-2,1_-2,2_-2,3_-2,4_-2,5_-2,6_-2,7_-2,8_-2,9_-2,10_1,11_-2,12_-2,13_-2,14_-2,15_0,16_0,17_-2,18_1,19_-2\t餐前，装冷菜的碟子是仿青瓷的碟子，高脚，可惜是塑料的，近看缺了质感，没有瓷器那种通透，远看挺不错。盛菜的器皿有些也挺有特色的，包厢布置也花了功夫，走古风挂字画。墙纸是青花瓷的，对面墙壁也贴了很多瓷器，有几个还是玲珑瓷，透光看比较好看。比较喜欢他们家的日式茶壶，铁壶，很沉但漂亮，据说冲泡能补铁？菜品味道一般，花样是挺多的，杨枝甘露换个器皿就能换来很大的拍照量，是不是桌桌都有？喜欢那个有小把的酱料器皿，拿着很可爱\n",
      "\n",
      "0_-2,1_-2,2_-2,3_-2,4_1,5_-2,6_-2,7_-2,8_-2,9_1,10_-2,11_-2,12_-2,13_-2,14_-2,15_-2,16_1,17_-2,18_1,19_-2\t吼吼吼，萌死人的棒棒糖，中了大众点评的霸王餐，太可爱了。一直就好奇这个棒棒糖是怎么个东西，大众点评给了我这个土老冒一个见识的机会。看介绍棒棒糖是用德国糖做的，不会很甜，中间的照片是糯米的，能食用，真是太高端大气上档次了，还可以买蝴蝶结扎口，送人可以买礼盒。我是先打的卖家电话，加了微信，给卖家传的照片。等了几天，卖家就告诉我可以取货了，去大官屯那取的。虽然连卖家的面都没见到，但是还是谢谢卖家送我这么可爱的东西，太喜欢了，这哪舍得吃啊。\n",
      "\n",
      "process_data.train.ended. 105000\n"
     ]
    }
   ],
   "source": [
    "target_file_train='./data/train.tsv'\n",
    "process_data(data_traininig_small,target_file_train)\n",
    "print(\"process_data.train.ended.\",len(data_traininig_small))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is end of genrate training data and validation data for bert\n",
    "# below is generate pre-train for bert, you can ignore if you don't want to pre-train bert."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_big: (335000, 22)\n"
     ]
    }
   ],
   "source": [
    "print(\"data_big:\",data_big.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "started...\n",
      "data_big.shape: (335000, 22)\n",
      "end...\n"
     ]
    }
   ],
   "source": [
    " # generate pre-train raw data using data_big\n",
    "def write_pre_train_doc(data_big,target_path,num_files=30):\n",
    "    print(\"data_big.shape:\",data_big.shape)\n",
    "    data_big=data_big.sample(frac=1.0)\n",
    "    count=0\n",
    "    data_big\n",
    "    target_object_list=[]\n",
    "    for i in range(num_files):\n",
    "        target_object=open(target_path+\"bert_\"+str(i)+\"_pretrain.txt\",\"w\")\n",
    "        target_object_list.append(target_object)\n",
    "        \n",
    "    for index, row in data_big.iterrows():  \n",
    "        content=row['content']\n",
    "        content=row['content'].replace(\"\\n\",\"。\").replace(\"\\r\",\"。\")\n",
    "        content=[content[i] for i in range(len(content)) if content[i].strip() and content[i]!=\"\\\"\" ]\n",
    "        sentence_list=\" \".join(content).split(\"。\")\n",
    "        reminder=count%num_files\n",
    "        for sentence in sentence_list:\n",
    "            sentence=sentence.strip()\n",
    "            target_object_list[reminder].write(sentence+\"\\n\") # each line contains one sentence\n",
    "        target_object_list[reminder].write(sentence+\"\\n\") # between each document has a blank line\n",
    "        count=count+1\n",
    "    for i in range(num_files):\n",
    "        target_object_list[i].close()\n",
    "        \n",
    "print(\"started...\")        \n",
    "target_path='./PRE_TRAIN_DIR/'\n",
    "write_pre_train_doc(data_big,target_path)\n",
    "print(\"end...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}