sms_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": "true"
   },
   "source": [
    "# Table of Contents\n",
    " <p><div class=\"lev1 toc-item\"><a href=\"#Imports\" data-toc-modified-id=\"Imports-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Imports</a></div><div class=\"lev1 toc-item\"><a href=\"#Load-the-data-from-disk-and-set-up-the-dataframes\" data-toc-modified-id=\"Load-the-data-from-disk-and-set-up-the-dataframes-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Load the data from disk and set up the dataframes</a></div><div class=\"lev1 toc-item\"><a href=\"#Show-a-heatmap-of-how-many-texts-you've-exchanged\" data-toc-modified-id=\"Show-a-heatmap-of-how-many-texts-you've-exchanged-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Show a heatmap of how many texts you've exchanged</a></div><div class=\"lev1 toc-item\"><a href=\"#Table-and-graph-of-who-you-text-the-most\" data-toc-modified-id=\"Table-and-graph-of-who-you-text-the-most-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Table and graph of who you text the most</a></div><div class=\"lev1 toc-item\"><a href=\"#Steamgraph\" data-toc-modified-id=\"Steamgraph-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>Steamgraph</a></div><div class=\"lev3 toc-item\"><a href=\"#Dump-the-necessary-data-to-JS\" data-toc-modified-id=\"Dump-the-necessary-data-to-JS-501\"><span class=\"toc-item-num\">5.0.1&nbsp;&nbsp;</span>Dump the necessary data to JS</a></div><div class=\"lev3 toc-item\"><a href=\"#Draw-the-graph!\" data-toc-modified-id=\"Draw-the-graph!-502\"><span class=\"toc-item-num\">5.0.2&nbsp;&nbsp;</span>Draw the graph!</a></div><div class=\"lev1 toc-item\"><a href=\"#Wordcloud\" data-toc-modified-id=\"Wordcloud-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>Wordcloud</a></div><div class=\"lev3 toc-item\"><a href=\"#Define-the-helper-method\" data-toc-modified-id=\"Define-the-helper-method-601\"><span class=\"toc-item-num\">6.0.1&nbsp;&nbsp;</span>Define the helper method</a></div><div class=\"lev3 toc-item\"><a href=\"#Texts-you've-sent\" data-toc-modified-id=\"Texts-you've-sent-602\"><span class=\"toc-item-num\">6.0.2&nbsp;&nbsp;</span>Texts you've sent</a></div><div class=\"lev3 toc-item\"><a href=\"#Texts-to/from-a-specific-contact\" data-toc-modified-id=\"Texts-to/from-a-specific-contact-603\"><span class=\"toc-item-num\">6.0.3&nbsp;&nbsp;</span>Texts to/from a specific contact</a></div><div class=\"lev1 toc-item\"><a href=\"#Diving-deeper-into-the-actual-text\" data-toc-modified-id=\"Diving-deeper-into-the-actual-text-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>Diving deeper into the actual text</a></div><div class=\"lev3 toc-item\"><a href=\"#Visualize-a-word-tree-of-texts-exchanged-with-a-specific-contact\" data-toc-modified-id=\"Visualize-a-word-tree-of-texts-exchanged-with-a-specific-contact-701\"><span class=\"toc-item-num\">7.0.1&nbsp;&nbsp;</span>Visualize a word tree of texts exchanged with a specific contact</a></div><div class=\"lev3 toc-item\"><a href=\"#Preprocessing-and-data-munging-for-TFIDF\" data-toc-modified-id=\"Preprocessing-and-data-munging-for-TFIDF-702\"><span class=\"toc-item-num\">7.0.2&nbsp;&nbsp;</span>Preprocessing and data munging for TFIDF</a></div><div class=\"lev3 toc-item\"><a href=\"#Create-TFIDF-matrix-for-all-contacts\" data-toc-modified-id=\"Create-TFIDF-matrix-for-all-contacts-703\"><span class=\"toc-item-num\">7.0.3&nbsp;&nbsp;</span>Create TFIDF matrix for all contacts</a></div><div class=\"lev3 toc-item\"><a href=\"#Helper-methods-to-leverage-the-TFIDF-matrix\" data-toc-modified-id=\"Helper-methods-to-leverage-the-TFIDF-matrix-704\"><span class=\"toc-item-num\">7.0.4&nbsp;&nbsp;</span>Helper methods to leverage the TFIDF matrix</a></div><div class=\"lev3 toc-item\"><a href=\"#Words-that-identify-a-specific-contact\" data-toc-modified-id=\"Words-that-identify-a-specific-contact-705\"><span class=\"toc-item-num\">7.0.5&nbsp;&nbsp;</span>Words that identify a specific contact</a></div><div class=\"lev3 toc-item\"><a href=\"#Words-that-identify-the-difference-between-two-contacts\" data-toc-modified-id=\"Words-that-identify-the-difference-between-two-contacts-706\"><span class=\"toc-item-num\">7.0.6&nbsp;&nbsp;</span>Words that identify the difference between two contacts</a></div><div class=\"lev1 toc-item\"><a href=\"#Looking-at-language-progression-over-the-years\" data-toc-modified-id=\"Looking-at-language-progression-over-the-years-8\"><span class=\"toc-item-num\">8&nbsp;&nbsp;</span>Looking at language progression over the years</a></div><div class=\"lev3 toc-item\"><a href=\"#Helper-methods-for-looking-at-TFIDF-by-year\" data-toc-modified-id=\"Helper-methods-for-looking-at-TFIDF-by-year-801\"><span class=\"toc-item-num\">8.0.1&nbsp;&nbsp;</span>Helper methods for looking at TFIDF by year</a></div><div class=\"lev3 toc-item\"><a href=\"#My-top-words-over-the-years\" data-toc-modified-id=\"My-top-words-over-the-years-802\"><span class=\"toc-item-num\">8.0.2&nbsp;&nbsp;</span>My top words over the years</a></div><div class=\"lev3 toc-item\"><a href=\"#Top-words-over-the-years-from/to-a-specific-contact\" data-toc-modified-id=\"Top-words-over-the-years-from/to-a-specific-contact-803\"><span class=\"toc-item-num\">8.0.3&nbsp;&nbsp;</span>Top words over the years from/to a specific contact</a></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See the README for an explanation of how this code runs and functions.\n",
    "\n",
    "Contact michaeldezube at gmail dot com with questions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "\n",
    "import copy\n",
    "import json\n",
    "import re\n",
    "import string\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn  # To improve the chart styling.\n",
    "import wordtree\n",
    "\n",
    "from IPython.display import display\n",
    "from IPython.display import HTML\n",
    "from IPython.display import Javascript\n",
    "from wordcloud import STOPWORDS\n",
    "import ipywidgets as widgets\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "import iphone_connector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data from disk and set up the dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "matplotlib.style.use('ggplot')\n",
    "pd.set_option('display.max_colwidth', 1000)\n",
    "iphone_connector.initialize()\n",
    "\n",
    "fully_merged_messages_df, address_book_df = iphone_connector.get_cleaned_fully_merged_messages()\n",
    "full_names = set(address_book_df.full_name)  # Handy set to check for misspellings later on.\n",
    "fully_merged_messages_df.full_name.replace('nan nan nan', 'Unknown', inplace=True)\n",
    "\n",
    "WORDS_PER_PAGE = 450  # Based upon http://wordstopages.com/\n",
    "print('\\nTotal pages if all texts were printed: {0:,d} (Arial size 12, single spaced)\\n'.format(\n",
    "    sum(fully_merged_messages_df.text.apply(lambda x: len(x.split())))//WORDS_PER_PAGE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fully_merged_messages_df = fully_merged_messages_df.reset_index(drop=True)\n",
    "fully_merged_messages_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "address_book_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Use `fully_merged_messages_df` and `address_book_df` for analysis, they contain all messages with columns for the sender and all contacts, respectively**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Show a heatmap of how many texts you've exchanged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def plot_year_month_heatmap(df, trim_incomplete=True, search_term=None, figsize=(18, 10)):\n",
    "    \"\"\"Plots a heatmap of the dataframe grouped by year and month.\n",
    "    \n",
    "    Args:\n",
    "        df: The dataframe, must contain a column named `date`.\n",
    "        trim_incomplete: If true, don't plot rows that lack 12 full months of data.  Default True.\n",
    "        search_term: A case insensitive term to require in all rows of the dataframe's `text`\n",
    "            column.  Default None.\n",
    "        figsize: The size of the plot as a tuple.  Default (18, 10);\n",
    "    \n",
    "    \"\"\"\n",
    "    if search_term:\n",
    "        df = df[df['text'].str.contains(search_term, case=False)]\n",
    "    month_year_messages = pd.DataFrame(df['date'])\n",
    "    month_year_messages['year'] = month_year_messages.apply(lambda row: row.date.year, axis=1)\n",
    "    month_year_messages['month'] = month_year_messages.apply(lambda row: row.date.month, axis=1)\n",
    "    month_year_messages = month_year_messages.drop('date', axis=1)\n",
    "\n",
    "    month_year_messages_pivot = month_year_messages.pivot_table(index='year',\n",
    "                                                                columns='month',\n",
    "                                                                aggfunc=len, dropna=True)\n",
    "    if trim_incomplete:\n",
    "        month_year_messages_pivot = month_year_messages_pivot[month_year_messages_pivot.count(axis=1) == 12]\n",
    "    if month_year_messages_pivot.shape[0] == 0:\n",
    "        print('After trimming rows that didn\\'t have 12 months, no rows remained, bailing out.')\n",
    "        return\n",
    "\n",
    "    f, ax = plt.subplots(figsize=figsize)\n",
    "    seaborn.heatmap(month_year_messages_pivot, annot=True, fmt=\".0f\", square=True, cmap=\"YlGnBu\", ax=ax)\n",
    "\n",
    "# Plot all text messages exchanges over the years.\n",
    "plot_year_month_heatmap(fully_merged_messages_df, search_term='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Table and graph of who you text the most"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Helper method to better support py2 and py3.\n",
    "\n",
    "def convert_unicode_to_str_if_needed(unicode_or_str):\n",
    "    if type(unicode_or_str).__name__ == 'unicode':\n",
    "        return unicode_or_str.encode('utf-8')\n",
    "    return unicode_or_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Note \"Unknown\" means the number was not found in your address book.\n",
    "\n",
    "def get_message_counts(dataframe):\n",
    "    return pd.Series({'Texts sent': dataframe[dataframe.is_from_me == 1].shape[0],\n",
    "                      'Texts received': dataframe[dataframe.is_from_me == 0].shape[0],\n",
    "                      'Texts exchanged': dataframe.shape[0]})\n",
    "messages_grouped = fully_merged_messages_df.groupby('full_name').apply(get_message_counts)\n",
    "messages_grouped = messages_grouped.sort_values(by='Texts exchanged', ascending=False)\n",
    "\n",
    "widgets.interact(messages_grouped.head,\n",
    "                 n=widgets.IntSlider(min=5, max=50, step=1, value=5, continuous_update=False,\n",
    "                                     description='Number of people to show:'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Helper method so we can wrap it with interact().\n",
    "def _plot_most_common_text(top_n=10):\n",
    "    messages_grouped.head(top_n).plot(figsize=(20,10), kind='bar')\n",
    "   \n",
    "widgets.interact(_plot_most_common_text,\n",
    "                 top_n=widgets.IntSlider(min=5, max=100, step=1, value=5, continuous_update=False,\n",
    "                                         description='Number of people to show:'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Steamgraph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dump the necessary data to JS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Restrict to the top N people you text the most so the steamgraph is legible.\n",
    "TOP_N = 10  # Freely change this value.\n",
    "\n",
    "sliced_df = fully_merged_messages_df[fully_merged_messages_df.full_name.isin(messages_grouped.head(TOP_N).index)]\n",
    "grouped_by_month = sliced_df.groupby([\n",
    "    sliced_df.apply(lambda x: x.date.strftime('%Y/%m'), axis=1),\n",
    "    'full_name']\n",
    ")['text'].count().to_frame()\n",
    "\n",
    "grouped_by_month = grouped_by_month.sort_index()\n",
    "# We create a dense dataframe for every year/month combination so even if a person didn't text in a specific\n",
    "# year/month, we have a 0 so the steamgraph can propertly graph the value.\n",
    "grouped_by_month_dense = grouped_by_month.unstack().fillna(0).stack()\n",
    "\n",
    "# Dump the dataframe to a global JS variable so we can access it in our JS code.\n",
    "# TODO(mdezube): Dump out as JSON instead.\n",
    "formatted_for_steamgraph = grouped_by_month_dense.reset_index(level=1)\n",
    "formatted_for_steamgraph.index.name = 'date'\n",
    "formatted_for_steamgraph.columns = ['key', 'value']\n",
    "Javascript(\"window.csvAsString='{}'\".format(formatted_for_steamgraph.to_csv(index_label='date').replace('\\n', '\\\\n')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Draw the graph!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%javascript\n",
    "// Draw the streamgraph using d3.\n",
    "\n",
    "element.append('<div class=\"chart\" style=\"height:600px; width:100%\"></div>')\n",
    "element.append('<style>.axis path, .axis line' + \n",
    "               '{fill: none; stroke: #000;stroke-width: 2px; shape-rendering: crispEdges;}' + \n",
    "               '</style>')\n",
    "\n",
    "element.append(\"<script src='d3.min.js'></script>\")\n",
    "element.append(\"<script src='colorbrewer.min.js'></script>\")\n",
    "element.append(\"<script src='steamgraph.js'></script>\")\n",
    "\n",
    "// Choose your favorite from https://bl.ocks.org/mbostock/5577023\n",
    "var colorBrewerPalette = \"Spectral\";\n",
    "\n",
    "// Set a timeout to let the JS scripts actually load into memory, this is a bit of a hack but works reliably.\n",
    "setTimeout(function(){createSteamgraph(csvAsString, colorBrewerPalette)}, 200);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wordcloud"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the helper method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_cloud(texts, max_words=30):\n",
    "    # Add more words here if you want to ignore them:\n",
    "    my_stopwords = STOPWORDS.copy()\n",
    "    my_stopwords.update(['go', 'ya', 'come', 'back', 'good', 'sound'])\n",
    "    words = ' '.join(texts).lower()\n",
    "    wordcloud = WordCloud(font_path='CabinSketch-Bold.ttf',\n",
    "                          stopwords=my_stopwords,\n",
    "                          background_color='black',\n",
    "                          width=800,\n",
    "                          height=600,\n",
    "                          relative_scaling=1,\n",
    "                          max_words=max_words\n",
    "                         ).generate_from_text(words)\n",
    "    print('Based on {0:,} texts'.format(len(texts)))\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(15,10))\n",
    "    ax.imshow(wordcloud)\n",
    "    ax.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Texts you've sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Word cloud of the top 25 words I use based on the most recent 30,000 messages.\n",
    "\n",
    "texts_from_me = fully_merged_messages_df[fully_merged_messages_df.is_from_me == 1].text[-30000:]\n",
    "widgets.interact(\n",
    "    generate_cloud,\n",
    "    texts=widgets.fixed(texts_from_me),\n",
    "    max_words=widgets.IntSlider(min=5,max=50,step=1,value=10, continuous_update=False,\n",
    "                                description='Max words to show:'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Texts to/from a specific contact"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def _word_cloud_specific_contact(max_words, from_me, contact):\n",
    "    contact = convert_unicode_to_str_if_needed(contact)\n",
    "    if contact not in full_names:\n",
    "        print('{} not found'.format(contact))\n",
    "        return\n",
    "    sliced_df = fully_merged_messages_df[(fully_merged_messages_df.full_name == contact) &\n",
    "                                         (fully_merged_messages_df.is_from_me == from_me)].text\n",
    "    generate_cloud(sliced_df, max_words)\n",
    "\n",
    "widgets.interact(\n",
    "    _word_cloud_specific_contact,\n",
    "    max_words=widgets.IntSlider(min=5, max=50, step=1, value=10,\n",
    "                                continuous_update=False, description='Max words to show:'),\n",
    "    from_me=widgets.RadioButtons(\n",
    "        options={'Show messages FROM me': True, 'Show messages TO me': False}, description=' '),\n",
    "    contact=widgets.Text(value='Mom', description='Contact name:')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Diving deeper into the actual text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize a word tree of texts exchanged with a specific contact"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note this requires an internet connection to load Google's JS library.\n",
    "def get_json_for_word_tree(contact):\n",
    "    df = fully_merged_messages_df[(fully_merged_messages_df.full_name == contact)]\n",
    "    print('Exchanged {0:,} texts with {1}'.format(df.shape[0], contact))\n",
    "    \n",
    "    array_for_json = [[text[1]] for text in df.text.iteritems()]\n",
    "    array_for_json.insert(0, [['Phrases']])\n",
    "    return json.dumps(array_for_json)\n",
    "    \n",
    "CONTACT_NAME = 'Mom'\n",
    "ROOT_WORD = 'feel'\n",
    "HTML(wordtree.get_word_tree_html(get_json_for_word_tree('Mom'),\n",
    "                                 ROOT_WORD.lower(),\n",
    "                                 lowercase=True,\n",
    "                                 tree_type='double'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing and data munging for TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation = copy.copy(string.punctuation)\n",
    "punctuation += u'“”‘’\\ufffc\\uff0c'  # Include some UTF-8 punctuation that occurred.\n",
    "punct_regex = re.compile(u'[{0}]'.format(punctuation))\n",
    "spaces_regex = re.compile(r'\\s{2,}')\n",
    "numbers_regex = re.compile(r'\\d+')\n",
    "\n",
    "def clean_text(input_str):\n",
    "    processed = input_str.lower()\n",
    "    processed = punct_regex.sub('', processed)\n",
    "    # Also try: processed = numbers_regex.sub('_NUMBER_', processed)\n",
    "    processed = numbers_regex.sub('', processed)\n",
    "    processed = spaces_regex.sub(' ', processed)\n",
    "    \n",
    "    return processed\n",
    "\n",
    "# The normal stopwords list contains words like \"i'll\" which is unprocessed.\n",
    "processed_stopwords = [clean_text(word) for word in STOPWORDS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Group the texts by person and collapse them into a single string per person.\n",
    "\n",
    "grouped_by_name = fully_merged_messages_df[fully_merged_messages_df.is_from_me == 0].groupby(\n",
    "    'full_name')['text'].apply(lambda x: ' '.join(x)).to_frame()\n",
    "grouped_by_name.info(memory_usage='deep')\n",
    "grouped_by_name.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create TFIDF matrix for all contacts\n",
    "\n",
    "Note the methods below focus on texts received from these contacts, not texts you've sent to them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk import tokenize\n",
    "import numpy as np\n",
    "\n",
    "vectorizer = TfidfVectorizer(preprocessor=clean_text,\n",
    "                             tokenizer=tokenize.WordPunctTokenizer().tokenize,\n",
    "                             stop_words=processed_stopwords,\n",
    "                             ngram_range=(1, 2), max_df=.9, max_features=50000)\n",
    "tfidf_transformed_dataset = vectorizer.fit_transform(grouped_by_name.text)\n",
    "word_list = pd.Series(vectorizer.get_feature_names())\n",
    "\n",
    "print('TFIDF sparse matrix is {0}MB'.format(tfidf_transformed_dataset.data.nbytes / 1024 / 1024))\n",
    "print('TFIDF matrix has shape: {0}'.format(tfidf_transformed_dataset.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper methods to leverage the TFIDF matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_word_summary_for_contact(contact, top_n=25):\n",
    "    contact = convert_unicode_to_str_if_needed(contact)\n",
    "    tfidf_record = _get_tfidf_record_for_contact(contact)\n",
    "    if tfidf_record is None:\n",
    "        print('\"{0}\" was not found.'.format(contact))\n",
    "        return\n",
    "    sorted_indices = tfidf_record.argsort()[::-1]\n",
    "    return pd.DataFrame({'Word': word_list.iloc[sorted_indices[:top_n]]}).reset_index(drop=True)\n",
    "\n",
    "def get_word_summary_for_diffs(contact, other_contact, top_n=25):\n",
    "    contact = convert_unicode_to_str_if_needed(contact)\n",
    "    other_contact = convert_unicode_to_str_if_needed(other_contact)\n",
    "    \n",
    "    tfidf_record_contact = _get_tfidf_record_for_contact(contact)\n",
    "    tfidf_record_other_contact = _get_tfidf_record_for_contact(other_contact)\n",
    "    \n",
    "    if tfidf_record_contact is None or tfidf_record_other_contact is None:\n",
    "        # Print out the first contact not found.\n",
    "        contact_not_found = contact if tfidf_record_contact is None else other_contact\n",
    "        print('\"{0}\" was not found.'.format(contact_not_found))\n",
    "        return\n",
    "    sorted_indices = (tfidf_record_contact - tfidf_record_other_contact).argsort()[::-1]\n",
    "    return pd.DataFrame({'Word': word_list.iloc[sorted_indices[:top_n]]}).reset_index(drop=True)\n",
    "\n",
    "# Returns the row in the TFIDF matrix for a given contact by name.\n",
    "def _get_tfidf_record_for_contact(contact):\n",
    "    if contact not in grouped_by_name.index:\n",
    "        return None\n",
    "    row = np.argmax(grouped_by_name.index == contact)\n",
    "    return tfidf_transformed_dataset.getrow(row).toarray().squeeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Words that identify a specific contact"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "widgets.interact(\n",
    "    get_word_summary_for_contact,\n",
    "    contact=widgets.Text(value='Mom', description='Contact name:', placeholder='Enter name'),\n",
    "    top_n=widgets.IntSlider(min=10, max=100, step=1, value=5, description='Max words to show:')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Words that identify the difference between two contacts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "widgets.interact(\n",
    "    get_word_summary_for_diffs,\n",
    "    contact=widgets.Text(description='1st Contact:', placeholder='Enter 1st name'),\n",
    "    other_contact=widgets.Text(description='2nd Contact:', placeholder='Enter 2nd name'),\n",
    "    top_n=widgets.IntSlider(description='Max words to show:', min=10, max=100, step=1, value=5)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Looking at language progression over the years"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper methods for looking at TFIDF by year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_words_by_year_from_tfidf(tfidf_by_year, years_as_list, top_n=15):\n",
    "    \"\"\"\n",
    "    Returns a dataframe of the top words for each year by their TFIDF score.\n",
    "    \n",
    "    To determine the \"top\", we look at one year's TFIDF - avg(other years' TFIDFs)\n",
    "    \n",
    "    Args:\n",
    "        tfidf_by_year: TFIDF matrix with as many rows as entries in years_as_list\n",
    "        years_as_list: Years that are represented in the TFIDF matrix\n",
    "        top_n: Number of top words per year to include in the result\n",
    "    \"\"\"\n",
    "    # Densify the tfidf matrix so we can operate on it.\n",
    "    tfidf_by_year_dense = tfidf_by_year.toarray()\n",
    "    df_by_year = []\n",
    "    for i in range(tfidf_by_year_dense.shape[0]):\n",
    "        this_year = years_as_list[i]\n",
    "        tfidf_this_year = tfidf_by_year_dense[i]\n",
    "        tfidf_other_years = np.delete(tfidf_by_year_dense, i, axis=0).mean(axis=0)\n",
    "        \n",
    "        sorted_indices = (tfidf_this_year - tfidf_other_years).argsort()[::-1]\n",
    "        df = pd.DataFrame({this_year: word_list.iloc[sorted_indices[:top_n]]})\n",
    "        df = df.reset_index(drop=True)\n",
    "        df_by_year.append(df)\n",
    "    return pd.concat(df_by_year, axis=1)\n",
    "\n",
    "def top_words_by_year_from_df(slice_of_texts_df, top_n=15, min_texts_required=100):\n",
    "    \"\"\"Returns a dataframe of the top words for each year by their TFIDF score.\n",
    "\n",
    "    Top is determined by the `top_words_by_year_from_tfidf` method.\n",
    "\n",
    "    Args:\n",
    "        slice_of_texts_df: A dataframe with the text messages to process\n",
    "        top_n: Number of top words per year to include in the result\n",
    "        min_texts_required: Number of texts to require in each year to not drop the record \n",
    "    \"\"\"\n",
    "    grouped_by_year_tfidf, years = _tfidf_by_year(slice_of_texts_df, min_texts_required)\n",
    "    return top_words_by_year_from_tfidf(grouped_by_year_tfidf, years, top_n)\n",
    "\n",
    "def _tfidf_by_year(slice_of_texts_df, min_texts_required=100):\n",
    "    \"\"\"Returns a TFIDF matrix of the texts grouped by year.\n",
    "    \n",
    "    Years with less than `min_texts_required` texts will be dropped.\n",
    "    \"\"\"\n",
    "    grouper = slice_of_texts_df.date.apply(lambda x: x.year)\n",
    "    grouped_by_year = slice_of_texts_df.groupby(grouper).apply(\n",
    "        lambda row: pd.Series({'count': len(row.date), 'text': ' '.join(row.text)})\n",
    "    )\n",
    "\n",
    "    # Drops years with less than min_texts_required texts since they won't be very meaningful.\n",
    "    years_to_drop = grouped_by_year[grouped_by_year['count'] < min_texts_required].index\n",
    "    print('Dropping year(s): {0}, each had fewer than {1} texts.'.format(\n",
    "        ', '.join(str(year) for year in years_to_drop), min_texts_required))\n",
    "    grouped_by_year = grouped_by_year[grouped_by_year['count'] >= min_texts_required]\n",
    "    grouped_by_year.index.name = 'year'\n",
    "\n",
    "    if grouped_by_year.shape[0] == 0:\n",
    "        print('Bailing out, no years found with at least {0} texts.'.format(min_texts_required))\n",
    "        return None\n",
    "\n",
    "    grouped_by_year_tfidf = vectorizer.transform(grouped_by_year['text'])\n",
    "    print('Found {0} years with more than {1} texts each.'.format(grouped_by_year_tfidf.shape[0],\n",
    "                                                                  min_texts_required))\n",
    "    return grouped_by_year_tfidf, grouped_by_year.index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### My top words over the years\n",
    "\n",
    "This offers an interesting insight into the main topics over the years."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words_by_year_from_df(fully_merged_messages_df[fully_merged_messages_df.is_from_me == 1],\n",
    "                          top_n=15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top words over the years from/to a specific contact\n",
    "\n",
    "This offers an interesting insight into the main topics over the years."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wrapper method so we can use interact().\n",
    "def _top_words_by_year_for_contact(contact, from_me, top_n):\n",
    "    contact = convert_unicode_to_str_if_needed(contact)\n",
    "    if contact not in full_names:\n",
    "        print('\"{0}\" not found'.format(contact))\n",
    "        return\n",
    "    # Slice to texts from/to the contact.\n",
    "    df = fully_merged_messages_df[(fully_merged_messages_df.is_from_me == from_me) &\n",
    "                                  (fully_merged_messages_df.full_name == contact)]\n",
    "    return top_words_by_year_from_df(df, top_n)\n",
    "\n",
    "widgets.interact(\n",
    "    _top_words_by_year_for_contact,\n",
    "    contact=widgets.Text(value='Mom', description='Contact name:', placeholder='Enter name'),\n",
    "    from_me=widgets.RadioButtons(\n",
    "        options={'Show messages FROM me': True, 'Show messages TO me': False}, description=' '),\n",
    "    top_n=widgets.IntSlider(min=15, max=100, step=1, value=5, description='Max words to show:')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "def _top_words_by_cluster_from_tfidf(\n",
    "    cluster_id,\n",
    "    tfidf_per_sender,\n",
    "    cluster_for_tfidf_index,\n",
    "    top_n=15,\n",
    "):\n",
    "    \"\"\"\n",
    "    Returns a dataframe of the top words for each cluster by their TFIDF score.\n",
    "    \n",
    "    To determine the \"top\", we look at one cluster's TFIDF - avg(other clusters' TFIDFs)\n",
    "    \n",
    "    Args:\n",
    "        cluster_id: The cluster we want to find the top words for (referred to as \"given cluster\")\n",
    "        tfidf_per_sender: TFIDF matrix with as many rows as entries in cluster_for_tfidf_index\n",
    "        cluster_for_tfidf_index: Cluster assignment for each entry in tfidf_per_sender\n",
    "        top_n: Number of top words per cluster to include in the result\n",
    "    \"\"\"\n",
    "    # First, we separate the given cluster we want to consider from all other entries.\n",
    "    this_cluster_records = tfidf_per_sender[cluster_for_tfidf_index == cluster_id]\n",
    "    other_cluster_records = tfidf_per_sender[cluster_for_tfidf_index != cluster_id]\n",
    "    \n",
    "    # Next, we calculate the mean for each: the given cluster and the rest of the corpus\n",
    "    mean_this_cluster = np.asarray(this_cluster_records.mean(axis=0)).squeeze()\n",
    "    mean_other_cluster = np.asarray(other_cluster_records.mean(axis=0)).squeeze()\n",
    "    \n",
    "    # Finally, we identify the words for which the given cluster shows the biggest difference.\n",
    "    difference = mean_this_cluster - mean_other_cluster\n",
    "    most_different_indicies = difference.argsort()\n",
    "    # Only display top_n\n",
    "    return most_different_indicies[::-1][:top_n]\n",
    "\n",
    "def _tfidf_by_sender(messages_df, min_texts_required=100):\n",
    "    \"\"\"Returns a TFIDF matrix of the texts grouped by sender.\n",
    "    \n",
    "    Message exchanges with less than `min_texts_required` texts will be dropped.\n",
    "    \"\"\"\n",
    "    # First we group messages by name, then we merge each conversation into one string.\n",
    "    grouped_by_name = messages_df.groupby(\"full_name\").apply(\n",
    "        lambda row: pd.Series({'count': len(row.full_name), 'text': ' '.join(row.text)})\n",
    "    )\n",
    "\n",
    "    # Drop all conversations that don't meet the requirements for minimum number of messages.\n",
    "    grouped_by_name = grouped_by_name[grouped_by_name['count'] >= min_texts_required]\n",
    "    grouped_by_name.index.name = 'full_name'\n",
    "\n",
    "    # Bail if we have no data\n",
    "    if grouped_by_name.shape[0] == 0:\n",
    "        print('Bailing out, no conversations found with at least {0} texts.'.format(min_texts_required))\n",
    "        return None\n",
    "\n",
    "    grouped_by_name_tfidf = vectorizer.transform(grouped_by_name['text'])\n",
    "    print('Found {0} conversations with at least than {1} texts each.'.format(grouped_by_name_tfidf.shape[0],\n",
    "                                                                  min_texts_required))\n",
    "    return grouped_by_name_tfidf, grouped_by_name.index\n",
    "\n",
    "# Get the TFIDF vector for each data point and the list of receivers.\n",
    "tfidf_per_sender, names_sender = _tfidf_by_sender(fully_merged_messages_df[fully_merged_messages_df.is_from_me == 0])\n",
    "\n",
    "# First, we reduce the dimensionality of the dataset.\n",
    "# This reduces the difference between the clusters found by KMeans and the 2D graphic of the clusters.\n",
    "tfidf_sender_reduced_dim = TruncatedSVD(n_components=7).fit_transform(tfidf_per_sender)\n",
    "\n",
    "# Let's run KMeans clustering on the data.\n",
    "NUMBER_OF_CLUSTERS = 7\n",
    "kmeans_tfidf_sender = KMeans(n_clusters=NUMBER_OF_CLUSTERS)\n",
    "tfidf_per_sender_cluster_assignment = kmeans_tfidf_sender.fit_transform(tfidf_sender_reduced_dim).argmin(axis=1)\n",
    "\n",
    "# We further reduce the dimensionality of the data, so that we can graph it.\n",
    "tfidf_per_sender_2d = TruncatedSVD(n_components=2).fit_transform(tfidf_sender_reduced_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clustered_tfidf_by_sender_df = pd.DataFrame({\n",
    "    \"x\": tfidf_per_sender_2d[:,0],\n",
    "    \"y\": tfidf_per_sender_2d[:,1],\n",
    "    \"name\": names_sender,\n",
    "    \"group\": [\"Cluster: \" + str(e) for e in tfidf_per_sender_cluster_assignment],\n",
    "})\n",
    "clustered_tfidf_by_sender_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import plotly.offline as py\n",
    "import plotly.figure_factory as ff\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "py.init_notebook_mode(connected=True)\n",
    "\n",
    "clusters = clustered_tfidf_by_sender_df.group.unique()\n",
    "\n",
    "def plot_data(cluster_selection):\n",
    "    traces = []\n",
    "    top_words = None\n",
    "    if cluster_selection == \"All\":\n",
    "        clusters_to_plot = clusters\n",
    "    else:\n",
    "        clusters_to_plot = [cluster_selection]\n",
    "        top_words_indexes = _top_words_by_cluster_from_tfidf(\n",
    "            int(cluster_selection[-1]),\n",
    "            tfidf_per_sender,\n",
    "            tfidf_per_sender_cluster_assignment\n",
    "        )[0:10]\n",
    "        top_words = word_list.iloc[top_words_indexes].to_frame()\n",
    "        top_words.columns = ['Top Words In Cluster']\n",
    "        top_words = top_words.reset_index(drop=True)\n",
    "    for cluster in clusters_to_plot:\n",
    "        cluster_data = clustered_tfidf_by_sender_df[clustered_tfidf_by_sender_df.group == cluster]\n",
    "        scatter = go.Scatter(\n",
    "            x=cluster_data[\"x\"],\n",
    "            y=cluster_data[\"y\"],\n",
    "            text=cluster_data[\"name\"],\n",
    "            mode = 'markers',\n",
    "            name=cluster\n",
    "        )\n",
    "        traces.append(scatter)\n",
    "    py.iplot(traces)\n",
    "    return top_words\n",
    "\n",
    "cluster_selection = widgets.Dropdown(\n",
    "    options=[\"All\"] + list(clusters),\n",
    "    value=\"All\",\n",
    "    description=\"Cluster: \"\n",
    ")\n",
    "print('We\\'ve clustered your contacts by their word usage, hover over the dots to see which '\n",
    "      'cluster each person is in. Adjust the dropdown to restrict to a cluster.\\nDots closer '\n",
    "      'to each other indicate the people talk similarly.')\n",
    "widgets.interact(\n",
    "    plot_data,\n",
    "    cluster_selection=cluster_selection,\n",
    ")\n",
    "display(cluster_selection)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}