Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Scweet.scweet import scrape\n",
    "from Scweet.user import get_user_information, get_users_following, get_users_followers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape tweets of a specific account or words or hashtag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping on headless mode.\n",
      "looking for tweets between 2015-04-01 and 2015-04-02 ...\n",
      " path : https://twitter.com/search?q=(bitcoin%20OR%20etherium)%20until%3A2015-04-02%20since%3A2015-04-01%20%20-filter%3Areplies&src=typed_query\n",
      "scroll  1\n",
      "Tweet made at: 2015-04-01T20:51:04.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:18:11.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:16:16.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:48:43.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:31:31.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:19:57.000Z is found.\n",
      "scroll  2\n",
      "Tweet made at: 2015-04-01T23:38:05.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:11:52.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:42:06.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:36:05.000Z is found.\n",
      "Tweet made at: 2015-04-01T21:31:53.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:32:30.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:22:00.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:23:16.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:22:05.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:26:38.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:04:20.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:05:57.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:33:16.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:45:02.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:35:10.000Z is found.\n",
      "scroll  3\n",
      "Tweet made at: 2015-04-01T21:50:09.000Z is found.\n",
      "Tweet made at: 2015-04-01T20:48:41.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:59:17.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:02:08.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:09:42.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:55:17.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:15:49.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:01:04.000Z is found.\n",
      "scroll  4\n",
      "Tweet made at: 2015-04-01T23:03:49.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:57:25.000Z is found.\n",
      "Tweet made at: 2015-04-01T20:48:44.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:55:57.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:45:28.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:10:45.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:08:13.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:25:08.000Z is found.\n",
      "Tweet made at: 2015-04-01T21:49:08.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:39:52.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:24:24.000Z is found.\n",
      "Tweet made at: 2015-04-01T21:56:52.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:21:31.000Z is found.\n",
      "Tweet made at: 2015-04-01T21:22:40.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:53:36.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:58:33.000Z is found.\n",
      "scroll  5\n",
      "Tweet made at: 2015-04-01T23:01:39.000Z is found.\n",
      "Tweet made at: 2015-04-01T20:50:09.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:15:08.000Z is found.\n",
      "Tweet made at: 2015-04-01T21:01:11.000Z is found.\n",
      "Tweet made at: 2015-04-01T21:53:05.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:55:10.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:41:13.000Z is found.\n",
      "Tweet made at: 2015-04-01T22:09:46.000Z is found.\n",
      "Tweet made at: 2015-04-01T23:07:38.000Z is found.\n",
      "Tweet made at: 2015-04-01T20:35:17.000Z is found.\n",
      "scroll  6\n",
      "scroll  7\n",
      "scroll  8\n"
     ]
    }
   ],
   "source": [
    "# scrape top tweets with the words 'covid','covid19' in proximity and without replies.\n",
    "# the process is slower as the interval is smaller (choose an interval that can divide the period of time betwee, start and max date)\n",
    "data = scrape(words=['bitcoin','etherium'], since=\"2015-04-01\", until=\"2015-04-2\", from_account = None,interval=1, \n",
    "      headless=True, display_type=\"Top\", save_images=False, proxy = None, save_dir = 'outputs',\n",
    "             resume=False, filter_replies=True, proximity=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# scrape top tweets of with the hashtag #covid19, in proximity and without replies.\n",
    "# the process is slower as the interval is smaller (choose an interval that can divide the period of time betwee, start and max date)\n",
    "data = scrap(hashtag=\"covid19\", start_date=\"2020-04-01\", max_date=\"2020-04-15\", from_account = None,interval=1, \n",
    "      headless=True, display_type=\"Top\", save_images=False, proxy = None, save_dir = 'outputs',\n",
    "             resume=False, filter_replies=True, proximity=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserScreenName</th>\n",
       "      <th>UserName</th>\n",
       "      <th>Timestamp</th>\n",
       "      <th>Text</th>\n",
       "      <th>Embedded_text</th>\n",
       "      <th>Emojis</th>\n",
       "      <th>Comments</th>\n",
       "      <th>Likes</th>\n",
       "      <th>Retweets</th>\n",
       "      <th>Image link</th>\n",
       "      <th>Tweet URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Daily Beast</td>\n",
       "      <td>@thedailybeast</td>\n",
       "      <td>2015-04-01T20:51:04.000Z</td>\n",
       "      <td>Rogue U.S. agents allegedly took down the worl...</td>\n",
       "      <td>Le média suivant comprend des contenus potenti...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>20</td>\n",
       "      <td>8</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://twitter.com/thedailybeast/status/58337...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pantera Capital</td>\n",
       "      <td>@PanteraCapital</td>\n",
       "      <td>2015-04-01T22:18:11.000Z</td>\n",
       "      <td>#Bitcoin as a superior utility for law enforce...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>22</td>\n",
       "      <td>[https://pbs.twimg.com/media/CBigysaUoAA2p6y?f...</td>\n",
       "      <td>https://twitter.com/PanteraCapital/status/5833...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>thebitcoinshark</td>\n",
       "      <td>@thebitcoinshark</td>\n",
       "      <td>2015-04-01T23:16:16.000Z</td>\n",
       "      <td>Bitcoin Trading Testing 1500 Chinese Yuan (CNY...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>https://twitter.com/thebitcoinshark/status/583...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CCN - Capital &amp; Celeb News</td>\n",
       "      <td>@CapitalAndCeleb</td>\n",
       "      <td>2015-04-01T23:48:43.000Z</td>\n",
       "      <td>Bitcoin Event: State of Digital Money Conferen...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td></td>\n",
       "      <td>[https://pbs.twimg.com/media/CBi1gp-WIAErf7t?f...</td>\n",
       "      <td>https://twitter.com/CapitalAndCeleb/status/583...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Skutch</td>\n",
       "      <td>@1constitution</td>\n",
       "      <td>2015-04-01T22:31:31.000Z</td>\n",
       "      <td>Collective ♪♫ Change\\nhttp://skutch.com.au/Cha...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://twitter.com/1constitution/status/58339...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               UserScreenName          UserName                 Timestamp  \\\n",
       "0             The Daily Beast    @thedailybeast  2015-04-01T20:51:04.000Z   \n",
       "1             Pantera Capital   @PanteraCapital  2015-04-01T22:18:11.000Z   \n",
       "2             thebitcoinshark  @thebitcoinshark  2015-04-01T23:16:16.000Z   \n",
       "3  CCN - Capital & Celeb News  @CapitalAndCeleb  2015-04-01T23:48:43.000Z   \n",
       "4                      Skutch    @1constitution  2015-04-01T22:31:31.000Z   \n",
       "\n",
       "                                                Text  \\\n",
       "0  Rogue U.S. agents allegedly took down the worl...   \n",
       "1  #Bitcoin as a superior utility for law enforce...   \n",
       "2  Bitcoin Trading Testing 1500 Chinese Yuan (CNY...   \n",
       "3  Bitcoin Event: State of Digital Money Conferen...   \n",
       "4  Collective ♪♫ Change\\nhttp://skutch.com.au/Cha...   \n",
       "\n",
       "                                       Embedded_text Emojis Comments Likes  \\\n",
       "0  Le média suivant comprend des contenus potenti...                    20   \n",
       "1                                                                  1    27   \n",
       "2                                                                        1   \n",
       "3                                                                       16   \n",
       "4                                                                        1   \n",
       "\n",
       "  Retweets                                         Image link  \\\n",
       "0        8                                                 []   \n",
       "1       22  [https://pbs.twimg.com/media/CBigysaUoAA2p6y?f...   \n",
       "2                                                          []   \n",
       "3           [https://pbs.twimg.com/media/CBi1gp-WIAErf7t?f...   \n",
       "4        2                                                 []   \n",
       "\n",
       "                                           Tweet URL  \n",
       "0  https://twitter.com/thedailybeast/status/58337...  \n",
       "1  https://twitter.com/PanteraCapital/status/5833...  \n",
       "2  https://twitter.com/thebitcoinshark/status/583...  \n",
       "3  https://twitter.com/CapitalAndCeleb/status/583...  \n",
       "4  https://twitter.com/1constitution/status/58339...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the main information of a given list of users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These users belongs to my following. \n",
    "users = ['nagouzil', '@yassineaitjeddi', 'TahaAlamIdrissi',] \n",
    "         #'@Nabila_Gl', 'geceeekusuu', '@pabu232', '@av_ahmet', '@x_born_to_die_x']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping on headless mode.\n",
      "--------------- nagouzil information : ---------------\n",
      "Following :  83\n",
      "Followers :  27\n",
      "Location :  \n",
      "Join date :  A rejoint Twitter en septembre 2012\n",
      "Birth date :  \n",
      "Description :  Make a difference\n",
      "Website :  \n",
      "--------------- @yassineaitjeddi information : ---------------\n",
      "Following :  125\n",
      "Followers :  5\n",
      "Location :  \n",
      "Join date :  A rejoint Twitter en octobre 2016\n",
      "Birth date :  \n",
      "Description :  \n",
      "Website :  \n",
      "--------------- TahaAlamIdrissi information : ---------------\n",
      "Following :  382\n",
      "Followers :  155\n",
      "Location :  France\n",
      "Join date :  A rejoint Twitter en août 2012\n",
      "Birth date :  \n",
      "Description :  #SoftwareEngineeringStudentAtTSE\n",
      "#DEVOPS_ENTHUSIAST\n",
      "#FULL_STACK_WEB_DEV\n",
      "Website :  \n"
     ]
    }
   ],
   "source": [
    "# this function return a list that contains : \n",
    "# [\"nb of following\",\"nb of followers\", \"join date\", \"birthdate\", \"location\", \"website\", \"description\"]\n",
    "users_info = get_user_information(users, headless=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nb of following</th>\n",
       "      <th>nb of followers</th>\n",
       "      <th>join date</th>\n",
       "      <th>birthdate</th>\n",
       "      <th>location</th>\n",
       "      <th>website</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>nagouzil</th>\n",
       "      <td>83</td>\n",
       "      <td>27</td>\n",
       "      <td>A rejoint Twitter en septembre 2012</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Make a difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>@yassineaitjeddi</th>\n",
       "      <td>125</td>\n",
       "      <td>5</td>\n",
       "      <td>A rejoint Twitter en octobre 2016</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TahaAlamIdrissi</th>\n",
       "      <td>382</td>\n",
       "      <td>155</td>\n",
       "      <td>A rejoint Twitter en août 2012</td>\n",
       "      <td></td>\n",
       "      <td>France</td>\n",
       "      <td></td>\n",
       "      <td>#SoftwareEngineeringStudentAtTSE\\n#DEVOPS_ENTH...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 nb of following nb of followers  \\\n",
       "nagouzil                      83              27   \n",
       "@yassineaitjeddi             125               5   \n",
       "TahaAlamIdrissi              382             155   \n",
       "\n",
       "                                            join date birthdate location  \\\n",
       "nagouzil          A rejoint Twitter en septembre 2012                      \n",
       "@yassineaitjeddi    A rejoint Twitter en octobre 2016                      \n",
       "TahaAlamIdrissi        A rejoint Twitter en août 2012             France   \n",
       "\n",
       "                 website                                        description  \n",
       "nagouzil                                                  Make a difference  \n",
       "@yassineaitjeddi                                                             \n",
       "TahaAlamIdrissi           #SoftwareEngineeringStudentAtTSE\\n#DEVOPS_ENTH...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "users_df = pd.DataFrame(users_info, index = [\"nb of following\",\"nb of followers\", \"join date\", \n",
    "                                             \"birthdate\", \"location\", \"website\", \"description\"]).T\n",
    "users_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get followers and following of a given list of users"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Enter your username and password in .env file. I recommande you dont use your main account.  \n",
    "Increase wait argument to avoid banning your account and maximise the crawling process if the internet is slow. I used 1 and it's safe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nagouzil'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_path = \".env\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping on headless mode.\n",
      "Crawling nagouzil following\n",
      "Found 8 following\n",
      "Found 37 following\n",
      "Found 59 following\n",
      "Found 79 following\n",
      "Found 83 following\n",
      "Found 83 following\n",
      "file saved in outputs/nagouzil_nagouzil_followers.json\n"
     ]
    }
   ],
   "source": [
    "following = get_users_following(users=['nagouzil'], env=env_path, verbose=0, headless = True, wait=2, file_path='outputs/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['@FabrizioRomano',\n",
       " '@TrollFootball',\n",
       " '@madelyncline',\n",
       " '@tweetsauce',\n",
       " '@jockowillink',\n",
       " '@brfootball',\n",
       " '@433',\n",
       " '@Ibra_official',\n",
       " '@HSHQ',\n",
       " '@Snowden']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "following['nagouzil'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping on headless mode.\n",
      "Crawling nagouzil followers\n",
      "Found 19 followers\n",
      "Found 26 followers\n",
      "Found 26 followers\n",
      "file saved in outputs/nagouzil_nagouzil_followers.json\n"
     ]
    }
   ],
   "source": [
    "followers = get_users_followers(users=['nagouzil'], env=env_path, verbose=0, headless = True, wait=2, file_path='outputs/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['@ayoubagouzil',\n",
       " '@zinebmouchrik',\n",
       " '@thehypevalue',\n",
       " '@MahmuduR_Design',\n",
       " '@DesignsNoved',\n",
       " '@Isaisma1',\n",
       " '@MehdiElIdriss23',\n",
       " '@zohair79262319',\n",
       " '@ZotefeJean',\n",
       " '@Fervidlife']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "followers['nagouzil'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}