process_CURIO_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "DOWNLOAD_LATEST = False\n",
    "import os\n",
    "if DOWNLOAD_LATEST:\n",
    "    env_vars = open(\".env.development\").read()\n",
    "    DB_CONNECTION_STRING = env_vars.split(\"DB_CONNECTION_STRING = \")[1].split(\"?authSource\")[0]\n",
    "    os.system(f\"mongoexport --uri {DB_CONNECTION_STRING} --collection users --out data/Curio/users.json\")\n",
    "    os.system(f\"mongoexport --uri {DB_CONNECTION_STRING} --collection posts --out data/Curio/posts.json\")\n",
    "    os.system(f\"mongoexport --uri {DB_CONNECTION_STRING} --collection groups --out data/Curio/groups.json\")\n",
    "    os.system(f\"mongoexport --uri {DB_CONNECTION_STRING} --collection reactions --out data/Curio/reactions.json\")\n",
    "    os.system(f\"mongoexport --uri {DB_CONNECTION_STRING} --collection echopostproposals --out data/Curio/echopostproposals.json\")\n",
    "    os.system(f\"mongoexport --uri {DB_CONNECTION_STRING} --collection viewtimestamps --out data/Curio/viewtimestamps.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jsonlines\n",
    "import pandas as pd\n",
    "from superdebug import debug\n",
    "from collections import defaultdict\n",
    "curio_data_path = \"data/Curio/\"\n",
    "users = jsonlines.Reader(open(curio_data_path + \"users.json\"))\n",
    "posts = jsonlines.Reader(open(curio_data_path + \"posts.json\"))\n",
    "groups = jsonlines.Reader(open(curio_data_path + \"groups.json\"))\n",
    "reactions = jsonlines.Reader(open(curio_data_path + \"reactions.json\"))\n",
    "echopostproposals = jsonlines.Reader(open(curio_data_path + \"echopostproposals.json\"))\n",
    "viewtimestamps = jsonlines.Reader(open(curio_data_path + \"viewtimestamps.json\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Record necessary information: `group_id_name_map, group_id_members_id_map, user_id_name_map, user_id_group_id_viewtime_map`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:48 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_member_num'], at \u001b[0m\u001b[1;32m<ipython-input-214-5d1bfe551d68>:20 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 205.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_member_num\u001b[0m\u001b[1;33m dict {...} with 20 keys ['LOLCHI', 'Trivia', 'Team|Llama', 'Team|Mignon', 'The|Positive|Corner', 'Random', 'Food', 'Out|of|Context|Quotes', 'Team|PhThree', 'Stanford|HCI', 'Campus|Events', 'Stanford|Wordle', 'Where|Am|I', 'Bugs', 'Cute|Animals', 'Music', 'Amazing|AI', 'World|Cup|2022', 'Stanford', 'Hot|Takes']\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mLOLCHI\u001b[0m\u001b[1;33m num val: 26\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTrivia\u001b[0m\u001b[1;33m num val: 18\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Llama\u001b[0m\u001b[1;33m num val: 7\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Mignon\u001b[0m\u001b[1;33m num val: 9\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mThe|Positive|Corner\u001b[0m\u001b[1;33m num val: 18\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mRandom\u001b[0m\u001b[1;33m num val: 18\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mFood\u001b[0m\u001b[1;33m num val: 24\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mOut|of|Context|Quotes\u001b[0m\u001b[1;33m num val: 19\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|PhThree\u001b[0m\u001b[1;33m num val: 7\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|HCI\u001b[0m\u001b[1;33m num val: 24\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCampus|Events\u001b[0m\u001b[1;33m num val: 6\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|Wordle\u001b[0m\u001b[1;33m num val: 4\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWhere|Am|I\u001b[0m\u001b[1;33m num val: 9\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mBugs\u001b[0m\u001b[1;33m num val: 12\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCute|Animals\u001b[0m\u001b[1;33m num val: 9\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mMusic\u001b[0m\u001b[1;33m num val: 9\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mAmazing|AI\u001b[0m\u001b[1;33m num val: 6\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWorld|Cup|2022\u001b[0m\u001b[1;33m num val: 3\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford\u001b[0m\u001b[1;33m num val: 11\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mHot|Takes\u001b[0m\u001b[1;33m num val: 3\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:48 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "group_id_members_name_map: {'LOLCHI': ['msb', 'wanrong', 'klee', 'parnoud', 'kjeong', 'joinbutton', 'Joyce', 'lindsay', 'amphinomid', 'leping', 'jigar', 'troutman', 'joon', 'elliejvela', 'michellelam', 'mitchell', 'nicolejg', 'Archit', 'jseering', 'ehansen', 'noahsveiven', 'alishah', 'parth'], 'Trivia': ['wanrong', 'nicolejg', 'parnoud', 'lindsay', 'msb', 'ulo', 'kjeong', 'th0mas', 'amphinomid', 'Joyce', 'troutman', 'joon', 'elliejvela', 'michellelam', 'mitchell', 'jigar', 'parth'], 'Team|Llama': ['wanrong', 'nicolejg', 'msb', 'kjeong', 'amphinomid', 'parth'], 'Team|Mignon': ['nicolejg', 'parnoud', 'lindsay', 'Joyce', 'th0mas', 'ulo', 'msb', 'parth'], 'The|Positive|Corner': ['wanrong', 'parnoud', 'Joyce', 'ulo', 'kjeong', 'th0mas', 'amphinomid', 'nicolejg', 'msb', 'nasty nas', 'jobrien', 'elliejvela', 'troutman', 'jseering', 'Archit', 'cpak23', 'ehansen', 'parth'], 'Random': ['nicolejg', 'parnoud', 'lindsay', 'kjeong', 'msb', 'amphinomid', 'Joyce', 'troutman', 'joon', 'elliejvela', 'michellelam', 'mitchell', 'wanrong', 'jseering', 'Archit', 'cpak23', 'parth'], 'Grandchild|1|(TEST)': ['parnoud', 'nicolejg', 'msb', 'kjeong', 'Archit', 'msbtest4', 'Gayoung', 'jigar', 'parth'], 'Food': ['msb', 'wanrong', 'klee', 'parnoud', 'kjeong', 'joinbutton', 'Joyce', 'lindsay', 'amphinomid', 'leping', 'nicolejg', 'joon', 'elliejvela', 'michellelam', 'mitchell', 'troutman', 'Archit', 'dirgh', 'cpak23', 'ehansen', 'parth'], 'Out|of|Context|Quotes': ['lindsay', 'parnoud', 'nicolejg', 'amphinomid', 'kjeong', 'joon', 'elliejvela', 'michellelam', 'mitchell', 'troutman', 'wanrong', 'Joyce', 'msb', 'jseering', 'cpak23', 'alishah', 'noahsveiven', 'parth'], 'Team|PhThree': ['nicolejg', 'joon', 'troutman', 'michellelam', 'mitchell', 'Archit', 'parth'], 'Child|2|(TEST)': ['parnoud', 'nicolejg', 'kjeong', 'Archit', 'msbtest4', 'Gayoung', 'parth', 'msb'], 'Stanford|HCI': ['msb', 'wanrong', 'parnoud', 'kjeong', 'joinbutton', 'lindsay', 'ulo', 'amphinomid', 'Joyce', 'th0mas', 'jigar', 'troutman', 'joon', 'elliejvela', 'michellelam', 'mitchell', 'msbtest4', 'Archit', 'jseering', 'nicolejg', 'ehansen', 'parth'], 'No|Posts|Group|(TEST)': ['msb', 'parnoud', 'kjeong', 'nicolejg', 'msbtest4', 'Gayoung', 'Archit'], 'Campus|Events': ['kjeong', 'msb', 'parnoud', 'wanrong', 'lindsay', 'ehansen'], 'Stanford|Wordle': ['kjeong', 'wanrong', 'parnoud', 'lindsay'], 'Where|Am|I': ['kjeong', 'nicolejg', 'msb', 'parnoud', 'wanrong', 'lindsay', 'esu', 'cpak23', 'parth'], 'Bugs': ['kjeong', 'cpak23', 'empark', 'msb', 'jacobrobertsbaca', 'ehansen', 'alishah', 'noahsveiven', 'ant.qin', 'parnoud', 'nicolejg'], 'Cute|Animals': ['kjeong', 'lindsay', 'parnoud', 'wanrong', 'nicolejg', 'ehansen', 'alishah', 'parth'], 'Music': ['nicolejg', 'esu', 'kjeong', 'cpak23', 'msb', 'wanrong', 'ehansen', 'lindsay', 'ant.qin'], 'Amazing|AI': ['wanrong', 'lindsay', 'kjeong', 'ehansen', 'parth', 'msb'], 'Parent|(TEST)': ['prinse', 'msb', 'nicolejg', 'jigar', 'parnoud', 'wanrong', 'lindsay', 'parth'], 'World|Cup|2022': ['lgetz', 'parnoud', 'alishah'], 'Stanford': ['wanrong', 'alishah', 'parnoud', 'lindsay', 'noahsveiven', 'ant.qin', 'empark', 'msb', 'parth', 'kjeong', 'nicolejg'], 'Child|1|(TEST)': ['prinse', 'msb', 'nicolejg', 'jigar', 'parnoud', 'wanrong', 'lindsay', 'parth'], 'Hot|Takes': ['alishah', 'parnoud', 'msb']}\n"
     ]
    }
   ],
   "source": [
    "user_id_name_map = {}\n",
    "for user in users:\n",
    "    user_id = user['_id'][\"$oid\"]\n",
    "    user_name = user[\"username\"]\n",
    "    user_id_name_map[user_id] = user_name\n",
    "    \n",
    "group_id_members_id_map = {}\n",
    "group_id_members_name_map = {}\n",
    "group_name_member_num_map = {}\n",
    "group_id_name_map = {}\n",
    "for group in groups:\n",
    "    group_id = group['_id'][\"$oid\"]\n",
    "    group_name = group[\"name\"].replace(\" \", \"|\")\n",
    "    group_id_name_map[group_id] = group_name\n",
    "    members = [_[\"$oid\"] for _ in group['members']]\n",
    "    group_id_members_id_map[group_id] = members\n",
    "    group_id_members_name_map[group_name] = [user_id_name_map[user_id] for user_id in members if user_id in user_id_name_map]\n",
    "    if \"TEST\" not in group_name:\n",
    "        group_name_member_num_map[group_name] = len(members)\n",
    "debug(group_member_num = group_name_member_num_map)\n",
    "print(\"group_id_members_name_map:\", group_id_members_name_map)\n",
    "    \n",
    "user_id_group_id_viewtime_map = defaultdict(dict)\n",
    "for viewtimestamp in viewtimestamps:\n",
    "    if viewtimestamp[\"user\"] is None: continue\n",
    "    user_id = viewtimestamp[\"user\"][\"$oid\"]\n",
    "    if \"group\" not in viewtimestamp or viewtimestamp[\"group\"] == None:\n",
    "        group_ids = []\n",
    "        for group_id in group_id_members_id_map:\n",
    "            if user_id in group_id_members_id_map[group_id]:\n",
    "                group_ids.append(group_id)\n",
    "    else:\n",
    "        group_ids = [viewtimestamp[\"group\"][\"$oid\"]]\n",
    "    dt_obj = pd.to_datetime(viewtimestamp[\"timestamp\"][\"$date\"], infer_datetime_format=True)\n",
    "    timestamp = dt_obj.timestamp()\n",
    "    for group_id in group_ids:\n",
    "        if group_id not in user_id_group_id_viewtime_map[user_id] or timestamp > user_id_group_id_viewtime_map[user_id][group_id]:\n",
    "            user_id_group_id_viewtime_map[user_id][group_id] = timestamp\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Aggregate information about each post, and compose `posts_df`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_active_post_member_num'], at \u001b[0m\u001b[1;32m<ipython-input-215-870b060c80b3>:53 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 206.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_active_post_member_num\u001b[0m\u001b[1;33m str len 348: {'LOLCHI': 11, 'Trivia': 9, 'Random': 7, 'Team|Llama': 5, 'Team|Mignon': 4, 'The|Positive|Corner': 9, 'Food': 6, 'Out|of|Context|Quotes': 9, 'Team|PhThree': 3, 'Campus|Events': 2, 'Where|Am|I': 5, 'Stanford': 2, 'Stanford|Wordle': 1, 'Cute|Animals': 1, 'Music': 4, 'Bugs': 6, 'Stanford|HCI': 1, 'Amazing|AI': 2, 'World|Cup|2022': 3, 'Hot|Takes': 1}\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_active_post_member_num_2'], at \u001b[0m\u001b[1;32m<ipython-input-215-870b060c80b3>:54 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 207.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_active_post_member_num_2\u001b[0m\u001b[1;33m str len 347: {'LOLCHI': 7, 'Trivia': 6, 'Random': 6, 'Team|Llama': 4, 'Team|Mignon': 4, 'The|Positive|Corner': 8, 'Food': 6, 'Out|of|Context|Quotes': 7, 'Team|PhThree': 3, 'Campus|Events': 2, 'Where|Am|I': 4, 'Stanford': 1, 'Stanford|Wordle': 0, 'Cute|Animals': 1, 'Music': 1, 'Bugs': 1, 'Stanford|HCI': 0, 'Amazing|AI': 1, 'World|Cup|2022': 1, 'Hot|Takes': 0}\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from collections import defaultdict, Counter\n",
    "posts_info_map = defaultdict(dict)\n",
    "group_name_post_num_map = defaultdict(int)\n",
    "group_name_first_post_time_map = defaultdict(int)\n",
    "group_name_active_post_member_map = defaultdict(Counter)\n",
    "group_name_active_member_map = defaultdict(Counter)\n",
    "active_member_day_thres = 10000 # TODO:\n",
    "for post in posts:\n",
    "    group_id = post[\"group\"][\"$oid\"]\n",
    "    group_name = group_id_name_map[group_id]\n",
    "    if \"TEST\" in group_name or \"test\" in group_name:\n",
    "        continue\n",
    "    \n",
    "    group_name_post_num_map[group_name] += 1\n",
    "    \n",
    "    post_id = post[\"_id\"][\"$oid\"]\n",
    "    post_info = posts_info_map[post_id] # information about the post\n",
    "    post_info[\"SUBMISSION_ID\"] = post_id\n",
    "    \n",
    "    post_info[\"SUBREDDIT\"] = group_name\n",
    "    post_info[\"group_id\"] = group_id\n",
    "    title = post[\"contents\"]\n",
    "    post_info[\"TITLE\"] = title\n",
    "    \n",
    "    author_id = post[\"author\"][\"$oid\"]\n",
    "    author = user_id_name_map[author_id]\n",
    "    post_info[\"author_id\"] = author_id\n",
    "    post_info[\"AUTHOR\"] = author\n",
    "    post_info[\"NSFW\"] = None\n",
    "    post_info[\"LINK\"] = \"\"\n",
    "    \n",
    "    dt_obj = pd.to_datetime(post[\"updatedAt\"][\"$date\"], infer_datetime_format=True)\n",
    "    timestamp = dt_obj.timestamp()\n",
    "    post_info[\"timestamp\"] = timestamp\n",
    "    if (group_name not in group_name_first_post_time_map) or timestamp < group_name_first_post_time_map[group_name]:\n",
    "        group_name_first_post_time_map[group_name] = timestamp\n",
    "    \n",
    "    if abs(time.time() - timestamp) <= active_member_day_thres * 24 * 60 * 60 and author_id in group_id_members_id_map[group_id]:\n",
    "        group_name_active_member_map[group_name][author_id] += 1\n",
    "        group_name_active_post_member_map[group_name][author_id] += 1\n",
    "    \n",
    "    if \"#_COMMENTS\" not in post_info:\n",
    "        post_info[\"#_COMMENTS\"] = 0\n",
    "        \n",
    "    if \"threadParent\" in post:\n",
    "        threadParent_id = post[\"threadParent\"][\"$oid\"]\n",
    "        if \"#_COMMENTS\" not in posts_info_map[threadParent_id]:\n",
    "            posts_info_map[threadParent_id][\"#_COMMENTS\"] = 1\n",
    "        else:\n",
    "            posts_info_map[threadParent_id][\"#_COMMENTS\"] += 1\n",
    "\n",
    "debug(group_active_post_member_num = str({name: len(members) for name, members in group_name_active_post_member_map.items()}))\n",
    "debug(group_active_post_member_num_2 = str({name: len([member for member in members if members[member] >=2]) for name, members in group_name_active_post_member_map.items()}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Analyze statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'LOLCHI': '0.1701', 'Trivia': '0.2675', 'Random': '0.2032', 'Team|Llama': '0.4445', 'Team|Mignon': '0.5668', 'The|Positive|Corner': '0.3441', 'Food': '0.2467', 'Out|of|Context|Quotes': '0.3825', 'Team|PhThree': '0.0940', 'Campus|Events': '0.5010', 'Where|Am|I': '0.4174', 'Stanford': '0.0501', 'Stanford|Wordle': '0.0214', 'Cute|Animals': '0.2033', 'Music': '0.3186', 'Bugs': '0.3742', 'Stanford|HCI': '0.0535', 'Amazing|AI': '0.1627', 'World|Cup|2022': '0.2810', 'Hot|Takes': '0.0910'}\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_posting_frequency_day'], at \u001b[0m\u001b[1;32m<ipython-input-216-43d8415bbca7>:3 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 208.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_posting_frequency_day\u001b[0m\u001b[1;33m dict {...} with 20 keys ['LOLCHI', 'Trivia', 'Random', 'Team|Llama', 'Team|Mignon', 'The|Positive|Corner', 'Food', 'Out|of|Context|Quotes', 'Team|PhThree', 'Campus|Events', 'Where|Am|I', 'Stanford', 'Stanford|Wordle', 'Cute|Animals', 'Music', 'Bugs', 'Stanford|HCI', 'Amazing|AI', 'World|Cup|2022', 'Hot|Takes']\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mLOLCHI\u001b[0m\u001b[1;33m str len 6: 0.1701\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTrivia\u001b[0m\u001b[1;33m str len 6: 0.2675\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mRandom\u001b[0m\u001b[1;33m str len 6: 0.2032\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Llama\u001b[0m\u001b[1;33m str len 6: 0.4445\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Mignon\u001b[0m\u001b[1;33m str len 6: 0.5668\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mThe|Positive|Corner\u001b[0m\u001b[1;33m str len 6: 0.3441\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mFood\u001b[0m\u001b[1;33m str len 6: 0.2467\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mOut|of|Context|Quotes\u001b[0m\u001b[1;33m str len 6: 0.3825\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|PhThree\u001b[0m\u001b[1;33m str len 6: 0.0940\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCampus|Events\u001b[0m\u001b[1;33m str len 6: 0.5010\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWhere|Am|I\u001b[0m\u001b[1;33m str len 6: 0.4174\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford\u001b[0m\u001b[1;33m str len 6: 0.0501\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|Wordle\u001b[0m\u001b[1;33m str len 6: 0.0214\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCute|Animals\u001b[0m\u001b[1;33m str len 6: 0.2033\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mMusic\u001b[0m\u001b[1;33m str len 6: 0.3186\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mBugs\u001b[0m\u001b[1;33m str len 6: 0.3742\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|HCI\u001b[0m\u001b[1;33m str len 6: 0.0535\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mAmazing|AI\u001b[0m\u001b[1;33m str len 6: 0.1627\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWorld|Cup|2022\u001b[0m\u001b[1;33m str len 6: 0.2810\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mHot|Takes\u001b[0m\u001b[1;33m str len 6: 0.0910\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "{'LOLCHI': '0.0065', 'Trivia': '0.0149', 'Random': '0.0113', 'Team|Llama': '0.0635', 'Team|Mignon': '0.0630', 'The|Positive|Corner': '0.0191', 'Food': '0.0103', 'Out|of|Context|Quotes': '0.0201', 'Team|PhThree': '0.0134', 'Campus|Events': '0.0835', 'Where|Am|I': '0.0464', 'Stanford': '0.0046', 'Stanford|Wordle': '0.0054', 'Cute|Animals': '0.0226', 'Music': '0.0354', 'Bugs': '0.0312', 'Stanford|HCI': '0.0022', 'Amazing|AI': '0.0271', 'World|Cup|2022': '0.0937', 'Hot|Takes': '0.0303'}\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_posting_frequency_per_person_day'], at \u001b[0m\u001b[1;32m<ipython-input-216-43d8415bbca7>:7 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 209.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_posting_frequency_per_person_day\u001b[0m\u001b[1;33m dict {...} with 20 keys ['LOLCHI', 'Trivia', 'Random', 'Team|Llama', 'Team|Mignon', 'The|Positive|Corner', 'Food', 'Out|of|Context|Quotes', 'Team|PhThree', 'Campus|Events', 'Where|Am|I', 'Stanford', 'Stanford|Wordle', 'Cute|Animals', 'Music', 'Bugs', 'Stanford|HCI', 'Amazing|AI', 'World|Cup|2022', 'Hot|Takes']\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mLOLCHI\u001b[0m\u001b[1;33m str len 6: 0.0065\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTrivia\u001b[0m\u001b[1;33m str len 6: 0.0149\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mRandom\u001b[0m\u001b[1;33m str len 6: 0.0113\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Llama\u001b[0m\u001b[1;33m str len 6: 0.0635\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Mignon\u001b[0m\u001b[1;33m str len 6: 0.0630\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mThe|Positive|Corner\u001b[0m\u001b[1;33m str len 6: 0.0191\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mFood\u001b[0m\u001b[1;33m str len 6: 0.0103\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mOut|of|Context|Quotes\u001b[0m\u001b[1;33m str len 6: 0.0201\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|PhThree\u001b[0m\u001b[1;33m str len 6: 0.0134\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCampus|Events\u001b[0m\u001b[1;33m str len 6: 0.0835\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWhere|Am|I\u001b[0m\u001b[1;33m str len 6: 0.0464\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford\u001b[0m\u001b[1;33m str len 6: 0.0046\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|Wordle\u001b[0m\u001b[1;33m str len 6: 0.0054\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCute|Animals\u001b[0m\u001b[1;33m str len 6: 0.0226\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mMusic\u001b[0m\u001b[1;33m str len 6: 0.0354\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mBugs\u001b[0m\u001b[1;33m str len 6: 0.0312\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|HCI\u001b[0m\u001b[1;33m str len 6: 0.0022\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mAmazing|AI\u001b[0m\u001b[1;33m str len 6: 0.0271\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWorld|Cup|2022\u001b[0m\u001b[1;33m str len 6: 0.0937\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mHot|Takes\u001b[0m\u001b[1;33m str len 6: 0.0303\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:49 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "group_name_post_frequency_map = {group_name: (60 * 60 * 24 * group_name_post_num_map[group_name])/(time.time() - group_name_first_post_time_map[group_name]) for group_name in group_name_post_num_map}\n",
    "print({group_name: f\"{group_name_post_frequency_map[group_name]:.4f}\" for group_name in group_name_post_frequency_map})\n",
    "debug(group_posting_frequency_day = {group_name: f\"{group_name_post_frequency_map[group_name]:.4f}\" for group_name in group_name_post_frequency_map})\n",
    "\n",
    "group_name_post_frequency_per_person_map = {group_name: group_name_post_frequency_map[group_name]/group_name_member_num_map[group_name] for group_name in group_name_post_frequency_map}\n",
    "print({group_name: f\"{group_name_post_frequency_per_person_map[group_name]:.4f}\" for group_name in group_name_post_frequency_per_person_map})\n",
    "debug(group_posting_frequency_per_person_day = {group_name: f\"{group_name_post_frequency_per_person_map[group_name]:.4f}\" for group_name in group_name_post_frequency_per_person_map})\n",
    "posts_df = pd.DataFrame.from_records(list(posts_info_map.values()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Record users' votes on posts, set curation parameters for each community and compose `votes_df`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts_users_votes = defaultdict(dict)\n",
    "for post_id, post_info in posts_info_map.items():\n",
    "    if \"SUBMISSION_ID\" not in post_info or \"SUBREDDIT\" not in post_info: continue\n",
    "    if post_info[\"SUBREDDIT\"] == \"\" or post_info[\"SUBREDDIT\"] == None: continue\n",
    "    assert post_id == post_info[\"SUBMISSION_ID\"]\n",
    "    post_timestamp = post_info[\"timestamp\"]\n",
    "    group_id = post_info[\"group_id\"]\n",
    "    author_id = post_info[\"author_id\"]\n",
    "    group_name = post_info[\"SUBREDDIT\"]\n",
    "    group_members = group_id_members_id_map[group_id]\n",
    "    \n",
    "    # fill in downvotes\n",
    "    users_votes = posts_users_votes[post_id]\n",
    "    for member_id in group_members:\n",
    "        if member_id in user_id_name_map and (group_id not in user_id_group_id_viewtime_map[member_id] or post_timestamp < user_id_group_id_viewtime_map[member_id][group_id]):\n",
    "            users_votes[member_id] = {\"SUBMISSION_ID\": post_id,\t\"SUBREDDIT\": \"r/\" + group_name, \"CREATED_TIME\": post_timestamp, \"USERNAME\": user_id_name_map[member_id], \"VOTE\": \"downvote\"}\n",
    "    if author_id in users_votes:\n",
    "        users_votes[author_id][\"VOTE\"] = \"upvote\" # author always upvotes\n",
    "    else:\n",
    "        users_votes[author_id] = {\"SUBMISSION_ID\": post_id,\t\"SUBREDDIT\": \"r/\" + group_name, \"CREATED_TIME\": post_timestamp, \"USERNAME\": user_id_name_map[author_id], \"VOTE\": \"upvote\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "\n",
    "\n",
    "for reaction in reactions:\n",
    "    post_id = reaction[\"post\"][\"$oid\"]\n",
    "    if post_id not in posts_users_votes: continue\n",
    "    author_id = reaction[\"author\"][\"$oid\"]\n",
    "    if author_id in posts_users_votes[post_id]:\n",
    "        posts_users_votes[post_id][author_id][\"VOTE\"] = \"upvote\"\n",
    "    else:\n",
    "        posts_users_votes[post_id][author_id] = copy.deepcopy(list(posts_users_votes[post_id].values())[0])\n",
    "        posts_users_votes[post_id][author_id][\"VOTE\"] = \"upvote\"\n",
    "        posts_users_votes[post_id][author_id][\"USERNAME\"] = user_id_name_map[author_id]\n",
    "    group_id = posts_info_map[post_id][\"group_id\"]\n",
    "    group_name = group_id_name_map[group_id]\n",
    "    dt_obj = pd.to_datetime(reaction[\"updatedAt\"][\"$date\"], infer_datetime_format=True)\n",
    "    timestamp = dt_obj.timestamp()\n",
    "    if abs(time.time() - timestamp) <= active_member_day_thres * 24 * 60 * 60 and author_id in group_id_members_id_map[group_id]:\n",
    "        group_name_active_member_map[group_name][author_id] += 1\n",
    "\n",
    "for echopostproposal in echopostproposals:\n",
    "    post_id = echopostproposal[\"echoParent\"][\"$oid\"]\n",
    "    if post_id not in posts_users_votes: continue\n",
    "    author_id = echopostproposal[\"echoAuthor\"][\"$oid\"]\n",
    "    if author_id not in posts_users_votes[post_id]:\n",
    "        posts_users_votes[post_id][author_id] = copy.deepcopy(list(posts_users_votes[post_id].values())[0])\n",
    "        posts_users_votes[post_id][author_id][\"VOTE\"] = \"upvote\"\n",
    "        posts_users_votes[post_id][author_id][\"USERNAME\"] = user_id_name_map[author_id]\n",
    "    else:\n",
    "        posts_users_votes[post_id][author_id][\"VOTE\"] = \"upvote\"\n",
    "        \n",
    "    group_id = posts_info_map[post_id][\"group_id\"]\n",
    "    group_name = group_id_name_map[group_id]\n",
    "    \n",
    "    post_timestamp = posts_info_map[post_id][\"timestamp\"]\n",
    "    if abs(time.time() - post_timestamp) <= active_member_day_thres * 24 * 60 * 60 and author_id in group_id_members_id_map[group_id]:\n",
    "        group_name_active_member_map[group_name][author_id] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'LOLCHI': '0.4633', 'Trivia': '0.3949', 'Random': '0.4762', 'Team|Llama': '0.5270', 'Team|Mignon': '0.6814', 'The|Positive|Corner': '0.8539', 'Food': '0.6600', 'Out|of|Context|Quotes': '1.1608', 'Team|PhThree': '0.1209', 'Campus|Events': '0.8182', 'Where|Am|I': '0.7012', 'Stanford': '0.0501', 'Stanford|Wordle': '0.0214', 'Cute|Animals': '0.8134', 'Music': '0.6371', 'Bugs': '0.5346', 'Stanford|HCI': '0.0535', 'Amazing|AI': '0.3797', 'World|Cup|2022': '0.2810', 'Hot|Takes': '0.0910'}\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:50 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_posting_frequency_day'], at \u001b[0m\u001b[1;32m<ipython-input-219-d6c74ea5c0bf>:16 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 210.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_posting_frequency_day\u001b[0m\u001b[1;33m dict {...} with 20 keys ['LOLCHI', 'Trivia', 'Random', 'Team|Llama', 'Team|Mignon', 'The|Positive|Corner', 'Food', 'Out|of|Context|Quotes', 'Team|PhThree', 'Campus|Events', 'Where|Am|I', 'Stanford', 'Stanford|Wordle', 'Cute|Animals', 'Music', 'Bugs', 'Stanford|HCI', 'Amazing|AI', 'World|Cup|2022', 'Hot|Takes']\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mLOLCHI\u001b[0m\u001b[1;33m str len 6: 0.4633\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTrivia\u001b[0m\u001b[1;33m str len 6: 0.3949\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mRandom\u001b[0m\u001b[1;33m str len 6: 0.4762\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Llama\u001b[0m\u001b[1;33m str len 6: 0.5270\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|Mignon\u001b[0m\u001b[1;33m str len 6: 0.6814\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mThe|Positive|Corner\u001b[0m\u001b[1;33m str len 6: 0.8539\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mFood\u001b[0m\u001b[1;33m str len 6: 0.6600\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mOut|of|Context|Quotes\u001b[0m\u001b[1;33m str len 6: 1.1608\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mTeam|PhThree\u001b[0m\u001b[1;33m str len 6: 0.1209\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCampus|Events\u001b[0m\u001b[1;33m str len 6: 0.8182\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWhere|Am|I\u001b[0m\u001b[1;33m str len 6: 0.7012\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford\u001b[0m\u001b[1;33m str len 6: 0.0501\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|Wordle\u001b[0m\u001b[1;33m str len 6: 0.0214\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mCute|Animals\u001b[0m\u001b[1;33m str len 6: 0.8134\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mMusic\u001b[0m\u001b[1;33m str len 6: 0.6371\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mBugs\u001b[0m\u001b[1;33m str len 6: 0.5346\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mStanford|HCI\u001b[0m\u001b[1;33m str len 6: 0.0535\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mAmazing|AI\u001b[0m\u001b[1;33m str len 6: 0.3797\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mWorld|Cup|2022\u001b[0m\u001b[1;33m str len 6: 0.2810\u001b[0m\n",
      "\u001b[1;33m    \u001b[0m\u001b[1;36mHot|Takes\u001b[0m\u001b[1;33m str len 6: 0.0910\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:50 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:50 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_active_member_num_1'], at \u001b[0m\u001b[1;32m<ipython-input-219-d6c74ea5c0bf>:17 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 211.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_active_member_num_1\u001b[0m\u001b[1;33m str len 351: {'LOLCHI': 12, 'Trivia': 10, 'Random': 9, 'Team|Llama': 5, 'Team|Mignon': 4, 'The|Positive|Corner': 11, 'Food': 9, 'Out|of|Context|Quotes': 11, 'Team|PhThree': 3, 'Campus|Events': 5, 'Where|Am|I': 7, 'Stanford': 2, 'Stanford|Wordle': 1, 'Cute|Animals': 6, 'Music': 6, 'Bugs': 7, 'Stanford|HCI': 1, 'Amazing|AI': 4, 'World|Cup|2022': 3, 'Hot|Takes': 1}\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:50 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:50 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['group_active_member_num_2'], at \u001b[0m\u001b[1;32m<ipython-input-219-d6c74ea5c0bf>:19 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 212.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mgroup_active_member_num_2\u001b[0m\u001b[1;33m str len 349: {'LOLCHI': 11, 'Trivia': 8, 'Random': 6, 'Team|Llama': 4, 'Team|Mignon': 4, 'The|Positive|Corner': 9, 'Food': 7, 'Out|of|Context|Quotes': 10, 'Team|PhThree': 3, 'Campus|Events': 3, 'Where|Am|I': 6, 'Stanford': 1, 'Stanford|Wordle': 0, 'Cute|Animals': 5, 'Music': 4, 'Bugs': 2, 'Stanford|HCI': 0, 'Amazing|AI': 3, 'World|Cup|2022': 1, 'Hot|Takes': 0}\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:49:50 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "all_votes = []\n",
    "group_name_upvote_num_map = defaultdict(int)\n",
    "for post_id, users_votes in posts_users_votes.items():\n",
    "    for user_id, vote in users_votes.items():\n",
    "        all_votes.append(vote)\n",
    "        if vote[\"VOTE\"] == \"upvote\":\n",
    "            group_name_upvote_num_map[vote[\"SUBREDDIT\"][2:]] += 1\n",
    "# group_name_upvote_frequency_map = {group_name: f\"{(time.time() - group_name_first_post_time_map[group_name])/(60 * 60 * 24 * group_name_upvote_num_map[group_name]):.4f}\" for group_name in group_name_upvote_num_map}\n",
    "# debug(group_upvoteing_frequency_day = group_name_upvote_frequency_map)\n",
    "\n",
    "group_name_upvote_frequency_map = {group_name: (60 * 60 * 24 * group_name_upvote_num_map[group_name])/(time.time() - group_name_first_post_time_map[group_name]) for group_name in group_name_upvote_num_map}\n",
    "print({group_name: f\"{group_name_upvote_frequency_map[group_name]:.4f}\" for group_name in group_name_upvote_frequency_map})\n",
    "debug(group_posting_frequency_day = {group_name: f\"{group_name_upvote_frequency_map[group_name]:.4f}\" for group_name in group_name_upvote_frequency_map})\n",
    "debug(group_active_member_num_1 = str({name: len(members) for name, members in group_name_active_member_map.items()})) # number of members who have at least posted/reacted once\n",
    "group_active_member_1 = {name: [user_id_name_map[member] for member in members if members[member] >=1] for name, members in group_name_active_member_map.items()}\n",
    "debug(group_active_member_num_2 = str({name: len([member for member in members if members[member] >=2]) for name, members in group_name_active_member_map.items()}))\n",
    "group_active_member_2 = {name: [user_id_name_map[member] for member in members if members[member] >=2] for name, members in group_name_active_member_map.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'LOLCHI': '2.7241', 'Trivia': '1.4762', 'Random': '2.3438', 'Team|Llama': '1.1857', 'Team|Mignon': '1.2022', 'The|Positive|Corner': '2.4815', 'Food': '2.6757', 'Out|of|Context|Quotes': '3.0351', 'Team|PhThree': '1.2857', 'Campus|Events': '1.6333', 'Where|Am|I': '1.6800', 'Stanford': '1.0000', 'Stanford|Wordle': '1.0000', 'Cute|Animals': '4.0000', 'Music': '2.0000', 'Bugs': '1.4286', 'Stanford|HCI': '1.0000', 'Amazing|AI': '2.3333', 'World|Cup|2022': '1.0000', 'Hot|Takes': '1.0000'}\n",
      "{'LOLCHI': '0.1048', 'Trivia': '0.0820', 'Random': '0.1302', 'Team|Llama': '0.1694', 'Team|Mignon': '0.1336', 'The|Positive|Corner': '0.1379', 'Food': '0.1115', 'Out|of|Context|Quotes': '0.1597', 'Team|PhThree': '0.1837', 'Campus|Events': '0.2722', 'Where|Am|I': '0.1867', 'Stanford': '0.0909', 'Stanford|Wordle': '0.2500', 'Cute|Animals': '0.4444', 'Music': '0.2222', 'Bugs': '0.1190', 'Stanford|HCI': '0.0417', 'Amazing|AI': '0.3889', 'World|Cup|2022': '0.3333', 'Hot|Takes': '0.3333'}\n"
     ]
    }
   ],
   "source": [
    "group_name_upvote_rate_map = {group_name: (group_name_upvote_num_map[group_name])/(group_name_post_num_map[group_name]) for group_name in group_name_upvote_num_map}\n",
    "print({group_name: f\"{group_name_upvote_rate_map[group_name]:.4f}\" for group_name in group_name_upvote_rate_map})\n",
    "group_name_upvote_rate_per_person_map = {group_name: (group_name_upvote_num_map[group_name])/(group_name_post_num_map[group_name] * group_name_member_num_map[group_name]) for group_name in group_name_upvote_num_map}\n",
    "print({group_name: f\"{group_name_upvote_rate_per_person_map[group_name]:.4f}\" for group_name in group_name_upvote_rate_per_person_map})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set curation parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:50:28 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m at \u001b[0m\u001b[1;32m<ipython-input-223-5d0204c7b3fe>:7 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;36mSaving curators to data/Curio/community_curators.json\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-12-23 18:50:28 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import pymongo\n",
    "community_curators = copy.deepcopy(group_active_member_2)\n",
    "for community in community_curators:\n",
    "    if len(community_curators[community]) == 0:\n",
    "        community_curators[community] = group_active_member_1[community]\n",
    "json.dump(community_curators, open(\"data/Curio/community_curators.json\", \"w\")) # Consider only members who have posted/reacted at least twice as curators\n",
    "debug(\"Saving curators to data/Curio/community_curators.json\")\n",
    "\n",
    "client = pymongo.MongoClient(DB_CONNECTION_STRING)\n",
    "groups_db = client.reverb.groups\n",
    "for community in community_curators:\n",
    "    groups_db.find_one_and_update({\"name\": community.replace(\"|\", \" \")}, {\"$set\": {\"curators\": community_curators[community]}})\n",
    "    groups_db.find_one_and_update({\"name\": community.replace(\"|\", \" \")}, {\"$set\": {\"curationThres\": min(1, 2/len(community_curators[community]) - 0.001)}})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dump posts and votes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "votes_df = pd.DataFrame.from_records(all_votes)\n",
    "posts_df.to_csv(curio_data_path + \"post_info.txt\", sep = \"\\t\", index = False)\n",
    "votes_df.to_csv(curio_data_path + \"all_votes.txt\", sep = \"\\t\", index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.12 ('cr4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "740471d2bc5e6e0b41d12bdc2e64373746aa6a34800f381ff958ff5f02fa0c53"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}