02_clean_texts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from mltb2.text import (\n",
    "    has_invisible_characters,\n",
    "    has_special_whitespaces,\n",
    "    clean_all_invisible_chars_and_whitespaces,\n",
    ")\n",
    "from tqdm import tqdm\n",
    "\n",
    "tqdm.pandas()  # activate tqdm pandas progress bar support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Nekrolog 2022</td>\n",
       "      <td>Dies ist eine Liste von im Jahr 2022 verstorbe...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12052792</td>\n",
       "      <td>12052792</td>\n",
       "      <td>4016.285400</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Deutschland (; Vollform des Staatennamens seit...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>0</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Das heutige Deutschland hat gut 84 Millionen E...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Auf dem Gebiet Deutschlands ist die Anwesenhei...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>2</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Die rasche Entwicklung vom Agrar- zum Industri...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>3</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995892</th>\n",
       "      <td>14995892</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>DB2 komplett in Richtung nativer Unterstützung...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>124</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995893</th>\n",
       "      <td>14995893</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Nachdem sich die eNF-Relationen am Ende bei de...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995894</th>\n",
       "      <td>14995894</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Jim Melton, der federführende Editor aller SQL...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995895</th>\n",
       "      <td>14995895</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Auch wenn sich das ISO-SQL-Gremium nicht dazu ...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995896</th>\n",
       "      <td>14995896</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Die Entwicklung und Implementierung eines so g...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14995897 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                                              title  \\\n",
       "0                0                                      Nekrolog 2022   \n",
       "1                1                                        Deutschland   \n",
       "2                2                                        Deutschland   \n",
       "3                3                                        Deutschland   \n",
       "4                4                                        Deutschland   \n",
       "...            ...                                                ...   \n",
       "14995892  14995892  Advanced Information Management Prototype (AIM-P)   \n",
       "14995893  14995893  Advanced Information Management Prototype (AIM-P)   \n",
       "14995894  14995894  Advanced Information Management Prototype (AIM-P)   \n",
       "14995895  14995895  Advanced Information Management Prototype (AIM-P)   \n",
       "14995896  14995896  Advanced Information Management Prototype (AIM-P)   \n",
       "\n",
       "                                                       text  \\\n",
       "0         Dies ist eine Liste von im Jahr 2022 verstorbe...   \n",
       "1         Deutschland (; Vollform des Staatennamens seit...   \n",
       "2         Das heutige Deutschland hat gut 84 Millionen E...   \n",
       "3         Auf dem Gebiet Deutschlands ist die Anwesenhei...   \n",
       "4         Die rasche Entwicklung vom Agrar- zum Industri...   \n",
       "...                                                     ...   \n",
       "14995892  DB2 komplett in Richtung nativer Unterstützung...   \n",
       "14995893  Nachdem sich die eNF-Relationen am Ende bei de...   \n",
       "14995894  Jim Melton, der federführende Editor aller SQL...   \n",
       "14995895  Auch wenn sich das ISO-SQL-Gremium nicht dazu ...   \n",
       "14995896  Die Entwicklung und Implementierung eines so g...   \n",
       "\n",
       "                                                   url   wiki_id        views  \\\n",
       "0         https://de.wikipedia.org/wiki?curid=12052792  12052792  4016.285400   \n",
       "1          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "2          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "3          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "4          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "...                                                ...       ...          ...   \n",
       "14995892  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995893  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995894  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995895  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995896  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "\n",
       "          paragraph_id  langs  \n",
       "0                    0     38  \n",
       "1                    0    295  \n",
       "2                    1    295  \n",
       "3                    2    295  \n",
       "4                    3    295  \n",
       "...                ...    ...  \n",
       "14995892           124      0  \n",
       "14995893           125      0  \n",
       "14995894           126      0  \n",
       "14995895           127      0  \n",
       "14995896           128      0  \n",
       "\n",
       "[14995897 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle(\"./data/wikipedia-22-12-de-no-emb-raw.pkl.gz\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14995897/14995897 [00:13<00:00, 1143405.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "text\n",
       "False    14985201\n",
       "True        10696\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"text\"].progress_apply(has_invisible_characters).value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14995897/14995897 [00:15<00:00, 941333.71it/s] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "text\n",
       "False    13279760\n",
       "True      1716137\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"text\"].progress_apply(has_special_whitespaces).value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14995897/14995897 [00:08<00:00, 1666471.68it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "title\n",
       "False    14995889\n",
       "True            8\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"title\"].progress_apply(has_invisible_characters).value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14995897/14995897 [00:12<00:00, 1160881.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "title\n",
       "False    14995897\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"title\"].progress_apply(has_special_whitespaces).value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14995897/14995897 [05:40<00:00, 44001.04it/s]\n"
     ]
    }
   ],
   "source": [
    "df[\"text\"] = df[\"text\"].progress_apply(clean_all_invisible_chars_and_whitespaces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14995897/14995897 [00:22<00:00, 668801.29it/s]\n"
     ]
    }
   ],
   "source": [
    "df[\"title\"] = df[\"title\"].progress_apply(clean_all_invisible_chars_and_whitespaces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14995897"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "original_data_len = len(df)\n",
    "original_data_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14876607"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop_duplicates(\"text\", inplace=True, keep=\"first\")\n",
    "new_data_len = len(df)\n",
    "new_data_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Texts droped: 119290\n"
     ]
    }
   ],
   "source": [
    "print(\"Texts droped:\", original_data_len - new_data_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>wiki_id</th>\n",
       "      <th>views</th>\n",
       "      <th>paragraph_id</th>\n",
       "      <th>langs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Nekrolog 2022</td>\n",
       "      <td>Dies ist eine Liste von im Jahr 2022 verstorbe...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12052792</td>\n",
       "      <td>12052792</td>\n",
       "      <td>4016.285400</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Deutschland (; Vollform des Staatennamens seit...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>0</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Das heutige Deutschland hat gut 84 Millionen E...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>1</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Auf dem Gebiet Deutschlands ist die Anwesenhei...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>2</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>Die rasche Entwicklung vom Agrar- zum Industri...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=1497705</td>\n",
       "      <td>1497705</td>\n",
       "      <td>3953.328369</td>\n",
       "      <td>3</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995892</th>\n",
       "      <td>14995892</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>DB2 komplett in Richtung nativer Unterstützung...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>124</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995893</th>\n",
       "      <td>14995893</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Nachdem sich die eNF-Relationen am Ende bei de...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995894</th>\n",
       "      <td>14995894</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Jim Melton, der federführende Editor aller SQL...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995895</th>\n",
       "      <td>14995895</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Auch wenn sich das ISO-SQL-Gremium nicht dazu ...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995896</th>\n",
       "      <td>14995896</td>\n",
       "      <td>Advanced Information Management Prototype (AIM-P)</td>\n",
       "      <td>Die Entwicklung und Implementierung eines so g...</td>\n",
       "      <td>https://de.wikipedia.org/wiki?curid=12485487</td>\n",
       "      <td>12485487</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14876607 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id                                              title  \\\n",
       "0                0                                      Nekrolog 2022   \n",
       "1                1                                        Deutschland   \n",
       "2                2                                        Deutschland   \n",
       "3                3                                        Deutschland   \n",
       "4                4                                        Deutschland   \n",
       "...            ...                                                ...   \n",
       "14995892  14995892  Advanced Information Management Prototype (AIM-P)   \n",
       "14995893  14995893  Advanced Information Management Prototype (AIM-P)   \n",
       "14995894  14995894  Advanced Information Management Prototype (AIM-P)   \n",
       "14995895  14995895  Advanced Information Management Prototype (AIM-P)   \n",
       "14995896  14995896  Advanced Information Management Prototype (AIM-P)   \n",
       "\n",
       "                                                       text  \\\n",
       "0         Dies ist eine Liste von im Jahr 2022 verstorbe...   \n",
       "1         Deutschland (; Vollform des Staatennamens seit...   \n",
       "2         Das heutige Deutschland hat gut 84 Millionen E...   \n",
       "3         Auf dem Gebiet Deutschlands ist die Anwesenhei...   \n",
       "4         Die rasche Entwicklung vom Agrar- zum Industri...   \n",
       "...                                                     ...   \n",
       "14995892  DB2 komplett in Richtung nativer Unterstützung...   \n",
       "14995893  Nachdem sich die eNF-Relationen am Ende bei de...   \n",
       "14995894  Jim Melton, der federführende Editor aller SQL...   \n",
       "14995895  Auch wenn sich das ISO-SQL-Gremium nicht dazu ...   \n",
       "14995896  Die Entwicklung und Implementierung eines so g...   \n",
       "\n",
       "                                                   url   wiki_id        views  \\\n",
       "0         https://de.wikipedia.org/wiki?curid=12052792  12052792  4016.285400   \n",
       "1          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "2          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "3          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "4          https://de.wikipedia.org/wiki?curid=1497705   1497705  3953.328369   \n",
       "...                                                ...       ...          ...   \n",
       "14995892  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995893  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995894  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995895  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "14995896  https://de.wikipedia.org/wiki?curid=12485487  12485487     0.000000   \n",
       "\n",
       "          paragraph_id  langs  \n",
       "0                    0     38  \n",
       "1                    0    295  \n",
       "2                    1    295  \n",
       "3                    2    295  \n",
       "4                    3    295  \n",
       "...                ...    ...  \n",
       "14995892           124      0  \n",
       "14995893           125      0  \n",
       "14995894           126      0  \n",
       "14995895           127      0  \n",
       "14995896           128      0  \n",
       "\n",
       "[14876607 rows x 8 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(\"./data/wikipedia-22-12-de-no-emb-clean.pkl.gz\", compression=\"gzip\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}