04-kNN/06-More-Hyper-Parameters-in-kNN-and-Grid-Search/06-More-Hyper-Parameters-in-kNN-and-Grid-Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 06 网格搜索和更多kNN中的超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9916666666666667"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "sk_knn_clf = KNeighborsClassifier(n_neighbors=4, weights=\"uniform\")\n",
    "sk_knn_clf.fit(X_train, y_train)\n",
    "sk_knn_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_grid = [\n",
    "    {\n",
    "        'weights': ['uniform'], \n",
    "        'n_neighbors': [i for i in range(1, 11)]\n",
    "    },\n",
    "    {\n",
    "        'weights': ['distance'],\n",
    "        'n_neighbors': [i for i in range(1, 11)], \n",
    "        'p': [i for i in range(1, 6)]\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knn_clf = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "grid_search = GridSearchCV(knn_clf, param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 60 candidates, totalling 180 fits\n",
      "CPU times: user 3min 32s, sys: 3.13 s, total: 3min 35s\n",
      "Wall time: 3min 54s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 180 out of 180 | elapsed:  3.9min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid=[{'weights': ['uniform'], 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, {'weights': ['distance'], 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 'p': [1, 2, 3, 4, 5]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=1)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=3, p=3,\n",
       "           weights='distance')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98538622129436326"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 60 candidates, totalling 180 fits\n",
      "[CV] n_neighbors=1, weights=uniform ..................................\n",
      "[CV] n_neighbors=1, weights=uniform ..................................\n",
      "[CV] n_neighbors=1, weights=uniform ..................................\n",
      "[CV] n_neighbors=2, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=1, weights=uniform, total=   0.2s\n",
      "[CV] ................... n_neighbors=1, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=2, weights=uniform ..................................\n",
      "[CV] n_neighbors=2, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=1, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=2, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=2, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=3, weights=uniform, total=   0.1s\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=2, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=3, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=3, weights=uniform, total=   0.3s\n",
      "[CV] ................... n_neighbors=4, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=4, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=4, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=6, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=5, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=6, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=5, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=6, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=5, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=7, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=6, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=7, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=6, weights=uniform, total=   0.3s\n",
      "[CV] n_neighbors=7, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=6, weights=uniform, total=   0.3s\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=7, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=7, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=7, weights=uniform, total=   0.3s\n",
      "[CV] ................... n_neighbors=8, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=8, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] ................... n_neighbors=8, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] ................... n_neighbors=9, weights=uniform, total=   0.1s\n",
      "[CV] ................... n_neighbors=9, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] ................... n_neighbors=9, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=1, p=1, weights=distance ............................\n",
      "[CV] .................. n_neighbors=10, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=1, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=1, p=1, weights=distance ............................\n",
      "[CV] .................. n_neighbors=10, weights=uniform, total=   0.2s\n",
      "[CV] n_neighbors=1, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=1, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=1, weights=distance, total=   0.2s\n",
      "[CV] .................. n_neighbors=10, weights=uniform, total=   0.3s\n",
      "[CV] n_neighbors=1, p=2, weights=distance ............................\n",
      "[CV] n_neighbors=1, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=1, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=1, p=3, weights=distance ............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    4.7s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............. n_neighbors=1, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=1, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=3, weights=distance, total=   1.1s\n",
      "[CV] n_neighbors=1, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=3, weights=distance, total=   1.3s\n",
      "[CV] n_neighbors=1, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=3, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=1, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=4, weights=distance, total=   1.3s\n",
      "[CV] n_neighbors=1, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=4, weights=distance, total=   1.2s\n",
      "[CV] n_neighbors=1, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=4, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=2, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=5, weights=distance, total=   1.3s\n",
      "[CV] ............. n_neighbors=2, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=2, p=1, weights=distance ............................\n",
      "[CV] n_neighbors=2, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=5, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=2, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=2, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=2, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=2, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=2, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=1, p=5, weights=distance, total=   1.1s\n",
      "[CV] n_neighbors=2, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=2, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=2, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=3, weights=distance, total=   1.3s\n",
      "[CV] n_neighbors=2, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=2, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=3, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=2, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=4, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=2, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=4, weights=distance, total=   1.2s\n",
      "[CV] n_neighbors=2, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=4, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=3, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=5, weights=distance, total=   1.3s\n",
      "[CV] n_neighbors=3, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=5, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=3, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=1, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=3, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=3, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=3, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=3, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=3, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=3, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=2, p=5, weights=distance, total=   1.3s\n",
      "[CV] n_neighbors=3, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=3, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=3, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=3, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=3, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=3, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=4, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=3, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=4, weights=distance, total=   1.2s\n",
      "[CV] n_neighbors=3, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=4, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=4, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=5, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=4, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=1, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=4, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=1, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=4, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=4, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=2, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=4, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=4, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=4, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=5, weights=distance, total=   1.3s\n",
      "[CV] n_neighbors=4, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=3, p=5, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=4, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=4, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=3, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=4, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=4, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=4, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=4, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=4, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=4, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=4, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=5, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=5, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=5, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=5, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=5, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=5, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=5, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=5, p=3, weights=distance ............................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............. n_neighbors=4, p=5, weights=distance, total=   1.4s\n",
      "[CV] n_neighbors=5, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=2, weights=distance, total=   0.3s\n",
      "[CV] n_neighbors=5, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=4, p=5, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=5, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=3, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=5, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=3, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=5, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=3, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=5, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=4, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=5, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=4, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=5, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=4, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=6, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=5, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=6, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=6, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=1, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=6, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=6, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=6, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=6, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=6, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=5, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=6, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=5, p=5, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=6, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=3, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=6, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=3, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=6, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=6, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=4, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=6, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=4, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=6, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=4, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=7, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=5, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=7, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=7, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=7, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=7, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=2, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=7, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=7, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=2, weights=distance, total=   0.3s\n",
      "[CV] n_neighbors=7, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=5, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=7, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=6, p=5, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=7, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=3, weights=distance, total=   2.4s\n",
      "[CV] n_neighbors=7, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=3, weights=distance, total=   2.3s\n",
      "[CV] n_neighbors=7, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=3, weights=distance, total=   2.2s\n",
      "[CV] n_neighbors=7, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=4, weights=distance, total=   2.7s\n",
      "[CV] n_neighbors=7, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=4, weights=distance, total=   2.0s\n",
      "[CV] n_neighbors=7, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=4, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=8, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=5, weights=distance, total=   2.0s\n",
      "[CV] n_neighbors=8, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=8, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=8, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=8, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=8, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=8, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=8, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=5, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=8, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=7, p=5, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=8, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=3, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=8, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=8, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=8, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=4, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=8, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=4, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=8, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=4, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=9, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=9, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=5, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=9, p=1, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=9, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=1, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=9, p=2, weights=distance ............................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............. n_neighbors=9, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=9, p=2, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=2, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=9, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=9, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=5, weights=distance, total=   1.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:  1.5min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] n_neighbors=9, p=3, weights=distance ............................\n",
      "[CV] ............. n_neighbors=8, p=5, weights=distance, total=   1.8s\n",
      "[CV] n_neighbors=9, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=3, weights=distance, total=   1.7s\n",
      "[CV] n_neighbors=9, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=9, p=4, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=9, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=4, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=9, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=4, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=9, p=5, weights=distance ............................\n",
      "[CV] ............. n_neighbors=9, p=4, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=10, p=1, weights=distance ...........................\n",
      "[CV] ............. n_neighbors=9, p=5, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=10, p=1, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=1, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=10, p=1, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=1, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=10, p=2, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=1, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=10, p=2, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=10, p=2, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=2, weights=distance, total=   0.1s\n",
      "[CV] n_neighbors=10, p=3, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=2, weights=distance, total=   0.2s\n",
      "[CV] n_neighbors=10, p=3, weights=distance ...........................\n",
      "[CV] ............. n_neighbors=9, p=5, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=10, p=3, weights=distance ...........................\n",
      "[CV] ............. n_neighbors=9, p=5, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=10, p=4, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=10, p=4, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=10, p=4, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=3, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=10, p=5, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=4, weights=distance, total=   1.5s\n",
      "[CV] n_neighbors=10, p=5, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=4, weights=distance, total=   1.6s\n",
      "[CV] n_neighbors=10, p=5, weights=distance ...........................\n",
      "[CV] ............ n_neighbors=10, p=4, weights=distance, total=   1.5s\n",
      "[CV] ............ n_neighbors=10, p=5, weights=distance, total=   1.6s\n",
      "[CV] ............ n_neighbors=10, p=5, weights=distance, total=   1.5s\n",
      "[CV] ............ n_neighbors=10, p=5, weights=distance, total=   1.2s\n",
      "CPU times: user 805 ms, sys: 304 ms, total: 1.11 s\n",
      "Wall time: 1min 54s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 180 out of 180 | elapsed:  1.9min finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "grid_search = GridSearchCV(knn_clf, param_grid, n_jobs=-1, verbose=2)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 其他超参数\n",
    "\n",
    "metrics: [http://scikit-learn.org/stable/modules/generated/sklearn.neighbors.DistanceMetric.html](http://scikit-learn.org/stable/modules/generated/sklearn.neighbors.DistanceMetric.html)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}