runoob_crawl_for_download_android.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys,re,os\n",
    "import time\n",
    "from PyPDF2 import utils, PdfFileReader, PdfFileWriter\n",
    "\n",
    "from urllib.request import urlopen#用于获取网页\n",
    "from bs4 import BeautifulSoup#用于解析网页\n",
    "import collections\n",
    "\n",
    "import subprocess\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def crawl(crawl_url):\n",
    "    html = urlopen(crawl_url)\n",
    "    bsObj = BeautifulSoup(html, 'html.parser')\n",
    "    left_col = bsObj.find_all(name='div',attrs={\"class\":\"listcol last right-column\"})\n",
    "\n",
    "    # print(left_col)\n",
    "\n",
    "    html_list = left_col[0].find_all(name='a')\n",
    "    html_dict = collections.OrderedDict()\n",
    "    for html in html_list:\n",
    "        if not html['href'].startswith('http'):\n",
    "            val = \"/\".join(crawl_url.split(\"/\")[0:-1]) + \"/\" + html['href']\n",
    "            html_dict[html.string.strip()] = val\n",
    "        else:\n",
    "            html_dict[html.string.strip()] = html['href']\n",
    "\n",
    "    return html_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def html2pdf(dic):\n",
    "    for ite in dic:\n",
    "        pdf_rename = ite.replace(\"`\",\"_\").replace(\"~\",\"_\").replace(\"!\",\"_\").replace(\"@\",\"_\").replace(\"#\",\"_\").replace(\"$\",\"_\").replace(\"%\",\"_\").replace(\"^\",\"_\").replace(\"&\",\"_\").replace(\"*\",\"_\").replace(\"(\",\"_\").replace(\")\",\"_\").replace(\"-\",\"_\").replace(\"=\",\"_\").replace(\")\",\"_\").replace(\"+\",\"_\").replace(\"[\",\"_\").replace(\"]\",\"_\").replace(\"\\\\\",\"_\").replace(\"{\",\"_\").replace(\"}\",\"_\").replace(\"|\",\"_\").replace(\";\",\"_\").replace(\"'\",\"_\").replace(\":\",\"_\").replace(\"\\\"\",\"_\").replace(\"<\",\"_\").replace(\">\",\"_\").replace(\"?\",\"_\").replace(\",\",\"_\").replace(\".\",\"_\").replace(\"/\",\"_\").replace(\" \",\"_\")\n",
    "        output = subprocess.call(\"sh html2pdf.sh \\'%s\\' \\'%s\\'\" %(dic[ite],pdf_rename),shell=True)\n",
    "        if output != 0:\n",
    "            print(\"Error in transfer html to pdf\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_pdfs(dic,chapter_name):\n",
    "    filenames = []\n",
    "    for ite in dic:\n",
    "        pdf_rename = \"./pdfs/\" + ite.replace(\"`\",\"_\").replace(\"~\",\"_\").replace(\"!\",\"_\").replace(\"@\",\"_\").replace(\"#\",\"_\").replace(\"$\",\"_\").replace(\"%\",\"_\").replace(\"^\",\"_\").replace(\"&\",\"_\").replace(\"*\",\"_\").replace(\"(\",\"_\").replace(\")\",\"_\").replace(\"-\",\"_\").replace(\"=\",\"_\").replace(\")\",\"_\").replace(\"+\",\"_\").replace(\"[\",\"_\").replace(\"]\",\"_\").replace(\"\\\\\",\"_\").replace(\"{\",\"_\").replace(\"}\",\"_\").replace(\"|\",\"_\").replace(\";\",\"_\").replace(\"'\",\"_\").replace(\":\",\"_\").replace(\"\\\"\",\"_\").replace(\"<\",\"_\").replace(\">\",\"_\").replace(\"?\",\"_\").replace(\",\",\"_\").replace(\".\",\"_\").replace(\"/\",\"_\").replace(\" \",\"_\") + \".pdf\"\n",
    "        filenames.append(pdf_rename)\n",
    "    merge_pdf_core(chapter_name,filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original author Nicholas Kim, modified by Yan Pashkovsky\n",
    "# New license - GPL v3\n",
    "def merge_pdf_core(chapter_name,filenames):\n",
    "#     options, filenames = options,filenames\n",
    "#     print(options, filenames)\n",
    "    output_pdf_name = chapter_name + \".pdf\"\n",
    "    files_to_merge = []\n",
    "\n",
    "    # get PDF files\n",
    "    for f in filenames:\n",
    "        try:\n",
    "            next_pdf_file = PdfFileReader(open(f, \"rb\"))\n",
    "        except(utils.PdfReadError):\n",
    "            print >>sys.stderr, \"%s is not a valid PDF file.\" % f\n",
    "            sys.exit(1)\n",
    "        except(IOError):\n",
    "            print >>sys.stderr, \"%s could not be found.\" % f\n",
    "            sys.exit(1)\n",
    "        else:\n",
    "            files_to_merge.append(next_pdf_file)\n",
    "\n",
    "    # merge page by page\n",
    "    output_pdf_stream = PdfFileWriter()\n",
    "    j=0\n",
    "    k=0\n",
    "    for f in files_to_merge:\n",
    "        for i in range(f.numPages):\n",
    "            output_pdf_stream.addPage(f.getPage(i))\n",
    "            if i==0:\n",
    "                output_pdf_stream.addBookmark(str(filenames[k]).split(\"/\")[-1].split(\".\")[-2],j)\n",
    "            j = j + 1\n",
    "        k += 1\n",
    "\n",
    "    # create output pdf file\n",
    "    try:\n",
    "        output_pdf_file = open(output_pdf_name, \"wb\")\n",
    "        output_pdf_stream.write(output_pdf_file)\n",
    "    finally:\n",
    "        output_pdf_file.close()\n",
    "\n",
    "    # print \"%s successfully created.\" % output_pdf_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_and_merge(URL,chapter_name):\n",
    "    DIC = crawl(URL)\n",
    "    html2pdf(DIC)\n",
    "    merge_pdfs(DIC,chapter_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get_and_merge(\"http://www.runoob.com/memcached/memcached-tutorial.html\",'【学习 memcached】')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    url =\"http://www.runoob.com/\"\n",
    "    html = urlopen(url)\n",
    "    bsObj = BeautifulSoup(html, 'html.parser')\n",
    "    pageClassPattern = re.compile(\"codelist codelist-desktop cate\\d\")\n",
    "    middle_col = bsObj.find_all(name='div',attrs={\"class\":pageClassPattern})\n",
    "\n",
    "    dic_total = collections.OrderedDict()\n",
    "    for iii in middle_col:\n",
    "        catalog = str(iii.h2).split(\"</i>\")[-1].split(\"</h2>\")[0].strip().replace(\"/\", \"_\").replace(\"$\", \"_\").replace(\" \", \"_\").replace(\"(\", \"_\").replace(\")\", \"_\")\n",
    "        html_list = iii.find_all(name='a')\n",
    "        dd = collections.OrderedDict()\n",
    "        for ii in html_list:\n",
    "            dd[ii.h4.string.strip()] = \"http:\" + ii['href']\n",
    "        dic_total[catalog] = dd\n",
    "\n",
    "    for i in dic_total:\n",
    "        FILE_PATH = \"./runoob/%s\"%(i)\n",
    "        if os.path.exists(FILE_PATH): \n",
    "            print('dir exists')\n",
    "        else:\n",
    "            os.makedirs(FILE_PATH)\n",
    "\n",
    "    for fold in dic_total:\n",
    "        for link in dic_total[fold]:\n",
    "            get_and_merge(dic_total[fold][link],link)\n",
    "            print(\"finish one\")\n",
    "            shutil.move(\"%s.pdf\"%(link),\"./runoob/%s\"%(fold))\n",
    "    print(\"All runoob was downloaded !\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-67-972361fa1b80>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"__main__\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-66-c85cb32a3266>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mfold\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdic_total\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdic_total\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfold\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m             \u001b[0mget_and_merge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdic_total\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfold\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"finish one\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s.pdf\"\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"./runoob/%s\"\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-62-0ae8b8b5a1b2>\u001b[0m in \u001b[0;36mget_and_merge\u001b[0;34m(URL, chapter_name)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m#     chapter_name = '【学习 Numpy】'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mDIC\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcrawl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mURL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mhtml2pdf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mmerge_pdfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDIC\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mchapter_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-59-e0520517f7b2>\u001b[0m in \u001b[0;36mhtml2pdf\u001b[0;34m(dic)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mite\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdic\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0mpdf_rename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mite\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"$\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\" \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"(\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\")\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'sh html2pdf.sh %s %s'\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdic\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mite\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpdf_rename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error in transfer html to pdf\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(timeout, *popenargs, **kwargs)\u001b[0m\n\u001b[1;32m    267\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpopenargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 269\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    270\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    271\u001b[0m             \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout, endtime)\u001b[0m\n\u001b[1;32m   1455\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1456\u001b[0m                             \u001b[0;32mbreak\u001b[0m  \u001b[0;31m# Another thread waited.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1457\u001b[0;31m                         \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1458\u001b[0m                         \u001b[0;31m# Check the pid and loop as waitpid has been known to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1459\u001b[0m                         \u001b[0;31m# return 0 even without WNOHANG in odd situations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m_try_wait\u001b[0;34m(self, wait_flags)\u001b[0m\n\u001b[1;32m   1402\u001b[0m             \u001b[0;34m\"\"\"All callers to this function MUST hold self._waitpid_lock.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1403\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1404\u001b[0;31m                 \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait_flags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1405\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mChildProcessError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1406\u001b[0m                 \u001b[0;31m# This happens if SIGCLD is set to be ignored or waiting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-1b741f58423c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_and_merge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://www.runoob.com/w3cnote/android-tutorial-intro.html'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'【学习 Android】'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-16-a7b7713a18ac>\u001b[0m in \u001b[0;36mget_and_merge\u001b[0;34m(URL, chapter_name)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_and_merge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mURL\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mchapter_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mDIC\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcrawl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mURL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mhtml2pdf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mmerge_pdfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDIC\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mchapter_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-1dfc04770579>\u001b[0m in \u001b[0;36mhtml2pdf\u001b[0;34m(dic)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mite\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdic\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0mpdf_rename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mite\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"`\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"~\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"!\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"@\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"#\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"$\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"^\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"&\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"*\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"(\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\")\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"-\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"=\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\")\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"+\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"[\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"]\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\\\\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"}\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"|\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\";\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"'\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\":\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\\"\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"<\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\">\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"?\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\",\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\".\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\" \"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"sh html2pdf.sh \\'%s\\' \\'%s\\'\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdic\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mite\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpdf_rename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error in transfer html to pdf\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(timeout, *popenargs, **kwargs)\u001b[0m\n\u001b[1;32m    267\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpopenargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 269\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    270\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    271\u001b[0m             \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout, endtime)\u001b[0m\n\u001b[1;32m   1455\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1456\u001b[0m                             \u001b[0;32mbreak\u001b[0m  \u001b[0;31m# Another thread waited.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1457\u001b[0;31m                         \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1458\u001b[0m                         \u001b[0;31m# Check the pid and loop as waitpid has been known to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1459\u001b[0m                         \u001b[0;31m# return 0 even without WNOHANG in odd situations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m_try_wait\u001b[0;34m(self, wait_flags)\u001b[0m\n\u001b[1;32m   1402\u001b[0m             \u001b[0;34m\"\"\"All callers to this function MUST hold self._waitpid_lock.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1403\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1404\u001b[0;31m                 \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait_flags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1405\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mChildProcessError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1406\u001b[0m                 \u001b[0;31m# This happens if SIGCLD is set to be ignored or waiting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "get_and_merge('http://www.runoob.com/w3cnote/android-tutorial-intro.html','【学习 Android】')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ll = os.listdir(\"./pdfs_android/\")\n",
    "# new = sorted(ll,key = lambda s:(int(s.split(\"_\")[0]),int(s.split(\"_\")[1]),int(s.split(\"_\")[2])))\n",
    "def get_file_list(file_path):\n",
    "    dir_list = os.listdir(file_path)\n",
    "    if not dir_list:\n",
    "        return\n",
    "    else:\n",
    "        # 注意，这里使用lambda表达式，将文件按照最后修改时间顺序升序排列\n",
    "        # os.path.getmtime() 函数是获取文件最后修改时间\n",
    "        # os.path.getctime() 函数是获取文件最后创建时间\n",
    "        dir_list = sorted(dir_list,  key=lambda x: os.path.getmtime(os.path.join(file_path, x)))\n",
    "        # print(dir_list)\n",
    "        return dir_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "lll = get_file_list(\"./pdfs_android/\")\n",
    "ll = []\n",
    "for i in lll:\n",
    "    ll.append('./pdfs_android/'+i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_pdf_core(\"【学习 Android】\",ll)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}