Conda-tools.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ryan\\Miniconda3\\envs\n",
      "C:\\Users\\Ryan\\Miniconda3\\pkgs\n"
     ]
    }
   ],
   "source": [
    "from conda_tools import (cache, environment)\n",
    "from conda_tools import environment_utils as eu\n",
    "from conda_tools import cache_utils as cu\n",
    "\n",
    "import os\n",
    "from os.path import join\n",
    "from itertools import groupby, chain\n",
    "from versio.version import Version\n",
    "\n",
    "# adjust root to be your Miniconda prefix\n",
    "root = r\"C:\\Users\\Ryan\\Miniconda3\"\n",
    "root_envs = join(root, 'envs')\n",
    "root_pkgs = join(root, 'pkgs')\n",
    "print(root_envs)\n",
    "print(root_pkgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The two core components of the conda ecosystem are the package cache and the environment subfolders.  These are abstracted with `PackageInfo` and `Environment` objects respectively.\n",
    "\n",
    "Here we create \"pools\" of `PackageInfo` and `Environment` objects.  These objects permit easy, read-only access to various bits of metadata stored in the package cache and conda-meta/ subfolders in the environment.  We want to reuse the objects as much as we can to minimize disk I/O.  All the disk reads are currently cached with the objects, so the more objects you work with, the more RAM will be required."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\alabaster-0.7.8-py35_0) @ 0x13dfd352828, PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\anaconda-client-1.4.0-py35_0) @ 0x13dfd3527b8, PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\babel-2.3.3-py35_0) @ 0x13dfd352860, PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\backports-1.0-py27_0) @ 0x13dfd3526d8, PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\backports-1.0-py35_0) @ 0x13dfd3527f0)\n",
      "\n",
      "(Environment(C:\\Users\\Ryan\\Miniconda3) @ 0x13dfd39aeb8, Environment(C:\\Users\\Ryan\\Miniconda3\\envs\\adamimage) @ 0x13dfd39afd0, Environment(C:\\Users\\Ryan\\Miniconda3\\envs\\ct_test1) @ 0x13dfd389080, Environment(C:\\Users\\Ryan\\Miniconda3\\envs\\ct_test2) @ 0x13dfd389128, Environment(C:\\Users\\Ryan\\Miniconda3\\envs\\ct_test3) @ 0x13dfd389198)\n"
     ]
    }
   ],
   "source": [
    "# Create pkg_cache and environments\n",
    "pkg_cache = cache.packages(root_pkgs)\n",
    "envs = environment.environments(root_envs)\n",
    "print(pkg_cache[:5])\n",
    "print()\n",
    "print(envs[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Packages\n",
    "Conda packages all have an info/ subdirectory for storing metadata about the package.  PackageInfo provide convenient access to this metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'arch': 'x86_64',\n",
       " 'build': 'py35_0',\n",
       " 'build_number': 0,\n",
       " 'depends': ['python 3.5*'],\n",
       " 'license': 'BSD',\n",
       " 'name': 'alabaster',\n",
       " 'platform': 'win',\n",
       " 'subdir': 'win-64',\n",
       " 'version': '0.7.8'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pi = pkg_cache[0]\n",
    "pi.index  # info/index.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('alabaster', '0.7.8', 'py35_0')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can access fields of index.json directly from the object.\n",
    "pi.name, pi.version, pi.build"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "frozenset({'Lib/site-packages/alabaster-0.7.8-py3.5.egg-info/PKG-INFO',\n",
       "           'Lib/site-packages/alabaster-0.7.8-py3.5.egg-info/SOURCES.txt',\n",
       "           'Lib/site-packages/alabaster-0.7.8-py3.5.egg-info/dependency_links.txt',\n",
       "           'Lib/site-packages/alabaster-0.7.8-py3.5.egg-info/pbr.json',\n",
       "           'Lib/site-packages/alabaster-0.7.8-py3.5.egg-info/top_level.txt',\n",
       "           'Lib/site-packages/alabaster/__init__.py',\n",
       "           'Lib/site-packages/alabaster/__pycache__/__init__.cpython-35.pyc',\n",
       "           'Lib/site-packages/alabaster/__pycache__/_version.cpython-35.pyc',\n",
       "           'Lib/site-packages/alabaster/__pycache__/support.cpython-35.pyc',\n",
       "           'Lib/site-packages/alabaster/_version.py',\n",
       "           'Lib/site-packages/alabaster/about.html',\n",
       "           'Lib/site-packages/alabaster/donate.html',\n",
       "           'Lib/site-packages/alabaster/layout.html',\n",
       "           'Lib/site-packages/alabaster/navigation.html',\n",
       "           'Lib/site-packages/alabaster/relations.html',\n",
       "           'Lib/site-packages/alabaster/static/alabaster.css_t',\n",
       "           'Lib/site-packages/alabaster/static/custom.css',\n",
       "           'Lib/site-packages/alabaster/support.py',\n",
       "           'Lib/site-packages/alabaster/theme.conf'})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Access to info/files\n",
    "pi.files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'alabaster-0.7.8-py35_0'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The full spec of the package.  This is always \"name-version-build\"\n",
    "pi.full_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'entrypoints-0.2-py35_1': 'MIT',\n",
       " 'entrypoints-0.2.2-py35_0': 'MIT',\n",
       " 'get_terminal_size-1.0.0-py27_0': 'MIT',\n",
       " 'get_terminal_size-1.0.0-py35_0': 'MIT',\n",
       " 'imagesize-0.7.1-py35_0': 'MIT',\n",
       " 'jsonschema-2.5.1-py35_0': 'MIT',\n",
       " 'nodejs-4.4.1-0': 'MIT',\n",
       " 'path.py-8.2.1-py27_0': 'MIT',\n",
       " 'path.py-8.2.1-py35_0': 'MIT',\n",
       " 'pathlib2-2.1.0-py27_0': 'MIT',\n",
       " 'pickleshare-0.7.2-py27_0': 'MIT',\n",
       " 'pickleshare-0.7.2-py35_0': 'MIT',\n",
       " 'pip-8.1.1-py35_1': 'MIT',\n",
       " 'pip-8.1.2-py27_0': 'MIT',\n",
       " 'pip-8.1.2-py34_0': 'MIT',\n",
       " 'pip-8.1.2-py35_0': 'MIT',\n",
       " 'py-1.4.31-py35_0': 'MIT',\n",
       " 'pyparsing-2.1.4-py35_0': 'MIT',\n",
       " 'pytest-2.9.2-py35_0': 'MIT',\n",
       " 'pytz-2016.4-py35_0': 'MIT',\n",
       " 'ruamel_yaml-0.11.7-py35_0': 'MIT',\n",
       " 'simplejson-3.8.2-py35_0': 'MIT',\n",
       " 'six-1.10.0-py27_0': 'MIT',\n",
       " 'six-1.10.0-py35_0': 'MIT',\n",
       " 'six-1.9.0-py35_0': 'MIT',\n",
       " 'wcwidth-0.1.7-py35_0': 'MIT',\n",
       " 'wheel-0.29.0-py27_0': 'MIT',\n",
       " 'wheel-0.29.0-py34_0': 'MIT',\n",
       " 'wheel-0.29.0-py35_0': 'MIT',\n",
       " 'win_unicode_console-0.5-py35_0': 'MIT'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can queries against the information we have on packages\n",
    "# For example, I want to find all MIT licensed packages in the cache\n",
    "{pi.full_spec: pi.license for pi in pkg_cache if pi.license == 'MIT'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Environments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Environment(C:\\Users\\Ryan\\Miniconda3\\envs\\ct_test1) @ 0x13dfd389080"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e = envs[2]\n",
    "e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C:\\\\Users\\\\Ryan\\\\Miniconda3': True,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\_build': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\adamimage': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\ct_test1': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\ct_test2': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\ct_test3': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\env1': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\env2': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\ftest': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\imagep': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\jupyterlab-dev': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\m1': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\old1': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\pipct': False,\n",
       " 'C:\\\\Users\\\\Ryan\\\\Miniconda3\\\\envs\\\\py27': False}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can discover the currently activated environment\n",
    "{e.path: e.activated() for e in envs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pip': PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\pip-8.1.2-py35_0) @ 0x13dfd389860,\n",
       " 'python': PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\python-3.5.1-5) @ 0x13dfd389b70,\n",
       " 'setuptools': PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\setuptools-23.0.0-py35_0) @ 0x13dfd389c88,\n",
       " 'ujson': PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\ujson-1.35-py35_0) @ 0x13dfd389ba8,\n",
       " 'vs2015_runtime': PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\vs2015_runtime-14.0.25123-0) @ 0x13dfd3db5f8,\n",
       " 'wget': PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\wget-2.2-py35_0) @ 0x13dfd389940,\n",
       " 'wheel': PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\wheel-0.29.0-py35_0) @ 0x13dfd46b7f0}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can see all the packages that claim to be linked into the environment, keyed by name\n",
    "e.linked_packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# linked packages are either hard-linked, symlinked, or copied into environments.\n",
    "set(chain(e.hard_linked, e.soft_linked, e.copy_linked)) ^ set(e.linked_packages.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pip': 'https://repo.continuum.io/pkgs/free/win-64',\n",
       " 'python': 'https://repo.continuum.io/pkgs/free/win-64',\n",
       " 'setuptools': 'https://repo.continuum.io/pkgs/free/win-64',\n",
       " 'ujson': 'https://repo.continuum.io/pkgs/free/win-64',\n",
       " 'vs2015_runtime': 'https://repo.continuum.io/pkgs/free/win-64',\n",
       " 'wget': 'https://repo.continuum.io/pkgs/free/win-64',\n",
       " 'wheel': 'https://repo.continuum.io/pkgs/free/win-64'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The origin channel of each package\n",
    "e.package_channels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'date': '2016-07-01 20:52:32',\n",
       "  'downgrade': [],\n",
       "  'install': ['pip-8.1.2-py35_0',\n",
       "   'python-3.5.1-5',\n",
       "   'setuptools-23.0.0-py35_0',\n",
       "   'ujson-1.35-py35_0',\n",
       "   'vs2015_runtime-14.0.25123-0',\n",
       "   'wget-2.2-py35_0',\n",
       "   'wheel-0.29.0-py35_0'],\n",
       "  'remove': [],\n",
       "  'rev': 0,\n",
       "  'upgrade': []}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We also have access to the history of the environment.\n",
    "# The history object is an adaptation of conda's history parser.\n",
    "# (note: The interface to this may change in the future)\n",
    "e.history.object_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neat stuff\n",
    "Convenient access to the package cache and environment metadata allows you to do some neat stuff relatively easily.\n",
    "\n",
    "Below are a few examples of some quick ideas that can be implemented with little effort."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hdf4-4.2.11-0 collides with hdf5-1.8.15.1-2\n",
      "\tCollisions:  frozenset({'Library/COPYING', 'Library/RELEASE.txt'})\n",
      "notebook-4.2.0-py35_0 collides with qtconsole-4.2.1-py35_0\n",
      "\tCollisions:  frozenset({'Menu/jupyter.ico'})\n",
      "notebook-4.2.1-py35_0 collides with qtconsole-4.2.1-py35_0\n",
      "\tCollisions:  frozenset({'Menu/jupyter.ico'})\n"
     ]
    }
   ],
   "source": [
    "# Calculate potential collisions in environments by packages claiming the same file paths\n",
    "# Very quick and naive way of detecting file path collisions.\n",
    "for i, p1 in enumerate(pkg_cache):\n",
    "    for p2 in pkg_cache[i+1:]:\n",
    "        if p1.name == p2.name:\n",
    "            continue\n",
    "        x = p1.files.intersection(p2.files)\n",
    "        if x:\n",
    "            print(\"{} collides with {}\".format(p1, p2))\n",
    "            print(\"\\tCollisions: \", x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\alabaster-0.7.8-py35_0) @ 0x13dfd352828: (Environment(C:\\Users\\Ryan\\Miniconda3) @ 0x13dfd39aeb8,)}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cache Utils has some higher order, convenience functions\n",
    "\n",
    "# See what environments a package is linked into\n",
    "# Note that this is a O(n) operation where n is the sum of the installed packages in each environment you're checking.\n",
    "# If you're running this for the first time, it has to read all the metadata for each environment.\n",
    "# Also note, that this creates new package info objects and environment objects each run, so each run\n",
    "# prompts a full scan of both the package cache and all environments.\n",
    "cu.linked_environments((pkg_cache[0],), envs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Skipping  jpeg 8d\n",
      "Skipping  jpeg 8d\n",
      "Skipping  openssl 1.0.2h\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'backports-1.0-py27_0': ['Environment: py27'],\n",
       " 'backports-1.0-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'conda-4.0.7-py35_0': [],\n",
       " 'conda-4.0.8-py35_0': [],\n",
       " 'conda-4.1.2-py35_0': [],\n",
       " 'conda-4.1.6-py35_0': [],\n",
       " 'conda-build-1.20.3-py35_0': [],\n",
       " 'conda-build-1.21.5-py35_0': [],\n",
       " 'conda-env-2.5.0-py35_0': [],\n",
       " 'conda-env-2.5.1-py35_0': [],\n",
       " 'decorator-4.0.10-py27_0': ['Environment: py27'],\n",
       " 'decorator-4.0.10-py35_0': ['Environment: env1',\n",
       "  'Environment: ftest',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'decorator-4.0.9-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env2',\n",
       "  'Environment: imagep'],\n",
       " 'entrypoints-0.2-py35_1': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: imagep'],\n",
       " 'get_terminal_size-1.0.0-py27_0': ['Environment: py27'],\n",
       " 'get_terminal_size-1.0.0-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'ipython-4.2.0-py27_0': ['Environment: py27'],\n",
       " 'ipython-4.2.0-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep'],\n",
       " 'ipython_genutils-0.1.0-py27_0': ['Environment: py27'],\n",
       " 'ipython_genutils-0.1.0-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'jupyter_client-4.2.2-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: imagep'],\n",
       " 'notebook-4.2.0-py35_0': ['Environment: Miniconda3'],\n",
       " 'numpy-1.10.4-py35_2': ['Environment: old1'],\n",
       " 'numpy-1.11.0-py35_1': ['Environment: adamimage',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep'],\n",
       " 'numpy-1.11.1-py27_0': ['Environment: py27'],\n",
       " 'numpy-1.11.1-py35_0': ['Environment: env1'],\n",
       " 'path.py-8.2.1-py27_0': ['Environment: py27'],\n",
       " 'path.py-8.2.1-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'pickleshare-0.7.2-py27_0': ['Environment: py27'],\n",
       " 'pickleshare-0.7.2-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'pillow-3.2.0-py35_0': ['Environment: adamimage', 'Environment: imagep'],\n",
       " 'pillow-3.2.0-py35_1': ['Environment: env1'],\n",
       " 'pip-8.1.1-py35_1': ['Environment: env1', 'Environment: env2'],\n",
       " 'pip-8.1.2-py27_0': ['Environment: pipct',\n",
       "  'Environment: py27',\n",
       "  'Environment: _build'],\n",
       " 'pip-8.1.2-py34_0': [],\n",
       " 'pip-8.1.2-py35_0': ['Environment: adamimage',\n",
       "  'Environment: ct_test1',\n",
       "  'Environment: ct_test2',\n",
       "  'Environment: ct_test3',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev',\n",
       "  'Environment: m1',\n",
       "  'Environment: old1'],\n",
       " 'pyreadline-2.1-py27_0': ['Environment: py27'],\n",
       " 'pyreadline-2.1-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'python-2.7.12-0': ['Environment: pipct',\n",
       "  'Environment: py27',\n",
       "  'Environment: _build'],\n",
       " 'python-3.4.5-0': [],\n",
       " 'python-3.5.1-4': ['Environment: adamimage',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep'],\n",
       " 'python-3.5.1-5': ['Environment: Miniconda3',\n",
       "  'Environment: ct_test1',\n",
       "  'Environment: ct_test2',\n",
       "  'Environment: ct_test3',\n",
       "  'Environment: env1',\n",
       "  'Environment: old1'],\n",
       " 'scikit-image-0.12.3-np111py35_0': ['Environment: adamimage',\n",
       "  'Environment: imagep'],\n",
       " 'scikit-image-0.12.3-np111py35_1': ['Environment: env1'],\n",
       " 'scipy-0.17.1-np111py27_1': ['Environment: py27'],\n",
       " 'scipy-0.17.1-np111py35_0': ['Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep'],\n",
       " 'setuptools-21.2.1-py35_0': ['Environment: env1'],\n",
       " 'setuptools-22.0.5-py35_0': ['Environment: adamimage',\n",
       "  'Environment: env2',\n",
       "  'Environment: imagep'],\n",
       " 'setuptools-23.0.0-py27_0': ['Environment: pipct',\n",
       "  'Environment: py27',\n",
       "  'Environment: _build'],\n",
       " 'setuptools-23.0.0-py34_0': [],\n",
       " 'setuptools-23.0.0-py35_0': ['Environment: ct_test1',\n",
       "  'Environment: ct_test2',\n",
       "  'Environment: ct_test3',\n",
       "  'Environment: ftest',\n",
       "  'Environment: jupyterlab-dev',\n",
       "  'Environment: m1',\n",
       "  'Environment: old1'],\n",
       " 'simplegeneric-0.8.1-py27_1': ['Environment: py27'],\n",
       " 'simplegeneric-0.8.1-py35_1': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'six-1.10.0-py27_0': ['Environment: py27'],\n",
       " 'six-1.10.0-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'six-1.9.0-py35_0': ['Environment: m1'],\n",
       " 'traitlets-4.2.1-py27_0': ['Environment: py27'],\n",
       " 'traitlets-4.2.1-py35_0': ['Environment: Miniconda3',\n",
       "  'Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev'],\n",
       " 'vs2015_runtime-14.00.23026.0-0': ['Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep'],\n",
       " 'wheel-0.29.0-py27_0': ['Environment: pipct',\n",
       "  'Environment: py27',\n",
       "  'Environment: _build'],\n",
       " 'wheel-0.29.0-py34_0': [],\n",
       " 'wheel-0.29.0-py35_0': ['Environment: adamimage',\n",
       "  'Environment: ct_test1',\n",
       "  'Environment: ct_test2',\n",
       "  'Environment: ct_test3',\n",
       "  'Environment: env1',\n",
       "  'Environment: env2',\n",
       "  'Environment: ftest',\n",
       "  'Environment: imagep',\n",
       "  'Environment: jupyterlab-dev',\n",
       "  'Environment: m1',\n",
       "  'Environment: old1'],\n",
       " 'zlib-1.2.8-0': [],\n",
       " 'zlib-1.2.8-vc14_3': ['Environment: adamimage',\n",
       "  'Environment: env1',\n",
       "  'Environment: imagep']}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find which environments the latest packages are linked to.\n",
    "# This example uses Versio to parse and compare PEP440 compliant version numbers\n",
    "# This will exclude packages like packages like jpeg and openssl\n",
    "\n",
    "# This loop simple creates Version objects so we can compare them later.\n",
    "Versions = {}\n",
    "for x in pkg_cache:\n",
    "    try:\n",
    "        if x.name in Versions:\n",
    "            Versions[x.name].append(Version(x.version))\n",
    "        else:\n",
    "            Versions[x.name] = [Version(x.version)]\n",
    "    except:\n",
    "        print(\"Skipping \", x.name, x.version)\n",
    "    \n",
    "# sort the value lists and pick the latest versions\n",
    "#pversions = {k: str(list(sorted(v))[-1]) for k, v in Versions.items()}\n",
    "\n",
    "# sort the value lists and pick the older versions\n",
    "pversions = {k: list(map(str, list(sorted(v))[:-1])) for k, v in Versions.items()}\n",
    "\n",
    "# The most up-to-date packages are linked to which environments?\n",
    "#latest_pkgs = [x for x in pkg_cache if x.name in pversions and x.version == pversions[x.name]]\n",
    "\n",
    "# Find the environments that older packages are linked to\n",
    "latest_pkgs = [x for x in pkg_cache if x.name in pversions and x.version in set(pversions[x.name])]\n",
    "\n",
    "# Simply print the results nicely\n",
    "{str(k): list(map(str, v)) for k, v in cu.linked_environments(latest_pkgs, envs).items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-4.0.7-py35_0) @ 0x13dfd352908,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\hdf5-1.8.15.1-2) @ 0x13dfd3815f8,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\pip-8.1.2-py34_0) @ 0x13dfd39a048,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-4.1.6-py35_0) @ 0x13dfd371860,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-4.1.2-py35_0) @ 0x13dfd371c88,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\xlwt-1.1.2-py35_0) @ 0x13dfd39acc0,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\python-3.4.5-0) @ 0x13dfd39a2e8,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-build-1.21.5-py35_0) @ 0x13dfd37d6a0,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-env-2.5.0-py35_0) @ 0x13dfd37d860,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\jpeg-8d-0) @ 0x13dfd381828,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\wheel-0.29.0-py34_0) @ 0x13dfd39ac18,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-build-1.20.3-py35_0) @ 0x13dfd37d400,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\hdf4-4.2.11-0) @ 0x13dfd3815c0,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\vs2010_runtime-10.00.40219.1-2) @ 0x13dfd39aac8,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-4.0.8-py35_0) @ 0x13dfd352978,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\setuptools-23.0.0-py34_0) @ 0x13dfd39a6d8,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\conda-env-2.5.1-py35_0) @ 0x13dfd37d898,\n",
       " PackageInfo(C:\\Users\\Ryan\\Miniconda3\\pkgs\\zlib-1.2.8-0) @ 0x13dfd39acf8)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# All packages that are not linked to any environment\n",
    "cu.unlinked_packages(pkg_cache, envs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Environment representation of root environment\n",
    "e = environment.Environment(join(root_envs, 'env2'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Long running.  Disk intensive.\n",
    "filter_pyc = lambda f: filter(lambda x: not x.endswith('.pyc'), f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'alabaster': (),\n",
       " 'anaconda-client': (),\n",
       " 'babel': (),\n",
       " 'backports': (),\n",
       " 'clyent': (),\n",
       " 'colorama': (),\n",
       " 'conda': (),\n",
       " 'conda-build': (),\n",
       " 'conda-env': (),\n",
       " 'decorator': (),\n",
       " 'docutils': ('Scripts/rst2pseudoxml.py',\n",
       "  'Scripts/rst2odt.py',\n",
       "  'Scripts/rst2latex.py',\n",
       "  'Scripts/rstpep2html.py',\n",
       "  'Scripts/rst2s5.py',\n",
       "  'Scripts/rst2odt_prepstyles.py',\n",
       "  'Scripts/rst2xml.py',\n",
       "  'Scripts/rst2man.py',\n",
       "  'Scripts/rst2html.py',\n",
       "  'Scripts/rst2xetex.py'),\n",
       " 'entrypoints': (),\n",
       " 'get_terminal_size': (),\n",
       " 'imagesize': (),\n",
       " 'ipykernel': (),\n",
       " 'ipython': (),\n",
       " 'ipython_genutils': (),\n",
       " 'jinja2': (),\n",
       " 'jsonschema': (),\n",
       " 'jupyter_client': (),\n",
       " 'jupyter_core': (),\n",
       " 'markupsafe': (),\n",
       " 'mistune': (),\n",
       " 'nbconvert': (),\n",
       " 'nbformat': (),\n",
       " 'nodejs': (),\n",
       " 'notebook': (),\n",
       " 'patch': (),\n",
       " 'path.py': (),\n",
       " 'pickleshare': (),\n",
       " 'psutil': (),\n",
       " 'py': (),\n",
       " 'pygments': (),\n",
       " 'pyreadline': (),\n",
       " 'pytest': (),\n",
       " 'python': ('python.exe', 'DLLs/_ctypes.pyd', 'python35.dll'),\n",
       " 'python-dateutil': (),\n",
       " 'pytz': (),\n",
       " 'pyzmq': (),\n",
       " 'ruamel_yaml': (),\n",
       " 'semantic_version': (),\n",
       " 'setuptools': (),\n",
       " 'simplegeneric': (),\n",
       " 'six': (),\n",
       " 'snowballstemmer': (),\n",
       " 'sphinx': (),\n",
       " 'sphinx_rtd_theme': (),\n",
       " 'tornado': (),\n",
       " 'traitlets': ()}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List all files in an environment that are not hardlinked (and should be).\n",
    "# Note that *.pyc files are filtered out.\n",
    "not_linked = {x: tuple(filter_pyc(y)) for x, y in eu.check_hardlinked_env(envs[0]).items()}\n",
    "\n",
    "# If you wish to see all the non-existant hardlinks, including *.pyc files, remove the filter_pyc function call\n",
    "# not_linked = {x: y for x, y in eu.check_hardlinked_env(envs[0]).items()}\n",
    "\n",
    "not_linked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'2016-06-06 16:45:23': {'ipython-4.2.0-py35_0', 'python-3.5.1-4'}}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can leverage the information in the environment's history to get packages \n",
    "# that were explicitly installed by the user.\n",
    "eu.explicitly_installed(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}