Numa-aware computation experiments.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numa\n",
      "from IPython.parallel import Client\n",
      "client = Client()\n",
      "len(client)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "30"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!echo 3 | sudo tee /proc/sys/vm/drop_caches"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3\r\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for node_id in range(numa.get_max_node() + 1):\n",
      "    print(numa.get_node_size(node_id))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(127552450560L, 131071533056L)\n",
        "(127627759616L, 131072000000L)\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "from os.path import exists, join\n",
      "\n",
      "filename = 'a.mmap'\n",
      "mode = 'r+' if exists(filename) else 'w+'\n",
      "a = np.memmap(filename, shape=(int(8e6), 784), dtype=np.float32, mode=mode)\n",
      "\n",
      "n_jobs = 60\n",
      "\n",
      "a.nbytes"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "25088000000"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def do_stuff(params):\n",
      "    seed, filename, n_iter, numa_aware = params\n",
      "    import numpy as np\n",
      "    import os\n",
      "    n_samples, n_features = int(8e6), 784\n",
      "    rng = np.random.RandomState(seed)\n",
      "    \n",
      "    if numa_aware:\n",
      "        # Create hard link for the current process numa node\n",
      "        import numa\n",
      "        effective_filename = \"%s_%d\" % (filename, numa.get_preferred())\n",
      "        if not os.path.exists(effective_filename):\n",
      "            os.link(filename, effective_filename)\n",
      "    else:\n",
      "        effective_filename = filename\n",
      "    data = np.memmap(effective_filename, shape=(n_samples, n_features),\n",
      "                     dtype=np.float32, mode='r')\n",
      "    # Trigger one sequential scan of the whole readonly data\n",
      "    data.max()\n",
      "    \n",
      "    # Trigger n_iter random access to chunks of data\n",
      "    for i in range(n_iter):\n",
      "        idx = rng.random_integers(low=0, high=n_samples - 1, size=1000)\n",
      "        np.mean(data[idx])\n",
      "        \n",
      "    return effective_filename\n",
      "\n",
      "workers = client.load_balanced_view()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%px\n",
      "\n",
      "import numa\n",
      "print(numa.get_preferred(), numa.get_affinity(0))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[stdout:0] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:1] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:2] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:3] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:4] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:5] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:6] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:7] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:8] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:9] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:10] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:11] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:12] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:13] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:14] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:15] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:16] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:17] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:18] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:19] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:20] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:21] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:22] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:23] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:24] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:25] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:26] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:27] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:28] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:29] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]))\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%time workers.map(do_stuff, [(i, filename, 50, False) for i in range(n_jobs)]).get()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "CPU times: user 6.56 s, sys: 592 ms, total: 7.15 s\n",
        "Wall time: 1min 11s\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "['a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap']"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def assign_cpu_numa_node(engine_id, fix_cpu=False):\n",
      "    import numa\n",
      "    n_nodes = numa.get_max_node() + 1\n",
      "    local_engine_id, node_id = divmod(engine_id, n_nodes)\n",
      "    # Assing current process to a fixed numa node\n",
      "    numa.set_preferred(node_id)\n",
      "    cpu_ids = list(sorted(numa.node_to_cpus(node_id)))\n",
      "    if fix_cpu:\n",
      "        # Fix engine to one specific CPU that is bound with node_id\n",
      "        cpu_id = cpu_ids[local_engine_id % len(cpu_ids)]\n",
      "        numa.set_affinity(0, {cpu_id})\n",
      "    else:\n",
      "        # Set affinity of current process to any of the CPUs bound\n",
      "        # with node_id\n",
      "        numa.set_affinity(0, cpu_ids)\n",
      "\n",
      "for engine_id in client.ids:\n",
      "    client[engine_id].apply(assign_cpu_numa_node, engine_id, fix_cpu=False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%px\n",
      "\n",
      "import numa\n",
      "print(numa.get_preferred(), numa.get_affinity(0))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[stdout:0] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:1] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:2] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:3] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:4] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:5] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:6] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:7] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:8] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:9] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:10] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:11] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:12] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:13] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:14] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:15] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:16] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:17] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:18] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:19] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:20] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:21] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:22] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:23] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:24] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:25] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:26] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:27] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n",
        "[stdout:28] (0, set([0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23]))\n",
        "[stdout:29] (1, set([8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31]))\n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!echo 3 | sudo tee /proc/sys/vm/drop_caches"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3\r\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%time workers.map(do_stuff, [(i, filename, 50, True) for i in range(n_jobs)]).get()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "CPU times: user 8.42 s, sys: 508 ms, total: 8.93 s\n",
        "Wall time: 1min 24s\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "['a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1']"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for engine_id in client.ids:\n",
      "    client[engine_id].apply(assign_cpu_numa_node, engine_id, fix_cpu=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%px\n",
      "\n",
      "import numa\n",
      "print(numa.get_preferred(), numa.get_affinity(0))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[stdout:0] (0, set([0]))\n",
        "[stdout:1] (1, set([8]))\n",
        "[stdout:2] (0, set([1]))\n",
        "[stdout:3] (1, set([9]))\n",
        "[stdout:4] (0, set([2]))\n",
        "[stdout:5] (1, set([10]))\n",
        "[stdout:6] (0, set([3]))\n",
        "[stdout:7] (1, set([11]))\n",
        "[stdout:8] (0, set([4]))\n",
        "[stdout:9] (1, set([12]))\n",
        "[stdout:10] (0, set([5]))\n",
        "[stdout:11] (1, set([13]))\n",
        "[stdout:12] (0, set([6]))\n",
        "[stdout:13] (1, set([14]))\n",
        "[stdout:14] (0, set([7]))\n",
        "[stdout:15] (1, set([15]))\n",
        "[stdout:16] (0, set([16]))\n",
        "[stdout:17] (1, set([24]))\n",
        "[stdout:18] (0, set([17]))\n",
        "[stdout:19] (1, set([25]))\n",
        "[stdout:20] (0, set([18]))\n",
        "[stdout:21] (1, set([26]))\n",
        "[stdout:22] (0, set([19]))\n",
        "[stdout:23] (1, set([27]))\n",
        "[stdout:24] (0, set([20]))\n",
        "[stdout:25] (1, set([28]))\n",
        "[stdout:26] (0, set([21]))\n",
        "[stdout:27] (1, set([29]))\n",
        "[stdout:28] (0, set([22]))\n",
        "[stdout:29] (1, set([30]))\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!echo 3 | sudo tee /proc/sys/vm/drop_caches"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3\r\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%time workers.map(do_stuff, [(i, filename, 50, True) for i in range(n_jobs)]).get()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "CPU times: user 5.54 s, sys: 520 ms, total: 6.06 s\n",
        "Wall time: 1min 11s\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "['a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1',\n",
        " 'a.mmap_0',\n",
        " 'a.mmap_1']"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!echo 3 | sudo tee /proc/sys/vm/drop_caches"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3\r\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%time workers.map(do_stuff, [(i, filename, 50, False) for i in range(n_jobs)]).get()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "CPU times: user 7.08 s, sys: 480 ms, total: 7.56 s\n",
        "Wall time: 1min 7s\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "['a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap',\n",
        " 'a.mmap']"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Conclusion apparently none of the numa aware modifications seem to have any impact on the total runtime while by having a look a top during the computation one can see that in all cases, a significant part of the CPU time of the engines is spent in \"sys\" instead of \"user\"."
     ]
    }
   ],
   "metadata": {}
  }
 ]
}