forked from pantsbuild/pantsbuild.github.io
/
pex_design.html
458 lines (427 loc) · 20 KB
/
pex_design.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
<!DOCTYPE html>
<html lang="en">
<!--
Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
Licensed under the Apache License, Version 2.0 (see LICENSE).
-->
<head>
<meta charset="utf-8" />
<title>PEX Design</title>
<link rel="stylesheet" href="s.css">
<script src="s.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="shortcut icon" href="logo.ico">
<!-- In case this is a "test publish", tell search engines where real version lives: -->
<link rel="canonical" href="https://pantsbuild.github.io/pex_design.html">
</head>
<body>
<div id="navtoggler">
◄ <!-- triangle -->
</div>
<div class="mainflow">
<nav class="breadcrumbs">
<ul>
<li><a href="index.html">Pants Build</a>
<li><a href="python-readme.html">Python Projects with Pants</a>
<li><a href="pex_design.html">PEX Design</a>
</ul>
</nav>
<nav class="pagetoc">
<ul>
<li class="toc-h1"><a href="#alternatives">Alternatives</a></li>
<li class="toc-h1"><a href="#pants-and-pex">Pants and PEX</a></li>
<li class="toc-h1"><a href="#the-utility-of-zipimport-and-__main__py">the utility of zipimport and __main__.py</a></li>
<li class="toc-h1"><a href="#pex-__main__py">PEX __main__.py</a></li>
</ul>
</nav>
<!-- main content start -->
<!-- generated by pants! -->
<style>
.codehilite .hll { background-color: #ffffcc }
.codehilite { background: #f0f0f0; }
.codehilite .c { color: #60a0b0; font-style: italic } /* Comment */
.codehilite .err { border: 1px solid #FF0000 } /* Error */
.codehilite .k { color: #007020; font-weight: bold } /* Keyword */
.codehilite .o { color: #666666 } /* Operator */
.codehilite .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */
.codehilite .cp { color: #007020 } /* Comment.Preproc */
.codehilite .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */
.codehilite .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */
.codehilite .gd { color: #A00000 } /* Generic.Deleted */
.codehilite .ge { font-style: italic } /* Generic.Emph */
.codehilite .gr { color: #FF0000 } /* Generic.Error */
.codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */
.codehilite .gi { color: #00A000 } /* Generic.Inserted */
.codehilite .go { color: #808080 } /* Generic.Output */
.codehilite .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */
.codehilite .gs { font-weight: bold } /* Generic.Strong */
.codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
.codehilite .gt { color: #0040D0 } /* Generic.Traceback */
.codehilite .kc { color: #007020; font-weight: bold } /* Keyword.Constant */
.codehilite .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */
.codehilite .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */
.codehilite .kp { color: #007020 } /* Keyword.Pseudo */
.codehilite .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */
.codehilite .kt { color: #902000 } /* Keyword.Type */
.codehilite .m { color: #40a070 } /* Literal.Number */
.codehilite .s { color: #4070a0 } /* Literal.String */
.codehilite .na { color: #4070a0 } /* Name.Attribute */
.codehilite .nb { color: #007020 } /* Name.Builtin */
.codehilite .nc { color: #0e84b5; font-weight: bold } /* Name.Class */
.codehilite .no { color: #60add5 } /* Name.Constant */
.codehilite .nd { color: #555555; font-weight: bold } /* Name.Decorator */
.codehilite .ni { color: #d55537; font-weight: bold } /* Name.Entity */
.codehilite .ne { color: #007020 } /* Name.Exception */
.codehilite .nf { color: #06287e } /* Name.Function */
.codehilite .nl { color: #002070; font-weight: bold } /* Name.Label */
.codehilite .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */
.codehilite .nt { color: #062873; font-weight: bold } /* Name.Tag */
.codehilite .nv { color: #bb60d5 } /* Name.Variable */
.codehilite .ow { color: #007020; font-weight: bold } /* Operator.Word */
.codehilite .w { color: #bbbbbb } /* Text.Whitespace */
.codehilite .mf { color: #40a070 } /* Literal.Number.Float */
.codehilite .mh { color: #40a070 } /* Literal.Number.Hex */
.codehilite .mi { color: #40a070 } /* Literal.Number.Integer */
.codehilite .mo { color: #40a070 } /* Literal.Number.Oct */
.codehilite .sb { color: #4070a0 } /* Literal.String.Backtick */
.codehilite .sc { color: #4070a0 } /* Literal.String.Char */
.codehilite .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */
.codehilite .s2 { color: #4070a0 } /* Literal.String.Double */
.codehilite .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */
.codehilite .sh { color: #4070a0 } /* Literal.String.Heredoc */
.codehilite .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */
.codehilite .sx { color: #c65d09 } /* Literal.String.Other */
.codehilite .sr { color: #235388 } /* Literal.String.Regex */
.codehilite .s1 { color: #4070a0 } /* Literal.String.Single */
.codehilite .ss { color: #517918 } /* Literal.String.Symbol */
.codehilite .bp { color: #007020 } /* Name.Builtin.Pseudo */
.codehilite .vc { color: #bb60d5 } /* Name.Variable.Class */
.codehilite .vg { color: #bb60d5 } /* Name.Variable.Global */
.codehilite .vi { color: #bb60d5 } /* Name.Variable.Instance */
.codehilite .il { color: #40a070 } /* Literal.Number.Integer.Long */
</style>
<table class="h-plus-pilcrow">
<tbody>
<tr>
<td class="h-plus-pilcrow-holder"><h1 id="pex-design">PEX Design</h1></td>
<td><div class="pilcrow-div">
<a class="pilcrow-link" href="#pex-design">¶</a>
</div></td>
</tr>
</tbody>
</table>
<p>But why another system?</p>
<table class="h-plus-pilcrow">
<tbody>
<tr>
<td class="h-plus-pilcrow-holder"><h2 id="alternatives">Alternatives</h2></td>
<td><div class="pilcrow-div">
<a class="pilcrow-link" href="#alternatives">¶</a>
</div></td>
</tr>
</tbody>
</table>
<p>There are several solutions for package management in Python. Almost
everyone is familiar with running <code>sudo easy_install PackageXYZ</code>. This
leaves a lot to be desired. Over time, your Python installation will
collect dozens of packages, become annoyingly slow or even broken, and
reinstalling it will invariably break a number of the applications that
you were using.</p>
<p>A marked improvement over the sudo <code>easy_install</code> model is
<a href="http://www.virtualenv.org">virtualenv</a> to isolate Python environments
on a project by project basis. This is useful for development but does
not directly solve any problems related to deployment, whether it be to
a production environment or to your peers. It is also challenging to
explain to a Python non-expert.</p>
<p>A different solution altogether, <a href="http://www.buildout.org/">zc.buildout</a>
attempts to provide a framework and recipes for many common development
environments. It has arguably gone the farthest for automating
environment reproducibility amongst the popular tools, but shares the
same complexity problems as all the other abovementioned solutions.</p>
<p>Most solutions leave deployment as an afterthought. Why not make the
development and deployment environments the same by taking the
environment along with you?</p>
<table class="h-plus-pilcrow">
<tbody>
<tr>
<td class="h-plus-pilcrow-holder"><h2 id="pants-and-pex">Pants and PEX</h2></td>
<td><div class="pilcrow-div">
<a class="pilcrow-link" href="#pants-and-pex">¶</a>
</div></td>
</tr>
</tbody>
</table>
<p>The lingua franca of Pants is the PEX file (PEX itself does not stand
for anything in particular, though in spirit you can think of it as a
"Python EXecutable".)</p>
<p><strong>PEX files are single-file lightweight virtual Python environments.</strong></p>
<p>The only difference is no virtualenv setup instructions or
pip install foo bar baz. PEX files are self-bootstrapping Python
environments with no strings attached and no side-effects. Just a simple
mechanism that unifies both your development and your deployment.</p>
<table class="h-plus-pilcrow">
<tbody>
<tr>
<td class="h-plus-pilcrow-holder"><h3 id="how-pex-files-work">How PEX files work</h3></td>
<td><div class="pilcrow-div">
<a class="pilcrow-link" href="#how-pex-files-work">¶</a>
</div></td>
</tr>
</tbody>
</table>
<table class="h-plus-pilcrow">
<tbody>
<tr>
<td class="h-plus-pilcrow-holder"><h2 id="the-utility-of-zipimport-and-__main__py">the utility of zipimport and <code>__main__.py</code></h2></td>
<td><div class="pilcrow-div">
<a class="pilcrow-link" href="#the-utility-of-zipimport-and-__main__py">¶</a>
</div></td>
</tr>
</tbody>
</table>
<p>As an aside, in Python, you may not know that you can import code from
directories:</p>
<div class="codehilite"><pre><span class="nv">$ </span>mkdir -p foo
<span class="nv">$ </span>touch foo/__init__.py
<span class="nv">$ </span><span class="nb">echo</span> <span class="s2">"print 'spam'"</span> > foo/bar.py
<span class="nv">$ </span>python -c <span class="s1">'import foo.bar'</span>
spam
</pre></div>
<p>All that is necessary is the presence of <code>__init__.py</code> to signal to
the importer that we are dealing with a package. Similarly, a directory
can be made "executable":</p>
<div class="codehilite"><pre><span class="nv">$ </span><span class="nb">echo</span> <span class="s2">"print 'i like flowers'"</span> > foo/__main__.py
<span class="nv">$ </span>python foo
i like flowers
</pre></div>
<p>And because the zipimport module now provides a default import hook for
Pythons >= 2.4, if the Python import framework sees a zip file, with
the inclusion of a proper <code>__init__.py</code>, it can be treated similarly
to a directory. But since a directory can be executable, if we just drop
a <code>__main__.py</code> into a zip file, it suddenly becomes executable:</p>
<div class="codehilite"><pre><span class="nv">$ </span><span class="nb">pushd </span>foo <span class="o">&&</span> zip /tmp/flower.zip __main__.py <span class="o">&&</span> <span class="nb">popd</span>
/tmp/foo /tmp
adding: __main__.py <span class="o">(</span>stored 0%<span class="o">)</span>
/tmp
<span class="nv">$ </span>python flower.zip
i like flowers
</pre></div>
<p>And since zip files don't actually start until the zip magic number, you
can embed arbitrary strings at the beginning of them and they're still
valid zips. Hence simple PEX files are born:</p>
<div class="codehilite"><pre><span class="nv">$ </span><span class="nb">echo</span> <span class="s1">'#!/usr/bin/env python2.6'</span> > flower.pex <span class="o">&&</span> cat flower.zip >> flower.pex
<span class="nv">$ </span>chmod +x flower.pex
<span class="nv">$ </span>./flower.pex
i like flowers
</pre></div>
<p>Remember pants.pex?</p>
<div class="codehilite"><pre><span class="nv">$ </span>unzip -l pants.pex | tail -2
warning <span class="o">[</span>pants.pex<span class="o">]</span>: 25 extra bytes at beginning or within zipfile
<span class="o">(</span>attempting to process anyway<span class="o">)</span>
-------- -------
7900812 543 files
<span class="nv">$ </span>head -c 25 pants.pex
<span class="c">#!/usr/bin/env python2.6</span>
</pre></div>
<table class="h-plus-pilcrow">
<tbody>
<tr>
<td class="h-plus-pilcrow-holder"><h2 id="pex-__main__py">PEX <code>__main__.py</code></h2></td>
<td><div class="pilcrow-div">
<a class="pilcrow-link" href="#pex-__main__py">¶</a>
</div></td>
</tr>
</tbody>
</table>
<p>The <code>__main__.py</code> in a real PEX file is somewhat special:</p>
<div class="codehilite"><pre><span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="n">__entry_point__</span> <span class="o">=</span> <span class="bp">None</span>
<span class="k">if</span> <span class="s">'__file__'</span> <span class="ow">in</span> <span class="nb">locals</span><span class="p">()</span> <span class="ow">and</span> <span class="n">__file__</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">__entry_point__</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">__file__</span><span class="p">)</span>
<span class="k">elif</span> <span class="s">'__loader__'</span> <span class="ow">in</span> <span class="nb">locals</span><span class="p">():</span>
<span class="kn">from</span> <span class="nn">pkgutil</span> <span class="kn">import</span> <span class="n">ImpLoader</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">__loader__</span><span class="p">,</span> <span class="s">'archive'</span><span class="p">):</span>
<span class="n">__entry_point__</span> <span class="o">=</span> <span class="n">__loader__</span><span class="o">.</span><span class="n">archive</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">__loader__</span><span class="p">,</span> <span class="n">ImpLoader</span><span class="p">):</span>
<span class="n">__entry_point__</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">__loader__</span><span class="o">.</span><span class="n">get_filename</span><span class="p">())</span>
<span class="k">if</span> <span class="n">__entry_point__</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">sys</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s">'Could not launch python executable!</span><span class="se">\n</span><span class="s">'</span><span class="p">)</span>
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">__entry_point__</span><span class="p">,</span> <span class="s">'.bootstrap'</span><span class="p">))</span>
<span class="kn">from</span> <span class="nn">twitter.common.python.importer</span> <span class="kn">import</span> <span class="n">monkeypatch</span>
<span class="n">monkeypatch</span><span class="p">()</span>
<span class="k">del</span> <span class="n">monkeypatch</span>
<span class="kn">from</span> <span class="nn">pex.pex</span> <span class="kn">import</span> <span class="n">PEX</span>
<span class="n">PEX</span><span class="p">(</span><span class="n">__entry_point__</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
</pre></div>
<p>PEX is just a class that manages requirements (often embedded within PEX
files as egg distributions in the .deps directory) and autoimports them
into the sys.path, then executes a prescribed entry point.</p>
<p>If you read the code closely, you'll notice that it relies upon
monkeypatching zipimport. Inside the twitter.common.python library we've
provided a recursive zip importer derived from Google's <a href="http://code.google.com/appengine/articles/django10_zipimport.html">pure Python
zipimport</a>
module that allows for depending upon eggs within eggs or zips (and so
forth) so that PEX files need not extract egg dependencies to disk a
priori. This even extends to C extensions (.so and .dylib files) which
are written to disk long enough to be dlopened before being unlinked.</p>
<p>Strictly speaking this monkeypatching is not necessary and we may
consider making that optional.</p>
<!-- main content end -->
<div class="generated">
Generated by <a href="http://pantsbuild.github.io/docs.html">publish_docs</a>
from dist/markdown/html/examples/src/python/example/pex_design.html 2016-03-20T07:25:04.690284
</div>
</div> <!-- end of mainflow -->
<nav class="bignav">
<div class="bignav-logo">
<a href="index.html">
<img src="logo.ico" alt="[pantsbuild logo]">
</a>
</div>
<form class="search" action="https://www.google.com/search">
<input name="as_q" class="query" placeholder="Search">
<input name="as_sitesearch" value="pantsbuild.github.io" type="hidden">
</form>
<div class="site-toc">
<ul>
<li class="toc-h1 ">
<a href="first_concepts.html">Pants Conceptual Overview</a>
</li>
<li class="toc-h1 ">
<a href="install.html">Installing Pants</a>
</li>
<li class="toc-h1 ">
<a href="first_tutorial.html">First Tutorial</a>
</li>
<li class="toc-h1 ">
<a href="target_addresses.html">Target Addresses</a>
</li>
<li class="toc-h1 ">
<a href="JVMProjects.html">JVM Projects with Pants</a>
</li>
<li class="toc-h2 ">
<a href="3rdparty_jvm.html">JVM 3rdparty Pattern</a>
</li>
<li class="toc-h2 ">
<a href="scala.html">Scala Projects with Pants</a>
</li>
<li class="toc-h2 ">
<a href="from_maven.html">Pants for Maven Experts</a>
</li>
<li class="toc-h1 ">
<a href="python-readme.html">Python Projects with Pants</a>
</li>
<li class="toc-h2 ">
<a href="3rdparty_py.html">Python 3rdparty Pattern</a>
</li>
<li class="toc-h2 toc-here">
PEX Design
</li>
<li class="toc-h1 ">
<a href="page.html">README Files and Markdown</a>
</li>
<li class="toc-h1 ">
<a href="build_files.html">BUILD files</a>
</li>
<li class="toc-h1 ">
<a href="invoking.html">Invoking Pants Build</a>
</li>
<li class="toc-h1 ">
<a href="tshoot.html">Troubleshooting</a>
</li>
<li class="toc-h1 ">
<a href="3rdparty.html">Third-Party Dependencies</a>
</li>
<li class="toc-h2 ">
<a href="3rdparty_jvm.html">JVM 3rdparty Pattern</a>
</li>
<li class="toc-h2 ">
<a href="3rdparty_py.html">Python 3rdparty Pattern</a>
</li>
<li class="toc-h1 ">
<a href="ThriftDeps.html">Using Pants with Thrift Dependencies</a>
</li>
<li class="toc-h1 ">
<a href="publish.html">Publishing Artifacts</a>
</li>
<li class="toc-h1 ">
<a href="with_emacs.html">Emacs With Pants</a>
</li>
<li class="toc-h1 ">
<a href="with_intellij.html">Using Pants with IntelliJ IDEA</a>
</li>
<li class="toc-h1 ">
<a href="setup_repo.html">Set up Your Source Tree for Pants</a>
</li>
<li class="toc-h1 ">
<a href="build_dictionary.html">Pants BUILD Dictionary</a>
</li>
<li class="toc-h1 ">
<a href="options_reference.html">Pants Reference</a>
</li>
<li class="toc-h1 ">
<a href="changelog.html">RELEASE HISTORY</a>
</li>
<li class="toc-h1 ">
<a href="export.html">Export Format</a>
</li>
<li class="toc-h1 ">
<a href="dev.html">Pants Developer Center</a>
</li>
<li class="toc-h2 ">
<a href="howto_contribute.html">Pants Contributors Guide</a>
</li>
<li class="toc-h2 ">
<a href="howto_ask.html">How To Ask</a>
</li>
<li class="toc-h2 ">
<a href="howto_develop.html">Pants Developers Guide</a>
</li>
<li class="toc-h2 ">
<a href="dev_tasks.html">Task Developer's Guide</a>
</li>
<li class="toc-h3 ">
<a href="dev_tasks_publish_extras.html">Develop a Task to Publish "Extra" Artifacts</a>
</li>
<li class="toc-h2 ">
<a href="styleguide.html">Pants Style Guide</a>
</li>
<li class="toc-h2 ">
<a href="howto_plugin.html">Developing a Pants Plugin</a>
</li>
<li class="toc-h2 ">
<a href="intellij.html">Pants Development with IntelliJ IDEA</a>
</li>
<li class="toc-h2 ">
<a href="internals.html">Pants Internals</a>
</li>
<li class="toc-h2 ">
<a href="deprecation_policy.html">Pants Deprecation Policy</a>
</li>
<li class="toc-h2 ">
<a href="release.html">Release Process</a>
</li>
<li class="toc-h3 ">
<a href="release_jvm.html">JVM Artifact Release Process</a>
</li>
<li class="toc-h2 ">
<a href="docs.html">About the documentation</a>
</li>
<li class="toc-h2 ">
<a href="credits.html">Credits</a>
</li>
</ul>
</div>
</nav>
<div class="ci-status">
<a href="https://travis-ci.org/pantsbuild/pants">
<img src="https://travis-ci.org/pantsbuild/pants.png?branch=master"
alt="Are the tests passing?" title="Are the tests passing?"></a><br>
<a href="https://coveralls.io/r/pantsbuild/pants?branch=master">
<img src="https://coveralls.io/repos/pantsbuild/pants/badge.png?branch=master"
alt="Test Coverage Status" title="Test Coverage Status"></a>
</div>
</body>
</html>