forked from wntrblm/nox
-
Notifications
You must be signed in to change notification settings - Fork 0
/
virtualenv.py
407 lines (328 loc) · 13.8 KB
/
virtualenv.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
# Copyright 2016 Alethea Katherine Flowers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import platform
import re
import shutil
import sys
from socket import gethostbyname
from typing import Any, List, Mapping, Optional, Tuple, Union
import nox
import nox.command
import py
from nox.logger import logger
from . import _typing
# Problematic environment variables that are stripped from all commands inside
# of a virtualenv. See https://github.com/theacodes/nox/issues/44
_BLACKLISTED_ENV_VARS = frozenset(
["PIP_RESPECT_VIRTUALENV", "PIP_REQUIRE_VIRTUALENV", "__PYVENV_LAUNCHER__"]
)
_SYSTEM = platform.system()
class InterpreterNotFound(OSError):
def __init__(self, interpreter: str) -> None:
super().__init__("Python interpreter {} not found".format(interpreter))
self.interpreter = interpreter
class ProcessEnv:
"""A environment with a 'bin' directory and a set of 'env' vars."""
# Does this environment provide any process isolation?
is_sandboxed = False
# Special programs that aren't included in the environment.
allowed_globals = () # type: _typing.ClassVar[Tuple[Any, ...]]
def __init__(self, bin_paths: None = None, env: Mapping[str, str] = None) -> None:
self._bin_paths = bin_paths
self.env = os.environ.copy()
if env is not None:
self.env.update(env)
for key in _BLACKLISTED_ENV_VARS:
self.env.pop(key, None)
if self.bin_paths:
self.env["PATH"] = os.pathsep.join(
self.bin_paths + [self.env.get("PATH", "")]
)
@property
def bin_paths(self) -> Optional[List[str]]:
return self._bin_paths
@property
def bin(self) -> Optional[str]:
"""The first bin directory for the virtualenv."""
paths = self.bin_paths
return paths[0] if paths is not None else None
def create(self) -> bool:
raise NotImplementedError("ProcessEnv.create should be overwritten in subclass")
def locate_via_py(version: str) -> Optional[str]:
"""Find the Python executable using the Windows Launcher.
This is based on :pep:397 which details that executing
``py.exe -{version}`` should execute python with the requested
version. We then make the python process print out its full
executable path which we use as the location for the version-
specific Python interpreter.
Args:
version (str): The desired Python version to pass to ``py.exe``. Of the form
``X.Y`` or ``X.Y-32``. For example, a usage of the Windows Launcher might
be ``py -3.6-32``.
Returns:
Optional[str]: The full executable path for the Python ``version``,
if it is found.
"""
script = "import sys; print(sys.executable)"
py_exe = py.path.local.sysfind("py")
if py_exe is not None:
try:
return py_exe.sysexec("-" + version, "-c", script).strip()
except py.process.cmdexec.Error:
return None
return None
def locate_using_path_and_version(version: str) -> Optional[str]:
"""Check the PATH's python interpreter and return it if the version
matches.
On systems without version-named interpreters and with missing
launcher (which is on all Windows Anaconda installations),
we search the PATH for a plain "python" interpreter and accept it
if its --version matches the specified interpreter version.
Args:
version (str): The desired Python version. Of the form ``X.Y``.
Returns:
Optional[str]: The full executable path for the Python ``version``,
if it is found.
"""
if not version:
return None
script = "import platform; print(platform.python_version())"
path_python = py.path.local.sysfind("python")
if path_python:
try:
prefix = "{}.".format(version)
version_string = path_python.sysexec("-c", script).strip()
if version_string.startswith(prefix):
return str(path_python)
except py.process.cmdexec.Error:
return None
return None
def _clean_location(self: "Union[CondaEnv, VirtualEnv]") -> bool:
"""Deletes any existing path-based environment"""
if os.path.exists(self.location):
if self.reuse_existing:
return False
else:
shutil.rmtree(self.location)
return True
class PassthroughEnv(ProcessEnv):
"""Represents the environment used to run nox itself
For now, this class is empty but it might contain tools to grasp some
hints about the actual env.
"""
@staticmethod
def is_offline() -> bool:
"""As of now this is only used in conda_install"""
return CondaEnv.is_offline() # pragma: no cover
class CondaEnv(ProcessEnv):
"""Conda environment management class.
Args:
location (str): The location on the filesystem where the conda environment
should be created.
interpreter (Optional[str]): The desired Python version. Of the form
* ``X.Y``, e.g. ``3.5``
* ``X.Y-32``. For example, a usage of the Windows Launcher might
be ``py -3.6-32``
* ``X.Y.Z``, e.g. ``3.4.9``
* ``pythonX.Y``, e.g. ``python2.7``
* A path in the filesystem to a Python executable
If not specified, this will use the currently running Python.
reuse_existing (Optional[bool]): Flag indicating if the conda environment
should be reused if it already exists at ``location``.
"""
is_sandboxed = True
allowed_globals = ("conda",)
def __init__(
self,
location: str,
interpreter: Optional[str] = None,
reuse_existing: bool = False,
venv_params: Any = None,
):
self.location_name = location
self.location = os.path.abspath(location)
self.interpreter = interpreter
self.reuse_existing = reuse_existing
self.venv_params = venv_params if venv_params else []
super(CondaEnv, self).__init__()
_clean_location = _clean_location
@property
def bin_paths(self) -> List[str]:
"""Returns the location of the conda env's bin folder."""
# see https://docs.anaconda.com/anaconda/user-guide/tasks/integration/python-path/#examples
if _SYSTEM == "Windows":
return [self.location, os.path.join(self.location, "Scripts")]
else:
return [os.path.join(self.location, "bin")]
def create(self) -> bool:
"""Create the conda env."""
if not self._clean_location():
logger.debug(
"Re-using existing conda env at {}.".format(self.location_name)
)
return False
cmd = [
"conda",
"create",
"--yes",
"--prefix",
self.location,
# Ensure the pip package is installed.
"pip",
]
cmd.extend(self.venv_params)
if self.interpreter:
python_dep = "python={}".format(self.interpreter)
else:
python_dep = "python"
cmd.append(python_dep)
logger.info(
"Creating conda env in {} with {}".format(self.location_name, python_dep)
)
nox.command.run(cmd, silent=True, log=nox.options.verbose or False)
return True
@staticmethod
def is_offline() -> bool:
"""Return `True` if we are sure that the user is not able to connect to https://repo.anaconda.com.
Since an HTTP proxy might be correctly configured for `conda` using the `.condarc` `proxy_servers` section,
while not being correctly configured in the OS environment variables used by all other tools including python
`urllib` or `requests`, we are basically not able to do much more than testing the DNS resolution.
See details in this explanation: https://stackoverflow.com/a/62486343/7262247
"""
try:
# DNS resolution to detect situation (1) or (2).
host = gethostbyname("repo.anaconda.com")
return host is None
except: # pragma: no cover # noqa E722
return True
class VirtualEnv(ProcessEnv):
"""Virtualenv management class.
Args:
location (str): The location on the filesystem where the virtual environment
should be created.
interpreter (Optional[str]): The desired Python version. Of the form
* ``X.Y``, e.g. ``3.5``
* ``X.Y-32``. For example, a usage of the Windows Launcher might
be ``py -3.6-32``
* ``X.Y.Z``, e.g. ``3.4.9``
* ``pythonX.Y``, e.g. ``python2.7``
* A path in the filesystem to a Python executable
If not specified, this will use the currently running Python.
reuse_existing (Optional[bool]): Flag indicating if the virtual environment
should be reused if it already exists at ``location``.
"""
is_sandboxed = True
def __init__(
self,
location: str,
interpreter: Optional[str] = None,
reuse_existing: bool = False,
*,
venv: bool = False,
venv_params: Any = None
):
self.location_name = location
self.location = os.path.abspath(location)
self.interpreter = interpreter
self._resolved = None # type: Union[None, str, InterpreterNotFound]
self.reuse_existing = reuse_existing
self.venv_or_virtualenv = "venv" if venv else "virtualenv"
self.venv_params = venv_params if venv_params else []
super(VirtualEnv, self).__init__(env={"VIRTUAL_ENV": self.location})
_clean_location = _clean_location
@property
def _resolved_interpreter(self) -> str:
"""Return the interpreter, appropriately resolved for the platform.
Based heavily on tox's implementation (tox/interpreters.py).
"""
# If there is no assigned interpreter, then use the same one used by
# Nox.
if isinstance(self._resolved, Exception):
raise self._resolved
if self._resolved is not None:
return self._resolved
if self.interpreter is None:
self._resolved = sys.executable
return self._resolved
# Otherwise we need to divine the path to the interpreter. This is
# designed to accept strings in the form of "2", "2.7", "2.7.13",
# "2.7.13-32", "python2", "python2.4", etc.
xy_version = ""
cleaned_interpreter = self.interpreter
# If this is just a X, X.Y, or X.Y.Z string, extract just the X / X.Y
# part and add Python to the front of it.
match = re.match(r"^(?P<xy_ver>\d(\.\d+)?)(\.\d+)?$", self.interpreter)
if match:
xy_version = match.group("xy_ver")
cleaned_interpreter = "python{}".format(xy_version)
# If the cleaned interpreter is on the PATH, go ahead and return it.
if py.path.local.sysfind(cleaned_interpreter):
self._resolved = cleaned_interpreter
return self._resolved
# The rest of this is only applicable to Windows, so if we don't have
# an interpreter by now, raise.
if _SYSTEM != "Windows":
self._resolved = InterpreterNotFound(self.interpreter)
raise self._resolved
# Allow versions of the form ``X.Y-32`` for Windows.
match = re.match(r"^\d\.\d+-32?$", cleaned_interpreter)
if match:
# preserve the "-32" suffix, as the Python launcher expects
# it.
xy_version = cleaned_interpreter
path_from_launcher = locate_via_py(xy_version)
if path_from_launcher:
self._resolved = path_from_launcher
return self._resolved
path_from_version_param = locate_using_path_and_version(xy_version)
if path_from_version_param:
self._resolved = path_from_version_param
return self._resolved
# If we got this far, then we were unable to resolve the interpreter
# to an actual executable; raise an exception.
self._resolved = InterpreterNotFound(self.interpreter)
raise self._resolved
@property
def bin_paths(self) -> List[str]:
"""Returns the location of the virtualenv's bin folder."""
if _SYSTEM == "Windows":
return [os.path.join(self.location, "Scripts")]
else:
return [os.path.join(self.location, "bin")]
def create(self) -> bool:
"""Create the virtualenv or venv."""
if not self._clean_location():
logger.debug(
"Re-using existing virtual environment at {}.".format(
self.location_name
)
)
return False
if self.venv_or_virtualenv == "virtualenv":
cmd = [sys.executable, "-m", "virtualenv", self.location]
if self.interpreter:
cmd.extend(["-p", self._resolved_interpreter])
else:
cmd = [self._resolved_interpreter, "-m", "venv", self.location]
cmd.extend(self.venv_params)
logger.info(
"Creating virtual environment ({}) using {} in {}".format(
self.venv_or_virtualenv,
os.path.basename(self._resolved_interpreter),
self.location_name,
)
)
nox.command.run(cmd, silent=True, log=False)
return True