Improve python healthy() and eliminate python_venv

- the `healthy()` check now requires virtualenv 20.x's metadata
- `python_venv` is obsolete now that `virtualenv` generates the same structure
  and `virtualenv` is more portable

pre_commit/languages/all.py

@@ -14,7 +14,6 @@ from pre_commit.languages import node
 from pre_commit.languages import perl
 from pre_commit.languages import pygrep
 from pre_commit.languages import python
-from pre_commit.languages import python_venv
 from pre_commit.languages import ruby
 from pre_commit.languages import rust
 from pre_commit.languages import script
@@ -49,7 +48,6 @@ languages = {
     'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook),  # noqa: E501
     'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook),  # noqa: E501
     'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook),  # noqa: E501
-    'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook),  # noqa: E501
     'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook),  # noqa: E501
     'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook),  # noqa: E501
     'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook),  # noqa: E501
@@ -57,4 +55,6 @@ languages = {
     'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook),  # noqa: E501
     # END GENERATED
 }
+# TODO: fully deprecate `python_venv`
+languages['python_venv'] = languages['python']
 all_languages = sorted(languages)

pre_commit/languages/python.py

@@ -2,8 +2,7 @@ import contextlib
 import functools
 import os
 import sys
-from typing import Callable
+from typing import Dict
-from typing import ContextManager
 from typing import Generator
 from typing import Optional
 from typing import Sequence
@@ -26,6 +25,28 @@ from pre_commit.util import cmd_output_b
 ENVIRONMENT_DIR = 'py_env'
 
 
+@functools.lru_cache(maxsize=None)
+def _version_info(exe: str) -> str:
+    prog = 'import sys;print(".".join(str(p) for p in sys.version_info))'
+    try:
+        return cmd_output(exe, '-S', '-c', prog)[1].strip()
+    except CalledProcessError:
+        return f'<<error retrieving version from {exe}>>'
+
+
+def _read_pyvenv_cfg(filename: str) -> Dict[str, str]:
+    ret = {}
+    with open(filename) as f:
+        for line in f:
+            try:
+                k, v = line.split('=')
+            except ValueError:  # blank line / comment / etc.
+                continue
+            else:
+                ret[k.strip()] = v.strip()
+    return ret
+
+
 def bin_dir(venv: str) -> str:
     """On windows there's a different directory for the virtualenv"""
     bin_part = 'Scripts' if os.name == 'nt' else 'bin'
@@ -116,6 +137,9 @@ def _sys_executable_matches(version: str) -> bool:
 
 
 def norm_version(version: str) -> str:
+    if version == C.DEFAULT:
+        return os.path.realpath(sys.executable)
+
     # first see if our current executable is appropriate
     if _sys_executable_matches(version):
         return sys.executable
@@ -140,70 +164,59 @@ def norm_version(version: str) -> str:
     return os.path.expanduser(version)
 
 
-def py_interface(
+@contextlib.contextmanager
-        _dir: str,
+def in_env(
-        _make_venv: Callable[[str, str], None],
+        prefix: Prefix,
-) -> Tuple[
+        language_version: str,
-    Callable[[Prefix, str], ContextManager[None]],
+) -> Generator[None, None, None]:
-    Callable[[Prefix, str], bool],
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
-    Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]],
+    envdir = prefix.path(directory)
-    Callable[[Prefix, str, Sequence[str]], None],
+    with envcontext(get_env_patch(envdir)):
-]:
+        yield
-    @contextlib.contextmanager
-    def in_env(
-            prefix: Prefix,
-            language_version: str,
-    ) -> Generator[None, None, None]:
-        envdir = prefix.path(helpers.environment_dir(_dir, language_version))
-        with envcontext(get_env_patch(envdir)):
-            yield
-
-    def healthy(prefix: Prefix, language_version: str) -> bool:
-        envdir = helpers.environment_dir(_dir, language_version)
-        exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
-        py_exe = prefix.path(bin_dir(envdir), exe_name)
-        with in_env(prefix, language_version):
-            retcode, _, _ = cmd_output_b(
-                py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref',
-                cwd='/',
-                retcode=None,
-            )
-        return retcode == 0
-
-    def run_hook(
-            hook: Hook,
-            file_args: Sequence[str],
-            color: bool,
-    ) -> Tuple[int, bytes]:
-        with in_env(hook.prefix, hook.language_version):
-            return helpers.run_xargs(hook, hook.cmd, file_args, color=color)
-
-    def install_environment(
-            prefix: Prefix,
-            version: str,
-            additional_dependencies: Sequence[str],
-    ) -> None:
-        directory = helpers.environment_dir(_dir, version)
-        install = ('python', '-mpip', 'install', '.', *additional_dependencies)
-
-        env_dir = prefix.path(directory)
-        with clean_path_on_failure(env_dir):
-            if version != C.DEFAULT:
-                python = norm_version(version)
-            else:
-                python = os.path.realpath(sys.executable)
-            _make_venv(env_dir, python)
-            with in_env(prefix, version):
-                helpers.run_setup_cmd(prefix, install)
-
-    return in_env, healthy, run_hook, install_environment
 
 
-def make_venv(envdir: str, python: str) -> None:
+def healthy(prefix: Prefix, language_version: str) -> bool:
-    env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
-    cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
+    envdir = prefix.path(directory)
-    cmd_output_b(*cmd, env=env, cwd='/')
+    pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg')
+
+    # created with "old" virtualenv
+    if not os.path.exists(pyvenv_cfg):
+        return False
+
+    exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
+    py_exe = prefix.path(bin_dir(envdir), exe_name)
+    cfg = _read_pyvenv_cfg(pyvenv_cfg)
+
+    return (
+        'version_info' in cfg and
+        _version_info(py_exe) == cfg['version_info'] and (
+            'base-executable' not in cfg or
+            _version_info(cfg['base-executable']) == cfg['version_info']
+        )
+    )
 
 
-_interface = py_interface(ENVIRONMENT_DIR, make_venv)
+def install_environment(
-in_env, healthy, run_hook, install_environment = _interface
+        prefix: Prefix,
+        version: str,
+        additional_dependencies: Sequence[str],
+) -> None:
+    envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
+    python = norm_version(version)
+    venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
+    install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies)
+
+    with clean_path_on_failure(envdir):
+        cmd_output_b(*venv_cmd, cwd='/')
+        with in_env(prefix, version):
+            helpers.run_setup_cmd(prefix, install_cmd)
+
+
+def run_hook(
+        hook: Hook,
+        file_args: Sequence[str],
+        color: bool,
+) -> Tuple[int, bytes]:
+    with in_env(hook.prefix, hook.language_version):
+        return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

pre_commit/languages/python_venv.py

@@ -1,46 +0,0 @@
-import os.path
-
-from pre_commit.languages import python
-from pre_commit.util import CalledProcessError
-from pre_commit.util import cmd_output
-from pre_commit.util import cmd_output_b
-
-ENVIRONMENT_DIR = 'py_venv'
-get_default_version = python.get_default_version
-
-
-def orig_py_exe(exe: str) -> str:  # pragma: no cover (platform specific)
-    """A -mvenv virtualenv made from a -mvirtualenv virtualenv installs
-    packages to the incorrect location.  Attempt to find the _original_ exe
-    and invoke `-mvenv` from there.
-
-    See:
-    - https://github.com/pre-commit/pre-commit/issues/755
-    - https://github.com/pypa/virtualenv/issues/1095
-    - https://bugs.python.org/issue30811
-    """
-    try:
-        prefix_script = 'import sys; print(sys.real_prefix)'
-        _, prefix, _ = cmd_output(exe, '-c', prefix_script)
-        prefix = prefix.strip()
-    except CalledProcessError:
-        # not created from -mvirtualenv
-        return exe
-
-    if os.name == 'nt':
-        expected = os.path.join(prefix, 'python.exe')
-    else:
-        expected = os.path.join(prefix, 'bin', os.path.basename(exe))
-
-    if os.path.exists(expected):
-        return expected
-    else:
-        return exe
-
-
-def make_venv(envdir: str, python: str) -> None:
-    cmd_output_b(orig_py_exe(python), '-mvenv', envdir, cwd='/')
-
-
-_interface = python.py_interface(ENVIRONMENT_DIR, make_venv)
-in_env, healthy, run_hook, install_environment = _interface

setup.cfg

@@ -27,7 +27,7 @@ install_requires =
     nodeenv>=0.11.1
     pyyaml>=5.1
     toml
-    virtualenv>=15.2
+    virtualenv>=20.0.8
     importlib-metadata;python_version<"3.8"
     importlib-resources;python_version<"3.7"
 python_requires = >=3.6.1

testing/gen-languages-all

@@ -3,8 +3,7 @@ import sys
 
 LANGUAGES = [
     'conda', 'docker', 'docker_image', 'fail', 'golang', 'node', 'perl',
-    'pygrep', 'python', 'python_venv', 'ruby', 'rust', 'script', 'swift',
+    'pygrep', 'python', 'ruby', 'rust', 'script', 'swift', 'system',
-    'system',
 ]
 FIELDS = [
     'ENVIRONMENT_DIR', 'get_default_version', 'healthy', 'install_environment',

testing/util.py

@@ -45,20 +45,6 @@ xfailif_windows_no_ruby = pytest.mark.xfail(
 xfailif_windows = pytest.mark.xfail(os.name == 'nt', reason='windows')
 
 
-def supports_venv():  # pragma: no cover (platform specific)
-    try:
-        __import__('ensurepip')
-        __import__('venv')
-        return True
-    except ImportError:
-        return False
-
-
-xfailif_no_venv = pytest.mark.xfail(
-    not supports_venv(), reason='Does not support venv module',
-)
-
-
 def run_opts(
         all_files=False,
         files=(),

tests/languages/python_test.py

@@ -5,10 +5,23 @@ from unittest import mock
 import pytest
 
 import pre_commit.constants as C
+from pre_commit.envcontext import envcontext
 from pre_commit.languages import python
 from pre_commit.prefix import Prefix
 
 
+def test_read_pyvenv_cfg(tmpdir):
+    pyvenv_cfg = tmpdir.join('pyvenv.cfg')
+    pyvenv_cfg.write(
+        '# I am a comment\n'
+        '\n'
+        'foo = bar\n'
+        'version-info=123\n',
+    )
+    expected = {'foo': 'bar', 'version-info': '123'}
+    assert python._read_pyvenv_cfg(pyvenv_cfg) == expected
+
+
 def test_norm_version_expanduser():
     home = os.path.expanduser('~')
     if os.name == 'nt':  # pragma: nt cover
@@ -21,6 +34,10 @@ def test_norm_version_expanduser():
     assert result == expected_path
 
 
+def test_norm_version_of_default_is_sys_executable():
+    assert python.norm_version('default') == os.path.realpath(sys.executable)
+
+
 @pytest.mark.parametrize('v', ('python3.6', 'python3', 'python'))
 def test_sys_executable_matches(v):
     with mock.patch.object(sys, 'version_info', (3, 6, 7)):
@@ -49,27 +66,78 @@ def test_find_by_sys_executable(exe, realpath, expected):
                 assert python._find_by_sys_executable() == expected
 
 
-def test_healthy_types_py_in_cwd(tmpdir):
+@pytest.fixture
+def python_dir(tmpdir):
     with tmpdir.as_cwd():
         prefix = tmpdir.join('prefix').ensure_dir()
         prefix.join('setup.py').write('import setuptools; setuptools.setup()')
         prefix = Prefix(str(prefix))
+        yield prefix, tmpdir
+
+
+def test_healthy_default_creator(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    # should be healthy right after creation
+    assert python.healthy(prefix, C.DEFAULT) is True
+
+    # even if a `types.py` file exists, should still be healthy
+    tmpdir.join('types.py').ensure()
+    assert python.healthy(prefix, C.DEFAULT) is True
+
+
+def test_healthy_venv_creator(python_dir):
+    # venv creator produces slightly different pyvenv.cfg
+    prefix, tmpdir = python_dir
+
+    with envcontext((('VIRTUALENV_CREATOR', 'venv'),)):
         python.install_environment(prefix, C.DEFAULT, ())
 
-        # even if a `types.py` file exists, should still be healthy
+    assert python.healthy(prefix, C.DEFAULT) is True
-        tmpdir.join('types.py').ensure()
-        assert python.healthy(prefix, C.DEFAULT) is True
 
 
-def test_healthy_python_goes_missing(tmpdir):
+def test_unhealthy_python_goes_missing(python_dir):
-    with tmpdir.as_cwd():
+    prefix, tmpdir = python_dir
-        prefix = tmpdir.join('prefix').ensure_dir()
-        prefix.join('setup.py').write('import setuptools; setuptools.setup()')
-        prefix = Prefix(str(prefix))
-        python.install_environment(prefix, C.DEFAULT, ())
 
-        exe_name = 'python' if sys.platform != 'win32' else 'python.exe'
+    python.install_environment(prefix, C.DEFAULT, ())
-        py_exe = prefix.path(python.bin_dir('py_env-default'), exe_name)
-        os.remove(py_exe)
 
-        assert python.healthy(prefix, C.DEFAULT) is False
+    exe_name = 'python' if sys.platform != 'win32' else 'python.exe'
+    py_exe = prefix.path(python.bin_dir('py_env-default'), exe_name)
+    os.remove(py_exe)
+
+    assert python.healthy(prefix, C.DEFAULT) is False
+
+
+def test_unhealthy_with_version_change(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    with open(prefix.path('py_env-default/pyvenv.cfg'), 'w') as f:
+        f.write('version_info = 1.2.3\n')
+
+    assert python.healthy(prefix, C.DEFAULT) is False
+
+
+def test_unhealthy_system_version_changes(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    with open(prefix.path('py_env-default/pyvenv.cfg'), 'a') as f:
+        f.write('base-executable = /does/not/exist\n')
+
+    assert python.healthy(prefix, C.DEFAULT) is False
+
+
+def test_unhealthy_old_virtualenv(python_dir):
+    prefix, tmpdir = python_dir
+
+    python.install_environment(prefix, C.DEFAULT, ())
+
+    # simulate "old" virtualenv by deleting this file
+    os.remove(prefix.path('py_env-default/pyvenv.cfg'))
+
+    assert python.healthy(prefix, C.DEFAULT) is False

tests/repository_test.py

@@ -33,7 +33,6 @@ from testing.util import cwd
 from testing.util import get_resource_path
 from testing.util import skipif_cant_run_docker
 from testing.util import skipif_cant_run_swift
-from testing.util import xfailif_no_venv
 from testing.util import xfailif_windows_no_ruby
 
 
@@ -163,7 +162,6 @@ def test_python_hook_weird_setup_cfg(in_git_dir, tempdir_factory, store):
     )
 
 
-@xfailif_no_venv
 def test_python_venv(tempdir_factory, store):  # pragma: no cover (no venv)
     _test_hook_repo(
         tempdir_factory, store, 'python_venv_hooks_repo',