From d6825fa0fca4a309f99d5a733eecbe9d5d8abdbd Mon Sep 17 00:00:00 2001 From: Jonas Obrist Date: Fri, 27 Apr 2018 17:13:47 +0900 Subject: [PATCH 1/3] added python venv language --- pre_commit/languages/all.py | 2 + pre_commit/languages/python_venv.py | 73 +++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 pre_commit/languages/python_venv.py diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index a56f7e79..504c28a0 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -7,6 +7,7 @@ from pre_commit.languages import node from pre_commit.languages import pcre from pre_commit.languages import pygrep from pre_commit.languages import python +from pre_commit.languages import python_venv from pre_commit.languages import ruby from pre_commit.languages import script from pre_commit.languages import swift @@ -57,6 +58,7 @@ languages = { 'pcre': pcre, 'pygrep': pygrep, 'python': python, + 'python_venv': python_venv, 'ruby': ruby, 'script': script, 'swift': swift, diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py new file mode 100644 index 00000000..26389c65 --- /dev/null +++ b/pre_commit/languages/python_venv.py @@ -0,0 +1,73 @@ +from __future__ import unicode_literals + +import contextlib +import os +import sys + +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import UNSET +from pre_commit.envcontext import Var +from pre_commit.languages import helpers +from pre_commit.languages.python import get_default_version # noqa: F401 +from pre_commit.languages.python import norm_version +from pre_commit.util import clean_path_on_failure +from pre_commit.util import cmd_output +from pre_commit.xargs import xargs + + +ENVIRONMENT_DIR = 'py_venv' + + +def bin_dir(venv): + """On windows there's a different directory for the virtualenv""" + bin_part = 'Scripts' if os.name == 'nt' else 'bin' + return os.path.join(venv, bin_part) + + +def get_env_patch(venv): + return ( + ('PYTHONHOME', UNSET), + ('VIRTUAL_ENV', venv), + ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))), + ) + + +@contextlib.contextmanager +def in_env(prefix, language_version): + envdir = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, language_version), + ) + with envcontext(get_env_patch(envdir)): + yield + + +def healthy(prefix, language_version): + with in_env(prefix, language_version): + retcode, _, _ = cmd_output( + 'python', '-c', 'import ctypes, datetime, io, os, ssl, weakref', + retcode=None, + ) + return retcode == 0 + + +def install_environment(prefix, version, additional_dependencies): + additional_dependencies = tuple(additional_dependencies) + directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + + # Install a virtualenv + env_dir = prefix.path(directory) + with clean_path_on_failure(env_dir): + if version != 'default': + executable = norm_version(version) + else: + executable = os.path.realpath(sys.executable) + cmd_output(executable, '-m', 'venv', env_dir, cwd='/') + with in_env(prefix, version): + helpers.run_setup_cmd( + prefix, ('pip', 'install', '.') + additional_dependencies, + ) + + +def run_hook(prefix, hook, file_args): + with in_env(prefix, hook['language_version']): + return xargs(helpers.to_cmd(hook), file_args) From e8954e2bf3c02bef1eae5ea4d3ff36e026b74c90 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 12 May 2018 09:26:20 -0700 Subject: [PATCH 2/3] Simplify python_venv interface --- pre_commit/languages/python.py | 82 ++++++++++++++++------------- pre_commit/languages/python_venv.py | 69 +++--------------------- 2 files changed, 50 insertions(+), 101 deletions(-) diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 7fc5443e..0840b900 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -16,6 +16,7 @@ from pre_commit.xargs import xargs ENVIRONMENT_DIR = 'py_env' +HEALTH_MODS = ('datetime', 'io', 'os', 'ssl', 'weakref') def bin_dir(venv): @@ -32,15 +33,6 @@ def get_env_patch(venv): ) -@contextlib.contextmanager -def in_env(prefix, language_version): - envdir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, language_version), - ) - with envcontext(get_env_patch(envdir)): - yield - - def _find_by_py_launcher(version): # pragma: no cover (windows only) if version.startswith('python'): try: @@ -98,15 +90,6 @@ def get_default_version(): return get_default_version() -def healthy(prefix, language_version): - with in_env(prefix, language_version): - retcode, _, _ = cmd_output( - 'python', '-c', 'import ctypes, datetime, io, os, ssl, weakref', - retcode=None, - ) - return retcode == 0 - - def norm_version(version): if os.name == 'nt': # pragma: no cover (windows) # Try looking up by name @@ -123,30 +106,53 @@ def norm_version(version): if version.startswith('python'): return r'C:\{}\python.exe'.format(version.replace('.', '')) - # Otherwise assume it is a path + # Otherwise assume it is a path return os.path.expanduser(version) -def install_environment(prefix, version, additional_dependencies): - additional_dependencies = tuple(additional_dependencies) - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) +def py_interface(_dir, _make_venv): + @contextlib.contextmanager + def in_env(prefix, language_version): + envdir = prefix.path(helpers.environment_dir(_dir, language_version)) + with envcontext(get_env_patch(envdir)): + yield - # Install a virtualenv - env_dir = prefix.path(directory) - with clean_path_on_failure(env_dir): - venv_cmd = [sys.executable, '-m', 'virtualenv', env_dir] - if version != 'default': - venv_cmd.extend(['-p', norm_version(version)]) - else: - venv_cmd.extend(['-p', os.path.realpath(sys.executable)]) - venv_env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') - cmd_output(*venv_cmd, cwd='/', env=venv_env) - with in_env(prefix, version): - helpers.run_setup_cmd( - prefix, ('pip', 'install', '.') + additional_dependencies, + def healthy(prefix, language_version): + with in_env(prefix, language_version): + retcode, _, _ = cmd_output( + 'python', '-c', 'import {}'.format(','.join(HEALTH_MODS)), + retcode=None, ) + return retcode == 0 + + def run_hook(prefix, hook, file_args): + with in_env(prefix, hook['language_version']): + return xargs(helpers.to_cmd(hook), file_args) + + def install_environment(prefix, version, additional_dependencies): + additional_dependencies = tuple(additional_dependencies) + directory = helpers.environment_dir(_dir, version) + + env_dir = prefix.path(directory) + with clean_path_on_failure(env_dir): + if version != 'default': + python = norm_version(version) + else: + python = os.path.realpath(sys.executable) + _make_venv(env_dir, python) + with in_env(prefix, version): + helpers.run_setup_cmd( + prefix, ('pip', 'install', '.') + additional_dependencies, + ) + + return in_env, healthy, run_hook, install_environment -def run_hook(prefix, hook, file_args): - with in_env(prefix, hook['language_version']): - return xargs(helpers.to_cmd(hook), file_args) +def make_venv(envdir, python): + env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') + cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) + cmd_output(*cmd, env=env, cwd='/') + + +_interface = py_interface(ENVIRONMENT_DIR, make_venv) +in_env, healthy, run_hook, install_environment = _interface diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py index 26389c65..20613a49 100644 --- a/pre_commit/languages/python_venv.py +++ b/pre_commit/languages/python_venv.py @@ -1,73 +1,16 @@ from __future__ import unicode_literals -import contextlib -import os -import sys - -from pre_commit.envcontext import envcontext -from pre_commit.envcontext import UNSET -from pre_commit.envcontext import Var -from pre_commit.languages import helpers -from pre_commit.languages.python import get_default_version # noqa: F401 -from pre_commit.languages.python import norm_version -from pre_commit.util import clean_path_on_failure +from pre_commit.languages import python from pre_commit.util import cmd_output -from pre_commit.xargs import xargs ENVIRONMENT_DIR = 'py_venv' -def bin_dir(venv): - """On windows there's a different directory for the virtualenv""" - bin_part = 'Scripts' if os.name == 'nt' else 'bin' - return os.path.join(venv, bin_part) +def make_venv(envdir, python): + cmd_output(python, '-mvenv', envdir, cwd='/') -def get_env_patch(venv): - return ( - ('PYTHONHOME', UNSET), - ('VIRTUAL_ENV', venv), - ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))), - ) - - -@contextlib.contextmanager -def in_env(prefix, language_version): - envdir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, language_version), - ) - with envcontext(get_env_patch(envdir)): - yield - - -def healthy(prefix, language_version): - with in_env(prefix, language_version): - retcode, _, _ = cmd_output( - 'python', '-c', 'import ctypes, datetime, io, os, ssl, weakref', - retcode=None, - ) - return retcode == 0 - - -def install_environment(prefix, version, additional_dependencies): - additional_dependencies = tuple(additional_dependencies) - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - - # Install a virtualenv - env_dir = prefix.path(directory) - with clean_path_on_failure(env_dir): - if version != 'default': - executable = norm_version(version) - else: - executable = os.path.realpath(sys.executable) - cmd_output(executable, '-m', 'venv', env_dir, cwd='/') - with in_env(prefix, version): - helpers.run_setup_cmd( - prefix, ('pip', 'install', '.') + additional_dependencies, - ) - - -def run_hook(prefix, hook, file_args): - with in_env(prefix, hook['language_version']): - return xargs(helpers.to_cmd(hook), file_args) +get_default_version = python.get_default_version +_interface = python.py_interface(ENVIRONMENT_DIR, make_venv) +in_env, healthy, run_hook, install_environment = _interface From b5af5a5b2761a09e8164e38dd520cd59cdd23ed0 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 12 May 2018 10:21:46 -0700 Subject: [PATCH 3/3] Add test for python_venv language --- pre_commit/languages/python.py | 4 ++-- .../{python3_hook/main.py => py3_hook.py} | 2 +- testing/resources/python3_hooks_repo/setup.py | 7 ++----- .../python_hooks_repo/{foo/main.py => foo.py} | 2 +- .../resources/python_hooks_repo/foo/__init__.py | 0 testing/resources/python_hooks_repo/setup.py | 9 +++------ .../python_venv_hooks_repo/.pre-commit-hooks.yaml | 5 +++++ testing/resources/python_venv_hooks_repo/foo.py | 9 +++++++++ .../foo}/__init__.py | 0 testing/resources/python_venv_hooks_repo/setup.py | 8 ++++++++ testing/util.py | 14 ++++++++++++++ tests/repository_test.py | 10 ++++++++++ 12 files changed, 55 insertions(+), 15 deletions(-) rename testing/resources/python3_hooks_repo/{python3_hook/main.py => py3_hook.py} (92%) rename testing/resources/python_hooks_repo/{foo/main.py => foo.py} (90%) delete mode 100644 testing/resources/python_hooks_repo/foo/__init__.py create mode 100644 testing/resources/python_venv_hooks_repo/.pre-commit-hooks.yaml create mode 100644 testing/resources/python_venv_hooks_repo/foo.py rename testing/resources/{python3_hooks_repo/python3_hook => python_venv_hooks_repo/foo}/__init__.py (100%) create mode 100644 testing/resources/python_venv_hooks_repo/setup.py diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 0840b900..ee7b2a4f 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -16,7 +16,6 @@ from pre_commit.xargs import xargs ENVIRONMENT_DIR = 'py_env' -HEALTH_MODS = ('datetime', 'io', 'os', 'ssl', 'weakref') def bin_dir(venv): @@ -120,7 +119,8 @@ def py_interface(_dir, _make_venv): def healthy(prefix, language_version): with in_env(prefix, language_version): retcode, _, _ = cmd_output( - 'python', '-c', 'import {}'.format(','.join(HEALTH_MODS)), + 'python', '-c', + 'import ctypes, datetime, io, os, ssl, weakref', retcode=None, ) return retcode == 0 diff --git a/testing/resources/python3_hooks_repo/python3_hook/main.py b/testing/resources/python3_hooks_repo/py3_hook.py similarity index 92% rename from testing/resources/python3_hooks_repo/python3_hook/main.py rename to testing/resources/python3_hooks_repo/py3_hook.py index 04f974e6..f0f88088 100644 --- a/testing/resources/python3_hooks_repo/python3_hook/main.py +++ b/testing/resources/python3_hooks_repo/py3_hook.py @@ -3,7 +3,7 @@ from __future__ import print_function import sys -def func(): +def main(): print(sys.version_info[0]) print(repr(sys.argv[1:])) print('Hello World') diff --git a/testing/resources/python3_hooks_repo/setup.py b/testing/resources/python3_hooks_repo/setup.py index bf7690c0..9125dc1d 100644 --- a/testing/resources/python3_hooks_repo/setup.py +++ b/testing/resources/python3_hooks_repo/setup.py @@ -1,11 +1,8 @@ -from setuptools import find_packages from setuptools import setup setup( name='python3_hook', version='0.0.0', - packages=find_packages('.'), - entry_points={ - 'console_scripts': ['python3-hook = python3_hook.main:func'], - }, + py_modules=['py3_hook'], + entry_points={'console_scripts': ['python3-hook = py3_hook:main']}, ) diff --git a/testing/resources/python_hooks_repo/foo/main.py b/testing/resources/python_hooks_repo/foo.py similarity index 90% rename from testing/resources/python_hooks_repo/foo/main.py rename to testing/resources/python_hooks_repo/foo.py index 78c2c0f7..412a5c62 100644 --- a/testing/resources/python_hooks_repo/foo/main.py +++ b/testing/resources/python_hooks_repo/foo.py @@ -3,7 +3,7 @@ from __future__ import print_function import sys -def func(): +def main(): print(repr(sys.argv[1:])) print('Hello World') return 0 diff --git a/testing/resources/python_hooks_repo/foo/__init__.py b/testing/resources/python_hooks_repo/foo/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testing/resources/python_hooks_repo/setup.py b/testing/resources/python_hooks_repo/setup.py index 556dd8f5..0559271e 100644 --- a/testing/resources/python_hooks_repo/setup.py +++ b/testing/resources/python_hooks_repo/setup.py @@ -1,11 +1,8 @@ -from setuptools import find_packages from setuptools import setup setup( - name='Foo', + name='foo', version='0.0.0', - packages=find_packages('.'), - entry_points={ - 'console_scripts': ['foo = foo.main:func'], - }, + py_modules=['foo'], + entry_points={'console_scripts': ['foo = foo:main']}, ) diff --git a/testing/resources/python_venv_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/python_venv_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 00000000..a666ed87 --- /dev/null +++ b/testing/resources/python_venv_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,5 @@ +- id: foo + name: Foo + entry: foo + language: python_venv + files: \.py$ diff --git a/testing/resources/python_venv_hooks_repo/foo.py b/testing/resources/python_venv_hooks_repo/foo.py new file mode 100644 index 00000000..412a5c62 --- /dev/null +++ b/testing/resources/python_venv_hooks_repo/foo.py @@ -0,0 +1,9 @@ +from __future__ import print_function + +import sys + + +def main(): + print(repr(sys.argv[1:])) + print('Hello World') + return 0 diff --git a/testing/resources/python3_hooks_repo/python3_hook/__init__.py b/testing/resources/python_venv_hooks_repo/foo/__init__.py similarity index 100% rename from testing/resources/python3_hooks_repo/python3_hook/__init__.py rename to testing/resources/python_venv_hooks_repo/foo/__init__.py diff --git a/testing/resources/python_venv_hooks_repo/setup.py b/testing/resources/python_venv_hooks_repo/setup.py new file mode 100644 index 00000000..0559271e --- /dev/null +++ b/testing/resources/python_venv_hooks_repo/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup + +setup( + name='foo', + version='0.0.0', + py_modules=['foo'], + entry_points={'console_scripts': ['foo = foo:main']}, +) diff --git a/testing/util.py b/testing/util.py index 025bc0bb..ae5ae338 100644 --- a/testing/util.py +++ b/testing/util.py @@ -78,6 +78,20 @@ xfailif_no_symlink = pytest.mark.xfail( ) +def supports_venv(): # pragma: no cover (platform specific) + try: + __import__('ensurepip') + __import__('venv') + return True + except ImportError: + return False + + +xfailif_no_venv = pytest.mark.xfail( + not supports_venv(), reason='Does not support venv module', +) + + def run_opts( all_files=False, files=(), diff --git a/tests/repository_test.py b/tests/repository_test.py index 63b9f1c9..67b8f3f6 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -33,6 +33,7 @@ from testing.util import skipif_cant_run_docker from testing.util import skipif_cant_run_swift from testing.util import xfailif_broken_deep_listdir from testing.util import xfailif_no_pcre_support +from testing.util import xfailif_no_venv from testing.util import xfailif_windows_no_ruby @@ -111,6 +112,15 @@ def test_python_hook_weird_setup_cfg(tempdir_factory, store): ) +@xfailif_no_venv +def test_python_venv(tempdir_factory, store): # pragma: no cover (no venv) + _test_hook_repo( + tempdir_factory, store, 'python_venv_hooks_repo', + 'foo', [os.devnull], + b"['" + five.to_bytes(os.devnull) + b"']\nHello World\n", + ) + + @pytest.mark.integration def test_switch_language_versions_doesnt_clobber(tempdir_factory, store): # We're using the python3 repo because it prints the python version