diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 4b7580a4..c9b924ae 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -79,6 +79,136 @@ def _get_default_version(): # pragma: no cover (platform dependent) return 'default' +def _parse_requirements_file(requirements_file): + options = [] + collected_requirements = [] + with open(requirements_file) as rfh: + for line in rfh: + req = line.strip() + if not req: + continue + if req.startswith('#'): + continue + if req.startswith(('-r ', '--requirement ')): + _, req_file = req.split(' ', 1) + req_file = os.path.realpath( + os.path.join(os.path.dirname(requirements_file), req_file), + ) + if not os.path.isfile(req_file): + continue + for rreq in _parse_requirements_file(req_file): + if rreq in collected_requirements: + continue + collected_requirements.append(rreq) + continue + if req.startswith('-r'): + req_file = req[:2] + req_file = os.path.realpath( + os.path.join(os.path.dirname(requirements_file), req_file), + ) + if not os.path.isfile(req_file): + continue + for rreq in _parse_requirements_file(req_file): + if rreq in collected_requirements: + continue + collected_requirements.append(rreq) + continue + if req.startswith('--requirement='): + req_file = req[:14] + req_file = os.path.realpath( + os.path.join(os.path.dirname(requirements_file), req_file), + ) + if not os.path.isfile(req_file): + continue + for rreq in _parse_requirements_file(req_file): + if rreq in collected_requirements: + continue + collected_requirements.append(rreq) + continue + if req.startswith('--'): + if req in options: + continue + options.append(req) + continue + if req in collected_requirements: + continue + collected_requirements.append(req) + return options + collected_requirements + + +def collect_requirements(git_root, additional_dependencies): + options = [] + collected_requirements = [] + next_is_requirements_file = False + for dep in additional_dependencies: + if dep in ('-r', '--requirement'): + # pip install -r requirement.txt or + # pip install --requirement requirement.txt + next_is_requirements_file = True + continue + elif dep.startswith('-r'): + # pip install -rrequirement.txt + requirements_file = os.path.join(git_root, dep[2:]) + if not os.path.isfile(requirements_file): + print('Not a requirements_file: {}'.format(requirements_file)) + continue + for rdep in _parse_requirements_file(requirements_file): + if rdep.startswith('--'): + for part in rdep.split(): + if not part: + continue + if part in options: + continue + options.append(part) + continue + if rdep in collected_requirements: + continue + collected_requirements.append(rdep) + elif dep.startswith('--requirement='): + # pip install --requirement=requirement.txt + requirements_file = os.path.join(git_root, dep[14:]) + if not os.path.isfile(requirements_file): + print('Not a requirements_file: {}'.format(requirements_file)) + continue + for rdep in _parse_requirements_file(requirements_file): + if rdep.startswith('--'): + for part in rdep.split(): + if not part: + continue + if part in options: + continue + options.append(part) + continue + if rdep in collected_requirements: + continue + collected_requirements.append(rdep) + continue + elif dep.startswith('--'): + options.append(dep) + continue + elif next_is_requirements_file: + next_is_requirements_file = False + requirements_file = os.path.join(git_root, dep) + if not os.path.isfile(requirements_file): + print('Not a requirements_file: {}'.format(requirements_file)) + continue + for rdep in _parse_requirements_file(requirements_file): + if rdep.startswith('--'): + for part in rdep.split(): + if not part: + continue + if part in options: + continue + options.append(part) + continue + if rdep in collected_requirements: + continue + collected_requirements.append(rdep) + else: + collected_requirements.append(dep) + return options + collected_requirements + + def get_default_version(): # TODO: when dropping python2, use `functools.lru_cache(maxsize=1)` try: @@ -108,6 +238,11 @@ def norm_version(version): return os.path.expanduser(version) +def process_additional_dependencies(additional_dependencies): + git_root = os.path.abspath(os.getcwd()) + return collect_requirements(git_root, additional_dependencies) + + def py_interface(_dir, _make_venv): @contextlib.contextmanager def in_env(prefix, language_version): diff --git a/pre_commit/repository.py b/pre_commit/repository.py index 2a435506..8bdd9735 100644 --- a/pre_commit/repository.py +++ b/pre_commit/repository.py @@ -183,7 +183,15 @@ class Repository(object): for _, hook in self.hooks: language = hook['language'] version = hook['language_version'] - deps = tuple(hook['additional_dependencies']) + additional_dependencies = hook['additional_dependencies'] + try: + deps = languages[language].process_additional_dependencies( + additional_dependencies, + ) + except AttributeError: + # Language does not implement process_additional_dependencies + deps = additional_dependencies + deps = tuple(deps) ret.add(( self._prefix_from_deps(language, deps), language, version, deps, diff --git a/testing/fixtures.py b/testing/fixtures.py index 5e6de5c8..3bdaf3db 100644 --- a/testing/fixtures.py +++ b/testing/fixtures.py @@ -64,7 +64,8 @@ def modify_manifest(path): with io.open(manifest_path, 'w') as manifest_file: manifest_file.write(ordered_dump(manifest, **C.YAML_DUMP_KWARGS)) cmd_output( - 'git', 'commit', '--no-gpg-sign', '-am', 'update {}'.format(C.MANIFEST_FILE), cwd=path, + 'git', 'commit', '--no-gpg-sign', '-am', + 'update {}'.format(C.MANIFEST_FILE), cwd=path, ) @@ -80,7 +81,10 @@ def modify_config(path='.', commit=True): with io.open(config_path, 'w', encoding='UTF-8') as config_file: config_file.write(ordered_dump(config, **C.YAML_DUMP_KWARGS)) if commit: - cmd_output('git', 'commit', '--no-gpg-sign', '-am', 'update config', cwd=path) + cmd_output( + 'git', 'commit', '--no-gpg-sign', '-am', 'update config', + cwd=path, + ) def config_with_local_hooks(): @@ -136,13 +140,19 @@ def write_config(directory, config, config_file=C.CONFIG_FILE): def add_config_to_repo(git_path, config, config_file=C.CONFIG_FILE): write_config(git_path, config, config_file=config_file) cmd_output('git', 'add', config_file, cwd=git_path) - cmd_output('git', 'commit', '--no-gpg-sign', '-m', 'Add hooks config', cwd=git_path) + cmd_output( + 'git', 'commit', '--no-gpg-sign', '-m', 'Add hooks config', + cwd=git_path, + ) return git_path def remove_config_from_repo(git_path, config_file=C.CONFIG_FILE): cmd_output('git', 'rm', config_file, cwd=git_path) - cmd_output('git', 'commit', '--no-gpg-sign', '-m', 'Remove hooks config', cwd=git_path) + cmd_output( + 'git', 'commit', '--no-gpg-sign', '-m', 'Remove hooks config', + cwd=git_path, + ) return git_path diff --git a/tests/languages/python_test.py b/tests/languages/python_test.py index 78211cb9..78853368 100644 --- a/tests/languages/python_test.py +++ b/tests/languages/python_test.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import os.path +import textwrap from pre_commit.languages import python @@ -16,3 +17,205 @@ def test_norm_version_expanduser(): expected_path = home + '/.pyenv/versions/3.4.3/bin/python' result = python.norm_version(path) assert result == expected_path + + +def test_single_requirements_file(tempdir_factory): + tmpdir = tempdir_factory.get() + req1 = os.path.join(tmpdir, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pep8 + ''')) + assert python.collect_requirements( + tmpdir, ['-r', 'req1.txt'], + ) == ['pep8'] + assert python.collect_requirements( + tmpdir, ['-rreq1.txt'], + ) == ['pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'req1.txt'], + ) == ['pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement=req1.txt'], + ) == ['pep8'] + + +def test_multiple_requirements_file(tempdir_factory): + tmpdir = tempdir_factory.get() + req1 = os.path.join(tmpdir, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pep8 + ''')) + req2 = os.path.join(tmpdir, 'req2.txt') + with open(req2, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pre-commit + ''')) + assert python.collect_requirements( + tmpdir, ['-r', 'req1.txt', '-r', 'req2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['-rreq1.txt', '-rreq2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'req1.txt', '--requirement', 'req2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['--requirement=req1.txt', '--requirement=req2.txt'], + ) == ['pep8', 'pre-commit'] + + +def test_nested_requirements_file(tempdir_factory): + tmpdir = tempdir_factory.get() + req1 = os.path.join(tmpdir, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pep8 + ''')) + req2 = os.path.join(tmpdir, 'req2.txt') + with open(req2, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + -r req1.txt + pre-commit + ''')) + assert python.collect_requirements( + tmpdir, ['-r', 'req2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['-rreq2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'req2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['--requirement=req2.txt'], + ) == ['pep8', 'pre-commit'] + + +def test_nested_requirements_files_subdir(tempdir_factory): + tmpdir = tempdir_factory.get() + req1 = os.path.join(tmpdir, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pep8 + ''')) + reqsdir = os.path.join(tmpdir, 'requirements') + os.makedirs(reqsdir) + req2 = os.path.join(reqsdir, 'req2.txt') + with open(req2, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + -r ../req1.txt + pre-commit + ''')) + assert python.collect_requirements( + tmpdir, ['-r', 'requirements/req2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['-rrequirements/req2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'requirements/req2.txt'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['--requirement=requirements/req2.txt'], + ) == ['pep8', 'pre-commit'] + + +def test_mixed_requirements(tempdir_factory): + tmpdir = tempdir_factory.get() + req1 = os.path.join(tmpdir, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pep8 + ''')) + assert python.collect_requirements( + tmpdir, ['pre-commit', '-r', 'req1.txt'], + ) == ['pre-commit', 'pep8'] + assert python.collect_requirements( + tmpdir, ['-rreq1.txt', 'pre-commit'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'req1.txt', 'pre-commit'], + ) == ['pep8', 'pre-commit'] + assert python.collect_requirements( + tmpdir, ['pre-commit', '--requirement=req1.txt'], + ) == ['pre-commit', 'pep8'] + + +def test_options_in_requirements_file(tempdir_factory): + tmpdir = tempdir_factory.get() + req1 = os.path.join(tmpdir, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + --index-url=https://domain.tld/repository/pypi/simple/ + pep8 + ''')) + assert python.collect_requirements( + tmpdir, ['-r', 'req1.txt'], + ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8'] + assert python.collect_requirements( + tmpdir, ['-rreq1.txt'], + ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'req1.txt'], + ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement=req1.txt'], + ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8'] + assert python.collect_requirements( + tmpdir, + [ + '--index-url=https://domain.tld/repository/pypi/simple/', + '-r', 'req1.txt', + ], + ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8'] + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + --index-url https://domain.tld/repository/pypi/simple/ + pep8 + ''')) + assert python.collect_requirements( + tmpdir, ['-r', 'req1.txt'], + ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8'] + assert python.collect_requirements( + tmpdir, ['-rreq1.txt'], + ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'req1.txt'], + ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement=req1.txt'], + ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8'] + + +def test_missing_requirements_file(tempdir_factory): + tmpdir = tempdir_factory.get() + req1 = os.path.join(tmpdir, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pep8 + ''')) + assert python.collect_requirements( + tmpdir, ['-r', 'req1.txt', '-r', 'req2.txt'], + ) == ['pep8'] + assert python.collect_requirements( + tmpdir, ['-rreq1.txt', '-rreq2.txt'], + ) == ['pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement', 'req1.txt', '--requirement', 'req2.txt'], + ) == ['pep8'] + assert python.collect_requirements( + tmpdir, ['--requirement=req1.txt', '--requirement=req2.txt'], + ) == ['pep8'] diff --git a/tests/repository_test.py b/tests/repository_test.py index f1b0f6e0..ebbaf9c9 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -6,6 +6,7 @@ import io import os.path import re import shutil +import textwrap import mock import pytest @@ -843,3 +844,50 @@ def test_manifest_hooks(tempdir_factory, store): 'exclude_types': [], 'verbose': False, } + + +def test_python_additional_dependencies_requirements_file( + tempdir_factory, + store, +): + path = make_repo(tempdir_factory, 'python_hooks_repo') + req1 = os.path.join(path, 'req1.txt') + with open(req1, 'w') as wfh: + wfh.write(textwrap.dedent(''' + # This is a comment in the pip file + pep8 + ''')) + with cwd(path): + config = make_config_from_repo(path) + config['hooks'][0]['additional_dependencies'] = ['-r', 'req1.txt'] + repo = Repository.create(config, store) + env, = repo._venvs() + assert env == ( + mock.ANY, 'python', python.get_default_version(), ('pep8',), + ) + config = make_config_from_repo(path) + config['hooks'][0]['additional_dependencies'] = ['-rreq1.txt'] + repo = Repository.create(config, store) + env, = repo._venvs() + assert env == ( + mock.ANY, 'python', python.get_default_version(), ('pep8',), + ) + config = make_config_from_repo(path) + config['hooks'][0]['additional_dependencies'] = [ + '--requirement', + 'req1.txt', + ] + repo = Repository.create(config, store) + env, = repo._venvs() + assert env == ( + mock.ANY, 'python', python.get_default_version(), ('pep8',), + ) + config = make_config_from_repo(path) + config['hooks'][0]['additional_dependencies'] = [ + '--requirement=req1.txt', + ] + repo = Repository.create(config, store) + env, = repo._venvs() + assert env == ( + mock.ANY, 'python', python.get_default_version(), ('pep8',), + )