Add support for requirements files in additional_dependencies

pre_commit/languages/python.py

@@ -79,6 +79,136 @@ def _get_default_version():  # pragma: no cover (platform dependent)
     return 'default'
 
 
+def _parse_requirements_file(requirements_file):
+    options = []
+    collected_requirements = []
+    with open(requirements_file) as rfh:
+        for line in rfh:
+            req = line.strip()
+            if not req:
+                continue
+            if req.startswith('#'):
+                continue
+            if req.startswith(('-r ', '--requirement ')):
+                _, req_file = req.split(' ', 1)
+                req_file = os.path.realpath(
+                    os.path.join(os.path.dirname(requirements_file), req_file),
+                )
+                if not os.path.isfile(req_file):
+                    continue
+                for rreq in _parse_requirements_file(req_file):
+                    if rreq in collected_requirements:
+                        continue
+                    collected_requirements.append(rreq)
+                continue
+            if req.startswith('-r'):
+                req_file = req[:2]
+                req_file = os.path.realpath(
+                    os.path.join(os.path.dirname(requirements_file), req_file),
+                )
+                if not os.path.isfile(req_file):
+                    continue
+                for rreq in _parse_requirements_file(req_file):
+                    if rreq in collected_requirements:
+                        continue
+                    collected_requirements.append(rreq)
+                continue
+            if req.startswith('--requirement='):
+                req_file = req[:14]
+                req_file = os.path.realpath(
+                    os.path.join(os.path.dirname(requirements_file), req_file),
+                )
+                if not os.path.isfile(req_file):
+                    continue
+                for rreq in _parse_requirements_file(req_file):
+                    if rreq in collected_requirements:
+                        continue
+                    collected_requirements.append(rreq)
+                continue
+            if req.startswith('--'):
+                if req in options:
+                    continue
+                options.append(req)
+                continue
+            if req in collected_requirements:
+                continue
+            collected_requirements.append(req)
+    return options + collected_requirements
+
+
+def collect_requirements(git_root, additional_dependencies):
+    options = []
+    collected_requirements = []
+    next_is_requirements_file = False
+    for dep in additional_dependencies:
+        if dep in ('-r', '--requirement'):
+            # pip install -r requirement.txt or
+            # pip install --requirement requirement.txt
+            next_is_requirements_file = True
+            continue
+        elif dep.startswith('-r'):
+            # pip install -rrequirement.txt
+            requirements_file = os.path.join(git_root, dep[2:])
+            if not os.path.isfile(requirements_file):
+                print('Not a requirements_file: {}'.format(requirements_file))
+                continue
+            for rdep in _parse_requirements_file(requirements_file):
+                if rdep.startswith('--'):
+                    for part in rdep.split():
+                        if not part:
+                            continue
+                        if part in options:
+                            continue
+                        options.append(part)
+                    continue
+                if rdep in collected_requirements:
+                    continue
+                collected_requirements.append(rdep)
+        elif dep.startswith('--requirement='):
+            # pip install --requirement=requirement.txt
+            requirements_file = os.path.join(git_root, dep[14:])
+            if not os.path.isfile(requirements_file):
+                print('Not a requirements_file: {}'.format(requirements_file))
+                continue
+            for rdep in _parse_requirements_file(requirements_file):
+                if rdep.startswith('--'):
+                    for part in rdep.split():
+                        if not part:
+                            continue
+                        if part in options:
+                            continue
+                        options.append(part)
+                    continue
+                if rdep in collected_requirements:
+                    continue
+                collected_requirements.append(rdep)
+            continue
+        elif dep.startswith('--'):
+            options.append(dep)
+            continue
+        elif next_is_requirements_file:
+            next_is_requirements_file = False
+            requirements_file = os.path.join(git_root, dep)
+            if not os.path.isfile(requirements_file):
+                print('Not a requirements_file: {}'.format(requirements_file))
+                continue
+            for rdep in _parse_requirements_file(requirements_file):
+                if rdep.startswith('--'):
+                    for part in rdep.split():
+                        if not part:
+                            continue
+                        if part in options:
+                            continue
+                        options.append(part)
+                    continue
+                if rdep in collected_requirements:
+                    continue
+                collected_requirements.append(rdep)
+        else:
+            collected_requirements.append(dep)
+    return options + collected_requirements
+
+
 def get_default_version():
     # TODO: when dropping python2, use `functools.lru_cache(maxsize=1)`
     try:
@@ -108,6 +238,11 @@ def norm_version(version):
     return os.path.expanduser(version)
 
 
+def process_additional_dependencies(additional_dependencies):
+    git_root = os.path.abspath(os.getcwd())
+    return collect_requirements(git_root, additional_dependencies)
+
+
 def py_interface(_dir, _make_venv):
     @contextlib.contextmanager
     def in_env(prefix, language_version):

pre_commit/repository.py

@@ -183,7 +183,15 @@ class Repository(object):
         for _, hook in self.hooks:
             language = hook['language']
             version = hook['language_version']
-            deps = tuple(hook['additional_dependencies'])
+            additional_dependencies = hook['additional_dependencies']
+            try:
+                deps = languages[language].process_additional_dependencies(
+                    additional_dependencies,
+                )
+            except AttributeError:
+                # Language does not implement process_additional_dependencies
+                deps = additional_dependencies
+            deps = tuple(deps)
             ret.add((
                 self._prefix_from_deps(language, deps),
                 language, version, deps,

testing/fixtures.py

@@ -64,7 +64,8 @@ def modify_manifest(path):
     with io.open(manifest_path, 'w') as manifest_file:
         manifest_file.write(ordered_dump(manifest, **C.YAML_DUMP_KWARGS))
     cmd_output(
-        'git', 'commit', '--no-gpg-sign', '-am', 'update {}'.format(C.MANIFEST_FILE), cwd=path,
+        'git', 'commit', '--no-gpg-sign', '-am',
+        'update {}'.format(C.MANIFEST_FILE), cwd=path,
     )
 
 
@@ -80,7 +81,10 @@ def modify_config(path='.', commit=True):
     with io.open(config_path, 'w', encoding='UTF-8') as config_file:
         config_file.write(ordered_dump(config, **C.YAML_DUMP_KWARGS))
     if commit:
-        cmd_output('git', 'commit', '--no-gpg-sign', '-am', 'update config', cwd=path)
+        cmd_output(
+            'git', 'commit', '--no-gpg-sign', '-am', 'update config',
+            cwd=path,
+        )
 
 
 def config_with_local_hooks():
@@ -136,13 +140,19 @@ def write_config(directory, config, config_file=C.CONFIG_FILE):
 def add_config_to_repo(git_path, config, config_file=C.CONFIG_FILE):
     write_config(git_path, config, config_file=config_file)
     cmd_output('git', 'add', config_file, cwd=git_path)
-    cmd_output('git', 'commit', '--no-gpg-sign', '-m', 'Add hooks config', cwd=git_path)
+    cmd_output(
+        'git', 'commit', '--no-gpg-sign', '-m', 'Add hooks config',
+        cwd=git_path,
+    )
     return git_path
 
 
 def remove_config_from_repo(git_path, config_file=C.CONFIG_FILE):
     cmd_output('git', 'rm', config_file, cwd=git_path)
-    cmd_output('git', 'commit', '--no-gpg-sign', '-m', 'Remove hooks config', cwd=git_path)
+    cmd_output(
+        'git', 'commit', '--no-gpg-sign', '-m', 'Remove hooks config',
+        cwd=git_path,
+    )
     return git_path
 
 

tests/languages/python_test.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import os.path
+import textwrap
 
 from pre_commit.languages import python
 
@@ -16,3 +17,205 @@ def test_norm_version_expanduser():
         expected_path = home + '/.pyenv/versions/3.4.3/bin/python'
     result = python.norm_version(path)
     assert result == expected_path
+
+
+def test_single_requirements_file(tempdir_factory):
+    tmpdir = tempdir_factory.get()
+    req1 = os.path.join(tmpdir, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pep8
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['-r', 'req1.txt'],
+    ) == ['pep8']
+    assert python.collect_requirements(
+        tmpdir, ['-rreq1.txt'],
+    ) == ['pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'req1.txt'],
+    ) == ['pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement=req1.txt'],
+    ) == ['pep8']
+
+
+def test_multiple_requirements_file(tempdir_factory):
+    tmpdir = tempdir_factory.get()
+    req1 = os.path.join(tmpdir, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pep8
+        '''))
+    req2 = os.path.join(tmpdir, 'req2.txt')
+    with open(req2, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pre-commit
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['-r', 'req1.txt', '-r', 'req2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['-rreq1.txt', '-rreq2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'req1.txt', '--requirement', 'req2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement=req1.txt', '--requirement=req2.txt'],
+    ) == ['pep8', 'pre-commit']
+
+
+def test_nested_requirements_file(tempdir_factory):
+    tmpdir = tempdir_factory.get()
+    req1 = os.path.join(tmpdir, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pep8
+        '''))
+    req2 = os.path.join(tmpdir, 'req2.txt')
+    with open(req2, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            -r req1.txt
+            pre-commit
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['-r', 'req2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['-rreq2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'req2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement=req2.txt'],
+    ) == ['pep8', 'pre-commit']
+
+
+def test_nested_requirements_files_subdir(tempdir_factory):
+    tmpdir = tempdir_factory.get()
+    req1 = os.path.join(tmpdir, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pep8
+        '''))
+    reqsdir = os.path.join(tmpdir, 'requirements')
+    os.makedirs(reqsdir)
+    req2 = os.path.join(reqsdir, 'req2.txt')
+    with open(req2, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            -r ../req1.txt
+            pre-commit
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['-r', 'requirements/req2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['-rrequirements/req2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'requirements/req2.txt'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement=requirements/req2.txt'],
+    ) == ['pep8', 'pre-commit']
+
+
+def test_mixed_requirements(tempdir_factory):
+    tmpdir = tempdir_factory.get()
+    req1 = os.path.join(tmpdir, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pep8
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['pre-commit', '-r', 'req1.txt'],
+    ) == ['pre-commit', 'pep8']
+    assert python.collect_requirements(
+        tmpdir, ['-rreq1.txt', 'pre-commit'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'req1.txt', 'pre-commit'],
+    ) == ['pep8', 'pre-commit']
+    assert python.collect_requirements(
+        tmpdir, ['pre-commit', '--requirement=req1.txt'],
+    ) == ['pre-commit', 'pep8']
+
+
+def test_options_in_requirements_file(tempdir_factory):
+    tmpdir = tempdir_factory.get()
+    req1 = os.path.join(tmpdir, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            --index-url=https://domain.tld/repository/pypi/simple/
+            pep8
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['-r', 'req1.txt'],
+    ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8']
+    assert python.collect_requirements(
+        tmpdir, ['-rreq1.txt'],
+    ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'req1.txt'],
+    ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement=req1.txt'],
+    ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8']
+    assert python.collect_requirements(
+        tmpdir,
+        [
+            '--index-url=https://domain.tld/repository/pypi/simple/',
+            '-r', 'req1.txt',
+        ],
+    ) == ['--index-url=https://domain.tld/repository/pypi/simple/', 'pep8']
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            --index-url https://domain.tld/repository/pypi/simple/
+            pep8
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['-r', 'req1.txt'],
+    ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8']
+    assert python.collect_requirements(
+        tmpdir, ['-rreq1.txt'],
+    ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'req1.txt'],
+    ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement=req1.txt'],
+    ) == ['--index-url', 'https://domain.tld/repository/pypi/simple/', 'pep8']
+
+
+def test_missing_requirements_file(tempdir_factory):
+    tmpdir = tempdir_factory.get()
+    req1 = os.path.join(tmpdir, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pep8
+        '''))
+    assert python.collect_requirements(
+        tmpdir, ['-r', 'req1.txt', '-r', 'req2.txt'],
+    ) == ['pep8']
+    assert python.collect_requirements(
+        tmpdir, ['-rreq1.txt', '-rreq2.txt'],
+    ) == ['pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement', 'req1.txt', '--requirement', 'req2.txt'],
+    ) == ['pep8']
+    assert python.collect_requirements(
+        tmpdir, ['--requirement=req1.txt', '--requirement=req2.txt'],
+    ) == ['pep8']

tests/repository_test.py

@@ -6,6 +6,7 @@ import io
 import os.path
 import re
 import shutil
+import textwrap
 
 import mock
 import pytest
@@ -843,3 +844,50 @@ def test_manifest_hooks(tempdir_factory, store):
         'exclude_types': [],
         'verbose': False,
     }
+
+
+def test_python_additional_dependencies_requirements_file(
+    tempdir_factory,
+    store,
+):
+    path = make_repo(tempdir_factory, 'python_hooks_repo')
+    req1 = os.path.join(path, 'req1.txt')
+    with open(req1, 'w') as wfh:
+        wfh.write(textwrap.dedent('''
+            # This is a comment in the pip file
+            pep8
+        '''))
+    with cwd(path):
+        config = make_config_from_repo(path)
+        config['hooks'][0]['additional_dependencies'] = ['-r', 'req1.txt']
+        repo = Repository.create(config, store)
+        env, = repo._venvs()
+        assert env == (
+            mock.ANY, 'python', python.get_default_version(), ('pep8',),
+        )
+        config = make_config_from_repo(path)
+        config['hooks'][0]['additional_dependencies'] = ['-rreq1.txt']
+        repo = Repository.create(config, store)
+        env, = repo._venvs()
+        assert env == (
+            mock.ANY, 'python', python.get_default_version(), ('pep8',),
+        )
+        config = make_config_from_repo(path)
+        config['hooks'][0]['additional_dependencies'] = [
+            '--requirement',
+            'req1.txt',
+        ]
+        repo = Repository.create(config, store)
+        env, = repo._venvs()
+        assert env == (
+            mock.ANY, 'python', python.get_default_version(), ('pep8',),
+        )
+        config = make_config_from_repo(path)
+        config['hooks'][0]['additional_dependencies'] = [
+            '--requirement=req1.txt',
+        ]
+        repo = Repository.create(config, store)
+        env, = repo._venvs()
+        assert env == (
+            mock.ANY, 'python', python.get_default_version(), ('pep8',),
+        )