Merge pull request #46 from pre-commit/autoupdate_command_44

Autoupdate command

.pre-commit-config.yaml

@@ -1,22 +1,20 @@
-
 -   repo: git@github.com:pre-commit/pre-commit-hooks
-    sha: 76739902911688e8d7b13241409f9facc0e534e4
+    sha: 8aa14d218d0fe5a2c486498269b3e37d3ba5aad2
     hooks:
-        -   id: pyflakes
-            files: '\.py$'
-        -   id: debug-statements
-            files: '\.py$'
-        -   id: trailing-whitespace
-            files: '\.(py|sh|yaml)$'
-        -   id: name-tests-test
-            files: 'tests/.+\.py$'
-        -   id: end-of-file-fixer
-            files: '\.(py|sh|yaml)$'
-
+    -   id: pyflakes
+        files: \.py$
+    -   id: debug-statements
+        files: \.py$
+    -   id: trailing-whitespace
+        files: \.(py|sh|yaml)$
+    -   id: name-tests-test
+        files: tests/.+\.py$
+    -   id: end-of-file-fixer
+        files: \.(py|sh|yaml)$
 -   repo: git@github.com:pre-commit/pre-commit
-    sha: 47b7ca44ed1fcaa83464ed00cef72049ae22c33d
+    sha: c695ee9a9a78ac73439c52e0085bafec8037bc2d
     hooks:
-        -   id: validate_manifest
-            files: '^manifest.yaml$'
-        -   id: validate_config
-            files: '^\.pre-commit-config.yaml$'
+    -   id: validate_manifest
+        files: ^manifest.yaml$
+    -   id: validate_config
+        files: ^\.pre-commit-config.yaml$

pre_commit/clientlib/validate_base.py

@@ -24,7 +24,7 @@ def get_validator(
             exception_type on failure.
     """
 
-    def validate(filename=None):
+    def validate(filename=None, load_strategy=yaml.load):
         filename = filename or os.path.join(git.get_root(), default_filename)
 
         if not os.path.exists(filename):
@@ -33,7 +33,7 @@ def get_validator(
         file_contents = open(filename, 'r').read()
 
         try:
-            obj = yaml.load(file_contents)
+            obj = load_strategy(file_contents)
         except Exception as e:
             raise exception_type(
                 'File {0} is not a valid yaml file'.format(filename), e,

pre_commit/commands.py

@@ -4,6 +4,14 @@ from __future__ import print_function
 import os
 import pkg_resources
 import stat
+from plumbum import local
+
+import pre_commit.constants as C
+from pre_commit.clientlib.validate_config import load_config
+from pre_commit.ordereddict import OrderedDict
+from pre_commit.repository import Repository
+from pre_commit.yaml_extensions import ordered_dump
+from pre_commit.yaml_extensions import ordered_load
 
 
 def install(runner):
@@ -29,3 +37,83 @@ def uninstall(runner):
         os.remove(runner.pre_commit_path)
         print('pre-commit uninstalled')
     return 0
+
+
+class RepositoryCannotBeUpdatedError(RuntimeError): pass
+
+
+def _update_repository(repo_config):
+    """Updates a repository to the tip of `master`.  If the repository cannot
+    be updated because a hook that is configured does not exist in `master`,
+    this raises a RepositoryCannotBeUpdatedError
+
+    Args:
+        repo_config - A config for a repository
+    """
+    repo = Repository(repo_config)
+
+    with repo.in_checkout():
+        local['git']['fetch']()
+        head_sha = local['git']['rev-parse', 'origin/master']().strip()
+
+    # Don't bother trying to update if our sha is the same
+    if head_sha == repo_config['sha']:
+        return repo_config
+
+    # Construct a new config with the head sha
+    new_config = OrderedDict(repo_config)
+    new_config['sha'] = head_sha
+    new_repo = Repository(new_config)
+
+    # See if any of our hooks were deleted with the new commits
+    hooks = set(repo.hooks.keys())
+    hooks_missing = hooks - (hooks & set(new_repo.manifest.keys()))
+    if hooks_missing:
+        raise RepositoryCannotBeUpdatedError(
+            'Cannot update because the tip of master is missing these hooks:\n'
+            '{0}'.format(', '.join(sorted(hooks_missing)))
+        )
+
+    return new_config
+
+
+def autoupdate(runner):
+    """Auto-update the pre-commit config to the latest versions of repos."""
+    retv = 0
+    output_configs = []
+    changed = False
+
+    input_configs = load_config(
+        runner.config_file_path,
+        load_strategy=ordered_load,
+    )
+
+    for repo_config in input_configs:
+        print('Updating {0}...'.format(repo_config['repo']), end='')
+        try:
+            new_repo_config = _update_repository(repo_config)
+        except RepositoryCannotBeUpdatedError as e:
+            print(e.args[0])
+            output_configs.append(repo_config)
+            retv = 1
+            continue
+
+        if new_repo_config['sha'] != repo_config['sha']:
+            changed = True
+            print(
+                'updating {0} -> {1}.'.format(
+                    repo_config['sha'], new_repo_config['sha'],
+                )
+            )
+            output_configs.append(new_repo_config)
+        else:
+            print('already up to date.')
+            output_configs.append(repo_config)
+
+    if changed:
+        with open(runner.config_file_path, 'w') as config_file:
+            config_file.write(
+                ordered_dump(output_configs, **C.YAML_DUMP_KWARGS)
+            )
+
+    return retv

pre_commit/constants.py

@@ -10,3 +10,9 @@ SUPPORTED_LANGUAGES = set([
     'ruby',
     'node',
 ])
+
+
+YAML_DUMP_KWARGS = {
+    'default_flow_style': False,
+    'indent': 4,
+}

pre_commit/run.py

@@ -100,14 +100,7 @@ def run(argv):
 
     subparsers.add_parser('uninstall', help='Uninstall the pre-commit script.')
 
-    execute_hook = subparsers.add_parser(
-        'execute-hook', help='Run a single hook.'
-    )
-    execute_hook.add_argument('hook', help='The hook-id to run.')
-    execute_hook.add_argument(
-        '--all-files', '-a', action='store_true', default=False,
-        help='Run on all the files in the repo.',
-    )
+    subparsers.add_parser('autoupdate', help='Auto-update hooks config.')
 
     run = subparsers.add_parser('run', help='Run hooks.')
     run.add_argument('hook', nargs='?', help='A single hook-id to run'),
@@ -130,6 +123,8 @@ def run(argv):
         return commands.install(runner)
     elif args.command == 'uninstall':
         return commands.uninstall(runner)
+    elif args.command == 'autoupdate':
+        return commands.autoupdate(runner)
     elif args.command == 'run':
         if args.hook:
             return run_single_hook(runner, args.hook, all_files=args.all_files)

pre_commit/yaml_extensions.py

@@ -0,0 +1,28 @@
+
+import yaml
+
+from pre_commit.ordereddict import OrderedDict
+
+
+# Adapted from http://stackoverflow.com/a/21912744/812183
+
+def ordered_load(s):
+    class OrderedLoader(yaml.loader.Loader): pass
+    def constructor(loader, node):
+        return OrderedDict(loader.construct_pairs(node))
+    OrderedLoader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        constructor,
+    )
+    return yaml.load(s, Loader=OrderedLoader)
+
+
+def ordered_dump(s, **kwargs):
+    class OrderedDumper(yaml.dumper.SafeDumper): pass
+    def dict_representer(dumper, data):
+        return dumper.represent_mapping(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            data.items(),
+        )
+    OrderedDumper.add_representer(OrderedDict, dict_representer)
+    return yaml.dump(s, Dumper=OrderedDumper, **kwargs)

testing/auto_namedtuple.py

@@ -0,0 +1,11 @@
+
+import collections
+
+def auto_namedtuple(classname='auto_namedtuple', **kwargs):
+    """Returns an automatic namedtuple object.
+
+    Args:
+        classname - The class name for the returned object.
+        **kwargs - Properties to give the returned object.
+    """
+    return (collections.namedtuple(classname, kwargs.keys())(**kwargs))

testing/resources/manifest_without_foo.yaml

@@ -0,0 +1,4 @@
+-   id: bar
+    name: Bar
+    entry: bar
+    language: python

testing/resources/ordering_data_test.yaml

@@ -0,0 +1,2 @@
+foo: bar
+bar: baz

tests/clientlib/validate_base_test.py

@@ -7,6 +7,8 @@ import pytest
 
 from pre_commit import git
 from pre_commit.clientlib.validate_base import get_validator
+from pre_commit.ordereddict import OrderedDict
+from pre_commit.yaml_extensions import ordered_load
 from testing.util import get_resource_path
 
 
@@ -71,3 +73,11 @@ def test_raises_when_additional_validation_fails(additional_validator):
 def test_returns_object_after_validating(noop_validator):
     ret = noop_validator(get_resource_path('array_yaml_file.yaml'))
     assert ret == ['foo', 'bar']
+
+
+def test_load_strategy(noop_validator):
+    ret = noop_validator(
+        get_resource_path('ordering_data_test.yaml'),
+        load_strategy=ordered_load,
+    )
+    assert type(ret) is OrderedDict

tests/commands_test.py

@@ -1,12 +1,27 @@
 
+import jsonschema
 import os
 import os.path
 import pkg_resources
+import pytest
+import shutil
 import stat
+from plumbum import local
 
+import pre_commit.constants as C
+from pre_commit import git
+from pre_commit.clientlib.validate_config import CONFIG_JSON_SCHEMA
+from pre_commit.clientlib.validate_config import validate_config_extra
+from pre_commit.commands import autoupdate
 from pre_commit.commands import install
+from pre_commit.commands import RepositoryCannotBeUpdatedError
 from pre_commit.commands import uninstall
+from pre_commit.commands import _update_repository
+from pre_commit.ordereddict import OrderedDict
 from pre_commit.runner import Runner
+from pre_commit.yaml_extensions import ordered_dump
+from testing.auto_namedtuple import auto_namedtuple
+from testing.util import get_resource_path
 
 
 def test_install_pre_commit(empty_git_dir):
@@ -35,3 +50,116 @@ def test_uninstall(empty_git_dir):
     assert os.path.exists(runner.pre_commit_path)
     uninstall(runner)
     assert not os.path.exists(runner.pre_commit_path)
+
+
+@pytest.yield_fixture
+def up_to_date_repo(python_hooks_repo):
+    config = OrderedDict((
+        ('repo', python_hooks_repo),
+        ('sha', git.get_head_sha(python_hooks_repo)),
+        ('hooks', [OrderedDict((('id', 'foo'), ('files', '')))]),
+    ))
+    jsonschema.validate([config], CONFIG_JSON_SCHEMA)
+    validate_config_extra([config])
+
+    with open(os.path.join(python_hooks_repo, C.CONFIG_FILE), 'w') as file_obj:
+        file_obj.write(
+            ordered_dump([config], **C.YAML_DUMP_KWARGS)
+        )
+
+    yield auto_namedtuple(
+        repo_config=config,
+        python_hooks_repo=python_hooks_repo,
+    )
+
+
+def test_up_to_date_repo(up_to_date_repo):
+    input_sha = up_to_date_repo.repo_config['sha']
+    ret = _update_repository(up_to_date_repo.repo_config)
+    assert ret['sha'] == input_sha
+
+
+def test_autoupdate_up_to_date_repo(up_to_date_repo):
+    before = open(C.CONFIG_FILE).read()
+    runner = Runner(up_to_date_repo.python_hooks_repo)
+    ret = autoupdate(runner)
+    after = open(C.CONFIG_FILE).read()
+    assert ret == 0
+    assert before == after
+
+
+@pytest.yield_fixture
+def out_of_date_repo(python_hooks_repo):
+    config = OrderedDict((
+        ('repo', python_hooks_repo),
+        ('sha', git.get_head_sha(python_hooks_repo)),
+        ('hooks', [OrderedDict((('id', 'foo'), ('files', '')))]),
+    ))
+    jsonschema.validate([config], CONFIG_JSON_SCHEMA)
+    validate_config_extra([config])
+    local['git']['commit', '--allow-empty', '-m', 'foo']()
+    head_sha = git.get_head_sha(python_hooks_repo)
+
+    with open(os.path.join(python_hooks_repo, C.CONFIG_FILE), 'w') as file_obj:
+        file_obj.write(
+            ordered_dump([config], **C.YAML_DUMP_KWARGS)
+        )
+
+    yield auto_namedtuple(
+        repo_config=config,
+        head_sha=head_sha,
+        python_hooks_repo=python_hooks_repo,
+    )
+
+
+def test_out_of_date_repo(out_of_date_repo):
+    ret = _update_repository(out_of_date_repo.repo_config)
+    assert ret['sha'] == out_of_date_repo.head_sha
+
+
+def test_autoupdate_out_of_date_repo(out_of_date_repo):
+    before = open(C.CONFIG_FILE).read()
+    runner = Runner(out_of_date_repo.python_hooks_repo)
+    ret = autoupdate(runner)
+    after = open(C.CONFIG_FILE).read()
+    assert ret == 0
+    assert before != after
+    assert out_of_date_repo.head_sha in after
+
+
+@pytest.yield_fixture
+def hook_disappearing_repo(python_hooks_repo):
+    config = OrderedDict((
+        ('repo', python_hooks_repo),
+        ('sha', git.get_head_sha(python_hooks_repo)),
+        ('hooks', [OrderedDict((('id', 'foo'), ('files', '')))]),
+    ))
+    jsonschema.validate([config], CONFIG_JSON_SCHEMA)
+    validate_config_extra([config])
+    shutil.copy(get_resource_path('manifest_without_foo.yaml'), C.MANIFEST_FILE)
+    local['git']['add', '.']()
+    local['git']['commit', '-m', 'Remove foo']()
+
+    with open(os.path.join(python_hooks_repo, C.CONFIG_FILE), 'w') as file_obj:
+        file_obj.write(
+            ordered_dump([config], **C.YAML_DUMP_KWARGS)
+        )
+
+    yield auto_namedtuple(
+        repo_config=config,
+        python_hooks_repo=python_hooks_repo,
+    )
+
+
+def test_hook_disppearing_repo_raises(hook_disappearing_repo):
+    with pytest.raises(RepositoryCannotBeUpdatedError):
+        _update_repository(hook_disappearing_repo.repo_config)
+
+
+def test_autoupdate_hook_disappearing_repo(hook_disappearing_repo):
+    before = open(C.CONFIG_FILE).read()
+    runner = Runner(hook_disappearing_repo.python_hooks_repo)
+    ret = autoupdate(runner)
+    after = open(C.CONFIG_FILE).read()
+    assert ret == 1
+    assert before == after

tests/yaml_extensions_test.py

@@ -0,0 +1,36 @@
+
+import pre_commit.constants as C
+from pre_commit.ordereddict import OrderedDict
+from pre_commit.yaml_extensions import ordered_dump
+from pre_commit.yaml_extensions import ordered_load
+
+
+def test_ordered_load():
+    ret = ordered_load(
+        'a: herp\n'
+        'c: derp\n'
+        'd: darp\n'
+        'b: harp\n'
+    )
+    # Original behavior
+    assert ret == {'a': 'herp', 'b': 'harp', 'c': 'derp', 'd': 'darp'}
+    # Ordered behavior
+    assert (
+        ret.items() ==
+        [('a', 'herp'), ('c', 'derp'), ('d', 'darp'), ('b', 'harp')]
+    )
+
+
+def test_ordered_dump():
+    ret = ordered_dump(
+        OrderedDict(
+            (('a', 'herp'), ('c', 'derp'), ('b', 'harp'), ('d', 'darp'))
+        ),
+        **C.YAML_DUMP_KWARGS
+    )
+    assert ret == (
+        'a: herp\n'
+        'c: derp\n'
+        'b: harp\n'
+        'd: darp\n'
+    )