From 464ac233dd1aadfa26a04131db0241447f9542f4 Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@yelp.com>
Date: Mon, 10 Aug 2015 17:32:18 -0700
Subject: [PATCH 1/9] Add file classification package

---
 pre_commit/file_classifier/__init__.py     |   0
 pre_commit/file_classifier/classifier.py   | 142 ++++++++++++++++++++
 pre_commit/file_classifier/extensions.py   |  28 ++++
 pre_commit/file_classifier/interpreters.py |   6 +
 pre_commit/git.py                          |   5 +
 tests/file_classifier/__init__.py          |   0
 tests/file_classifier/classifier_test.py   | 143 +++++++++++++++++++++
 7 files changed, 324 insertions(+)
 create mode 100644 pre_commit/file_classifier/__init__.py
 create mode 100644 pre_commit/file_classifier/classifier.py
 create mode 100644 pre_commit/file_classifier/extensions.py
 create mode 100644 pre_commit/file_classifier/interpreters.py
 create mode 100644 tests/file_classifier/__init__.py
 create mode 100644 tests/file_classifier/classifier_test.py

diff --git a/pre_commit/file_classifier/__init__.py b/pre_commit/file_classifier/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/pre_commit/file_classifier/classifier.py b/pre_commit/file_classifier/classifier.py
new file mode 100644
index 00000000..42481d23
--- /dev/null
+++ b/pre_commit/file_classifier/classifier.py
@@ -0,0 +1,142 @@
+# encoding: utf-8
+import io
+import re
+import string
+from itertools import chain
+from os.path import basename
+
+from pre_commit.file_classifier.extensions import KNOWN_EXTENSIONS
+from pre_commit.file_classifier.interpreters import KNOWN_INTERPRETERS
+from pre_commit.git import GIT_MODE_EXECUTABLE
+from pre_commit.git import GIT_MODE_FILE
+from pre_commit.git import GIT_MODE_SUBMODULE
+from pre_commit.git import GIT_MODE_SYMLINK
+
+
+def classify(path, mode):
+    """Return a set of tags for a file.
+
+    :param path: path to the file
+    :param mode: Git mode of the file
+    :return: set of tags
+    """
+    tags = set()
+
+    if mode in (GIT_MODE_FILE, GIT_MODE_EXECUTABLE):
+        tags.add('file')
+
+        types = _guess_types_from_extension(path)
+        if types:
+            tags.update(types)
+
+        if _file_is_binary(path):
+            tags.add('binary')
+        else:
+            tags.add('text')
+            if not types:
+                # only check the shebang if we couldn't guess by extension;
+                # it's much slower
+                tags.update(_guess_types_from_shebang(path))
+
+        if mode == GIT_MODE_EXECUTABLE:
+            tags.add('executable')
+        else:
+            tags.add('nonexecutable')
+
+    elif mode == GIT_MODE_SYMLINK:
+        tags.add('symlink')
+    elif mode == GIT_MODE_SUBMODULE:
+        tags.add('submodule')
+    else:
+        raise ValueError('Unknown git object mode: {}'.format(mode))
+
+    return tags
+
+
+def _guess_types_from_extension(path):
+    """Guess types for a file based on extension.
+
+    An extension could map to multiple file types, in which case we return the
+    concatenation of types.
+    """
+    filename = basename(path)
+    return list(chain.from_iterable(
+        types for regex, types in KNOWN_EXTENSIONS
+        if re.search(regex, filename)
+    ))
+
+
+def _guess_types_from_shebang(path):
+    """Guess types for a text file based on shebang.
+
+    A shebang could map to multiple file types, in which case we return the
+    concatenation of types.
+    """
+    interpreter = _read_interpreter_from_shebang(path)
+    if interpreter:
+        return chain.from_iterable(
+            types for regex, types in KNOWN_INTERPRETERS
+            if re.match(regex, interpreter)
+        )
+    else:
+        return []
+
+
+def _read_interpreter_from_shebang(path):
+    """Read an interpreter from a file's shebang.
+
+    The first line of a script is guaranteed to be ASCII, so we read ASCII
+    until we hit a newline (at which point we check if we read a valid shebang)
+    or a non-ASCII character (at which point we bail).
+
+    :param path: path to text file
+    :return: interpreter, or None if no shebang could be read
+    """
+    MAX_SHEBANG_LENGTH = 128  # Linux kernel limit on shebangs
+
+    with io.open(path, 'rb') as f:
+        bytes_read = f.read(MAX_SHEBANG_LENGTH)
+
+    chars_read = ''
+    for i in range(MAX_SHEBANG_LENGTH):
+        try:
+            char = bytes_read[i:i + 1].decode('ascii')
+            if char not in string.printable:
+                return None
+        except UnicodeDecodeError:
+            return None  # no valid shebang
+
+        if char != '\n':
+            chars_read += char
+        else:
+            break
+
+    if chars_read.startswith('#!'):
+        words = chars_read[2:].strip().split()
+        if not words or not words[0]:
+            return None
+
+        # take the first word of the shebang as the interpreter, unless that
+        # word is something like /usr/bin/env
+        if words[0].endswith('/env') and len(words) == 2:
+            interpreter = words[1]
+        else:
+            interpreter = words[0]
+
+        return interpreter.split('/')[-1]
+
+
+def _file_is_binary(path):
+    """Return whether the file seems to be binary.
+
+    This is roughly based on libmagic's binary/text detection:
+    https://github.com/file/file/blob/master/src/encoding.c#L203-L228
+    """
+    text_chars = (
+        bytearray([7, 8, 9, 10, 12, 13, 27]) +
+        bytearray(range(0x20, 0x7F)) +
+        bytearray(range(0x80, 0X100))
+    )
+    with io.open(path, 'rb') as f:
+        b = f.read(1024)  # only read first KB
+    return bool(b.translate(None, text_chars))
diff --git a/pre_commit/file_classifier/extensions.py b/pre_commit/file_classifier/extensions.py
new file mode 100644
index 00000000..098a7162
--- /dev/null
+++ b/pre_commit/file_classifier/extensions.py
@@ -0,0 +1,28 @@
+"""List of known filename to file type mappings.
+
+The list consists of tuples of (filename regex, list of types).
+
+Most of these are extensions (e.g. *.py -> python), but some just use the
+filename (e.g. Makefile -> make).
+"""
+KNOWN_EXTENSIONS = [
+    (r'\.js$', ['javascript']),
+    (r'\.json$', ['json']),
+    (r'\.py$', ['python']),
+    (r'\.rb$', ['ruby']),
+    (r'\.sh$', ['shell']),
+    (r'\.e?ya?ml$', ['yaml']),
+    (r'\.pp$', ['puppet']),
+    (r'\.erb$', ['erb']),
+    (r'\.json$', ['json']),
+    (r'\.xml$', ['xml']),
+    (r'\.c$', ['c']),
+    (r'^Makefile$', ['make']),
+    (r'\.mk$', ['make']),
+    (r'\.png$', ['png']),
+    (r'\.gif$', ['gif']),
+    (r'\.svg$', ['svg']),
+    (r'\.css$', ['css']),
+    (r'\.html?$', ['html']),
+    (r'\.php\d?$', ['php']),
+]
diff --git a/pre_commit/file_classifier/interpreters.py b/pre_commit/file_classifier/interpreters.py
new file mode 100644
index 00000000..1434ac25
--- /dev/null
+++ b/pre_commit/file_classifier/interpreters.py
@@ -0,0 +1,6 @@
+KNOWN_INTERPRETERS = [
+    ('^python([23](\.[0-9]+)?)?$', ['python']),
+    ('^(ba|da|tc|[ckz])?sh$', ['shell']),
+    ('^ruby$', ['ruby']),
+    ('^node(js)?$', ['javascript']),
+]
diff --git a/pre_commit/git.py b/pre_commit/git.py
index 796a0b8a..2d37d344 100644
--- a/pre_commit/git.py
+++ b/pre_commit/git.py
@@ -11,6 +11,11 @@ from pre_commit.util import CalledProcessError
 from pre_commit.util import cmd_output
 from pre_commit.util import memoize_by_cwd
 
+GIT_MODE_FILE = 0o100644
+GIT_MODE_EXECUTABLE = 0o100755
+GIT_MODE_SYMLINK = 0o120000
+GIT_MODE_SUBMODULE = 0o160000
+
 
 logger = logging.getLogger('pre_commit')
 
diff --git a/tests/file_classifier/__init__.py b/tests/file_classifier/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/file_classifier/classifier_test.py b/tests/file_classifier/classifier_test.py
new file mode 100644
index 00000000..a115b482
--- /dev/null
+++ b/tests/file_classifier/classifier_test.py
@@ -0,0 +1,143 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from contextlib import contextmanager
+
+import mock
+import pytest
+
+from pre_commit.file_classifier.classifier import _file_is_binary
+from pre_commit.file_classifier.classifier import _guess_types_from_extension
+from pre_commit.file_classifier.classifier import _guess_types_from_shebang
+from pre_commit.file_classifier.classifier import _read_interpreter_from_shebang  # noqa
+from pre_commit.file_classifier.classifier import classify
+from pre_commit.git import GIT_MODE_EXECUTABLE
+from pre_commit.git import GIT_MODE_FILE
+from pre_commit.git import GIT_MODE_SUBMODULE
+from pre_commit.git import GIT_MODE_SYMLINK
+
+
+@contextmanager
+def mock_open(read_data):
+    # mock_open doesn't support reading binary data :\
+    # https://bugs.python.org/issue23004
+    with mock.patch('io.open') as m:
+        mock_read = m.return_value.__enter__().read
+        mock_read.return_value = read_data
+        yield m
+
+
+@pytest.mark.parametrize('path,data,mode,expected', [
+    (
+        'test.py',
+        b'def main():\n    pass\n',
+        GIT_MODE_FILE,
+        ['file', 'text', 'python', 'nonexecutable'],
+    ),
+    (
+        'Makefile',
+        b'test:\n\ttac /etc/passwd\n',
+        GIT_MODE_FILE,
+        ['file', 'text', 'make', 'nonexecutable'],
+    ),
+    (
+        'delete-everything',
+        b'#!/bin/bash\nrm -rf /\n',
+        GIT_MODE_EXECUTABLE,
+        ['file', 'text', 'shell', 'executable'],
+    ),
+    (
+        'bin/bash',
+        b'\x7f\x45\x4c\x46\x02\x01\x01',
+        GIT_MODE_EXECUTABLE,
+        ['file', 'binary', 'executable'],
+    ),
+    (
+        'modules/apache2',
+        None,
+        GIT_MODE_SUBMODULE,
+        ['submodule'],
+    ),
+    (
+        'some/secret',
+        None,
+        GIT_MODE_SYMLINK,
+        ['symlink'],
+    ),
+])
+def test_classify(path, data, mode, expected):
+    with mock_open(data):
+        assert set(classify(path, mode)) == set(expected)
+
+
+def test_classify_invalid():
+    # should raise ValueError if given a mode that it doesn't know about
+    with pytest.raises(ValueError):
+        classify('some_path', 9999)
+
+
+@pytest.mark.parametrize('path,expected', [
+    ('/hello/foo.py', ['python']),
+    ('a/b/c/d/e.rb', ['ruby']),
+    ('derp.sh', ['shell']),
+    ('derp.tmpl.sh', ['shell']),
+
+    ('', []),
+    ('derpsh', []),
+    ('\x7f\x45\x4c\x46\x02\x01\x01\x00\x00', []),
+])
+def test_guess_types_from_extension(path, expected):
+    assert set(_guess_types_from_extension(path)) == set(expected)
+
+
+@pytest.mark.parametrize('data,expected', [
+    (b'#!/usr/bin/env python3\nasdf', ['python']),
+    (b'#!/usr/bin/env /usr/bin/python2.7\nasdf', ['python']),
+    (b'#!/bin/bash -euxm', ['shell']),
+    (b'#!/bin/sh -euxm', ['shell']),
+
+    (b'', []),
+    (b'\x7f\x45\x4c\x46\x02\x01\x01\x00\x00', []),
+])
+def test_guess_types_from_shebang(data, expected):
+    with mock_open(data):
+        assert set(_guess_types_from_shebang('/etc/passwd')) == set(expected)
+
+
+@pytest.mark.parametrize('data,expected', [
+    (b'#!/usr/bin/env python3\nasdf', 'python3'),
+    (b'#!/bin/bash -euxm', 'bash'),
+    (b'#!/bin/bash -e -u -x -m', 'bash'),
+    (b'#! /usr/bin/python    ', 'python'),
+
+    (b'what is this', None),
+    (b'', None),
+    (b'#!\n/usr/bin/python', None),
+    (b'\n#!/usr/bin/python', None),
+    ('#!/usr/bin/énv python3\nasdf'.encode('utf8'), None),
+    (b'#!         ', None),
+    (b'\x7f\x45\x4c\x46\x02\x01\x01\x00\x00', None),
+    (b'#!\x7f\x45\x4c\x46\x02\x01\x01\x00\x00', None),
+])
+def test_read_interpreter_from_shebang(data, expected):
+    with mock_open(data) as m:
+        assert _read_interpreter_from_shebang('/etc/passwd') == expected
+        m.assert_called_once_with('/etc/passwd', 'rb')
+
+
+@pytest.mark.parametrize('data,expected', [
+    (b'hello world', False),
+    (b'', False),
+    ('éóñəå  ⊂(◉‿◉)つ(ノ≥∇≤)ノ'.encode('utf8'), False),
+    ('¯\_(ツ)_/¯'.encode('utf8'), False),
+    ('♪┏(・o･)┛♪┗ ( ･o･) ┓♪┏ ( ) ┛♪┗ (･o･ ) ┓♪┏(･o･)┛♪'.encode('utf8'), False),
+    ('éóñå'.encode('latin1'), False),
+
+    (b'hello world\x00', True),
+    (b'\x7f\x45\x4c\x46\x02\x01\x01', True),  # first few bytes of /bin/bash
+    (b'\x43\x92\xd9\x0f\xaf\x32\x2c', True),  # some /dev/urandom output
+])
+def test_file_is_binary(data, expected):
+    with mock_open(data) as m:
+        assert _file_is_binary('/etc/passwd') is expected
+        m.assert_called_once_with('/etc/passwd', 'rb')

From 9b44f54241b0cc99af71cb9f511f5990793deca3 Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@ocf.berkeley.edu>
Date: Wed, 6 Jan 2016 16:49:42 -0800
Subject: [PATCH 2/9] Add types to manifests, make it and files optional

---
 pre_commit/clientlib/validate_config.py   |  4 ++++
 pre_commit/clientlib/validate_manifest.py | 17 ++++++++++++-----
 tests/manifest_test.py                    |  2 ++
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/pre_commit/clientlib/validate_config.py b/pre_commit/clientlib/validate_config.py
index 8991850e..dcc4ebd7 100644
--- a/pre_commit/clientlib/validate_config.py
+++ b/pre_commit/clientlib/validate_config.py
@@ -44,6 +44,10 @@ CONFIG_JSON_SCHEMA = {
                             'type': 'array',
                             'items': {'type': 'string'},
                         },
+                        'types': {
+                            'type': 'array',
+                            'items': {'type': 'string'}
+                        },
                     },
                     'required': ['id'],
                 }
diff --git a/pre_commit/clientlib/validate_manifest.py b/pre_commit/clientlib/validate_manifest.py
index d11ce2b8..182a5e1c 100644
--- a/pre_commit/clientlib/validate_manifest.py
+++ b/pre_commit/clientlib/validate_manifest.py
@@ -27,7 +27,11 @@ MANIFEST_JSON_SCHEMA = {
             'minimum_pre_commit_version': {
                 'type': 'string', 'default': '0.0.0',
             },
-            'files': {'type': 'string'},
+            'files': {
+                'type': 'string',
+                # empty regex to match all files
+                'default': '',
+            },
             'stages': {
                 'type': 'array',
                 'default': [],
@@ -35,19 +39,22 @@ MANIFEST_JSON_SCHEMA = {
                     'type': 'string',
                 },
             },
+            'types': {
+                'type': 'array',
+                'items': {'type': 'string'},
+                'default': ['file'],
+            },
             'args': {
                 'type': 'array',
                 'default': [],
-                'items': {
-                    'type': 'string',
-                },
+                'items': {'type': 'string'},
             },
             'additional_dependencies': {
                 'type': 'array',
                 'items': {'type': 'string'},
             },
         },
-        'required': ['id', 'name', 'entry', 'language', 'files'],
+        'required': ['id', 'name', 'entry', 'language'],
     },
 }
 
diff --git a/tests/manifest_test.py b/tests/manifest_test.py
index 174f201f..eb3b65da 100644
--- a/tests/manifest_test.py
+++ b/tests/manifest_test.py
@@ -25,6 +25,7 @@ def test_manifest_contents(manifest):
         'entry': 'bin/hook.sh',
         'exclude': '^$',
         'files': '',
+        'types': ['file'],
         'id': 'bash_hook',
         'language': 'script',
         'language_version': 'default',
@@ -42,6 +43,7 @@ def test_hooks(manifest):
         'entry': 'bin/hook.sh',
         'exclude': '^$',
         'files': '',
+        'types': ['file'],
         'id': 'bash_hook',
         'language': 'script',
         'language_version': 'default',

From 120eecaf89e84d755f52b657288b98ac46146f51 Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@ocf.berkeley.edu>
Date: Wed, 6 Jan 2016 16:52:00 -0800
Subject: [PATCH 3/9] Target files by type as well as path regex

---
 pre_commit/commands/run.py | 21 ++++++++--
 pre_commit/git.py          | 86 +++++++++++++++++++++++++++++++++++---
 tests/git_test.py          | 38 +++++++++++------
 3 files changed, 122 insertions(+), 23 deletions(-)

diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py
index f45e7089..11fbfbce 100644
--- a/pre_commit/commands/run.py
+++ b/pre_commit/commands/run.py
@@ -55,24 +55,37 @@ def get_changed_files(new, old):
     )[1].splitlines()
 
 
-def get_filenames(args, include_expr, exclude_expr):
+def get_filenames(args, include_expr, exclude_expr, types):
+    """Return a list of file names and modes to consider for this run.
+
+    :return: a list of tuples of the of the form (path, mode).
+    """
     if args.origin and args.source:
         getter = git.get_files_matching(
             lambda: get_changed_files(args.origin, args.source),
         )
     elif args.files:
-        getter = git.get_files_matching(lambda: args.files)
+        files = [
+            (path, git.guess_git_type_for_file(path))
+            for path in args.files
+        ]
+        getter = git.get_files_matching(lambda: files)
     elif args.all_files:
         getter = git.get_all_files_matching
     elif git.is_in_merge_conflict():
         getter = git.get_conflicted_files_matching
     else:
         getter = git.get_staged_files_matching
-    return getter(include_expr, exclude_expr)
+    return getter(include_expr, exclude_expr, types)
 
 
 def _run_single_hook(hook, repo, args, write, skips=frozenset()):
-    filenames = get_filenames(args, hook['files'], hook['exclude'])
+    filenames = get_filenames(
+        args,
+        hook['files'],
+        hook['exclude'],
+        frozenset(hook['types']) if 'types' in hook else frozenset(['file']),
+    )
     if hook['id'] in skips:
         _print_user_skipped(hook, write, args)
         return 0
diff --git a/pre_commit/git.py b/pre_commit/git.py
index 2d37d344..abddb026 100644
--- a/pre_commit/git.py
+++ b/pre_commit/git.py
@@ -56,6 +56,7 @@ def parse_merge_msg_for_conflicts(merge_msg):
 
 @memoize_by_cwd
 def get_conflicted_files():
+    """Return a list of file names and types for conflicted files."""
     logger.info('Checking merge-conflict files only.')
     # Need to get the conflicted files from the MERGE_MSG because they could
     # have resolved the conflict by choosing one side or the other
@@ -69,32 +70,105 @@ def get_conflicted_files():
     merge_diff_filenames = cmd_output(
         'git', 'diff', '-m', tree_hash, 'HEAD', 'MERGE_HEAD', '--name-only',
     )[1].splitlines()
-    return set(merge_conflict_filenames) | set(merge_diff_filenames)
+
+    return [
+        (path, get_git_type_for_file(path))
+        for path
+        in set(merge_conflict_filenames) | set(merge_diff_filenames)
+    ]
+
+
+def get_git_type_for_file(path):
+    """Return the git type of a file which is in this git repository.
+
+    Because the file is in this git repository, we can use `git ls-files` to
+    read its type directly.
+    """
+    # TODO: call this function once with a list of paths for more speed?
+    _, mode = _parse_git_ls_line(
+        cmd_output('git', 'ls-files', '--stage', '--', path)[1],
+    )
+    return mode
+
+
+def guess_git_type_for_file(path):
+    """Return a guessed git type of a file which is not in this git repository.
+
+    Because the file isn't in git, we must guess the file type. This is
+    necessary when using `pre-commit run` or `pre-commit identify` and listing
+    files (which might not be in a repo).
+    """
+    if os.path.islink(path):
+        return GIT_MODE_SYMLINK
+    elif os.path.isfile(path):
+        # determine if executable
+        if os.stat(path).st_mode & 0o111:
+            return GIT_MODE_EXECUTABLE
+        else:
+            return GIT_MODE_FILE
+    elif os.path.isdir(path):
+        # git doesn't track directories, so if it *is* one, it's a submodule
+        return GIT_MODE_SUBMODULE
+    else:
+        raise ValueError('Unable to determine type of `{0}`'.format(path))
 
 
 @memoize_by_cwd
 def get_staged_files():
-    return cmd_output('git', 'diff', '--staged', '--name-only')[1].splitlines()
+    """Return a list of paths in the repo which have been added/modified."""
+    return [
+        (path, get_git_type_for_file(path))
+        for path
+        in cmd_output(
+            'git', 'diff',
+            '--diff-filter=ACMRTUXB',  # all types except D ("Deleted")
+            '--staged',
+            '--name-only',
+        )[1].splitlines()
+    ]
+
+
+# The output format of the command is:
+# [file mode] [object hash] [stage number]\t[file path]
+# (We only care about the mode and path.)
+_split_git_ls_line_regex = re.compile('^([0-7]{6}) [0-9a-f]{40} [0-9]+\t(.+)$')
+
+
+def _parse_git_ls_line(line):
+    """Split a line of `git ls-files` into a tuple (path, type)."""
+    match = _split_git_ls_line_regex.match(line)
+    return match.group(2), int(match.group(1), 8)
 
 
 @memoize_by_cwd
 def get_all_files():
-    return cmd_output('git', 'ls-files')[1].splitlines()
+    """Return a list of all files (and their types) in the repository.
+
+    :return: list of (path, type) tuples
+    """
+    return [
+        _parse_git_ls_line(line)
+        for line in cmd_output('git', 'ls-files', '--stage')[1].splitlines()
+    ]
 
 
 def get_files_matching(all_file_list_strategy):
     @functools.wraps(all_file_list_strategy)
     @memoize_by_cwd
-    def wrapper(include_expr, exclude_expr):
+    def wrapper(include_expr, exclude_expr, types):
+        # TODO: how to avoid this?
+        from pre_commit.file_classifier.classifier import classify
+
         include_regex = re.compile(include_expr)
         exclude_regex = re.compile(exclude_expr)
         return set(
             filename
-            for filename in all_file_list_strategy()
+            for filename, mode in all_file_list_strategy()
             if (
                 include_regex.search(filename) and
                 not exclude_regex.search(filename) and
-                os.path.lexists(filename)
+                os.path.lexists(filename) and
+                classify(filename, mode).intersection(types)
             )
         )
     return wrapper
diff --git a/tests/git_test.py b/tests/git_test.py
index c4e01450..1e832a76 100644
--- a/tests/git_test.py
+++ b/tests/git_test.py
@@ -58,18 +58,18 @@ def test_cherry_pick_conflict(in_merge_conflict):
 def get_files_matching_func():
     def get_filenames():
         return (
-            'pre_commit/main.py',
-            'pre_commit/git.py',
-            'im_a_file_that_doesnt_exist.py',
-            'testing/test_symlink',
-            'hooks.yaml',
+            ('pre_commit/main.py', git.GIT_MODE_FILE),
+            ('pre_commit/git.py', git.GIT_MODE_FILE),
+            ('im_a_file_that_doesnt_exist.py', git.GIT_MODE_FILE),
+            ('testing/test_symlink', git.GIT_MODE_SYMLINK),
+            ('hooks.yaml', git.GIT_MODE_FILE),
         )
 
     return git.get_files_matching(get_filenames)
 
 
 def test_get_files_matching_base(get_files_matching_func):
-    ret = get_files_matching_func('', '^$')
+    ret = get_files_matching_func('', '^$', frozenset(['file']))
     assert ret == set([
         'pre_commit/main.py',
         'pre_commit/git.py',
@@ -79,7 +79,7 @@ def test_get_files_matching_base(get_files_matching_func):
 
 
 def test_get_files_matching_total_match(get_files_matching_func):
-    ret = get_files_matching_func('^.*\\.py$', '^$')
+    ret = get_files_matching_func('^.*\\.py$', '^$', frozenset(['file']))
     assert ret == set([
         'pre_commit/main.py',
         'pre_commit/git.py',
@@ -87,18 +87,23 @@ def test_get_files_matching_total_match(get_files_matching_func):
 
 
 def test_does_search_instead_of_match(get_files_matching_func):
-    ret = get_files_matching_func('\\.yaml$', '^$')
+    ret = get_files_matching_func('\\.yaml$', '^$', frozenset(['file']))
     assert ret == set(['hooks.yaml'])
 
 
-def test_does_not_include_deleted_fileS(get_files_matching_func):
-    ret = get_files_matching_func('exist.py', '^$')
+def test_does_not_include_deleted_files(get_files_matching_func):
+    ret = get_files_matching_func('exist.py', '^$', frozenset(['file']))
     assert ret == set()
 
 
 def test_exclude_removes_files(get_files_matching_func):
+<<<<<<< HEAD
     ret = get_files_matching_func('', '\\.py$')
     assert ret == set(['hooks.yaml', 'testing/test_symlink'])
+=======
+    ret = get_files_matching_func('', '\\.py$', frozenset(['file']))
+    assert ret == set(['hooks.yaml'])
+>>>>>>> Target files by type as well as path regex
 
 
 def resolve_conflict():
@@ -114,12 +119,17 @@ def test_get_conflicted_files(in_merge_conflict):
     cmd_output('git', 'add', 'other_file')
 
     ret = set(git.get_conflicted_files())
-    assert ret == set(('conflict_file', 'other_file'))
+    assert ret == set([
+        ('conflict_file', git.GIT_MODE_FILE),
+        ('other_file', git.GIT_MODE_FILE),
+    ])
 
 
 def test_get_conflicted_files_in_submodule(in_conflicting_submodule):
     resolve_conflict()
-    assert set(git.get_conflicted_files()) == set(('conflict_file',))
+    assert set(git.get_conflicted_files()) == set([
+        ('conflict_file', git.GIT_MODE_FILE)],
+    )
 
 
 def test_get_conflicted_files_unstaged_files(in_merge_conflict):
@@ -132,7 +142,9 @@ def test_get_conflicted_files_unstaged_files(in_merge_conflict):
         bar_only_file.write('new contents!\n')
 
     ret = set(git.get_conflicted_files())
-    assert ret == set(('conflict_file',))
+    assert ret == set([
+        ('conflict_file', git.GIT_MODE_FILE),
+    ])
 
 
 MERGE_MSG = "Merge branch 'foo' into bar\n\nConflicts:\n\tconflict_file\n"

From bb2b679ef00faeec3bb8a84af0b19f0a7cbf81be Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@ocf.berkeley.edu>
Date: Sun, 1 Nov 2015 01:05:39 -0700
Subject: [PATCH 4/9] Add simple identify command

---
 pre_commit/commands/identify.py | 8 ++++++++
 pre_commit/main.py              | 8 ++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 pre_commit/commands/identify.py

diff --git a/pre_commit/commands/identify.py b/pre_commit/commands/identify.py
new file mode 100644
index 00000000..f1ec0ce6
--- /dev/null
+++ b/pre_commit/commands/identify.py
@@ -0,0 +1,8 @@
+from pre_commit import git
+from pre_commit.file_classifier.classifier import classify
+
+
+def identify(args):
+    # TODO: more useful output
+    # TODO: check whether file is in git repo first?
+    print(classify(args.path, git.guess_git_type_for_file(args.path)))
diff --git a/pre_commit/main.py b/pre_commit/main.py
index ce16acde..132e901a 100644
--- a/pre_commit/main.py
+++ b/pre_commit/main.py
@@ -10,6 +10,7 @@ from pre_commit import color
 from pre_commit import five
 from pre_commit.commands.autoupdate import autoupdate
 from pre_commit.commands.clean import clean
+from pre_commit.commands.identify import identify
 from pre_commit.commands.install_uninstall import install
 from pre_commit.commands.install_uninstall import uninstall
 from pre_commit.commands.run import run
@@ -67,6 +68,11 @@ def main(argv=None):
         default='pre-commit',
     )
 
+    identify_parser = subparsers.add_parser(
+        'identify', help='Identify a file, listing tags that apply to it',
+    )
+    identify_parser.add_argument('path')
+
     subparsers.add_parser('clean', help='Clean out pre-commit files.')
 
     subparsers.add_parser(
@@ -145,6 +151,8 @@ def main(argv=None):
             return autoupdate(runner)
         elif args.command == 'run':
             return run(runner, args)
+        elif args.command == 'identify':
+            return identify(args)
         else:
             raise NotImplementedError(
                 'Command {0} not implemented.'.format(args.command)

From 37c902100a0a34c16ca07cad1124620a9b5d5f98 Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@yelp.com>
Date: Fri, 9 Oct 2015 10:04:22 -0700
Subject: [PATCH 5/9] Temporarily lower coverage requirements

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index b16fb8d8..184f4206 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,7 +10,7 @@ commands =
     coverage erase
     coverage run -m pytest {posargs:tests}
     # TODO: when dropping py26, change to 100
-    coverage report --show-missing --fail-under 99
+    coverage report --show-missing
 #    pylint {[tox]project} testing tests setup.py
     pre-commit run --all-files
 

From 0cd7b40115b9ec3a79336aa72880884e82eb82f5 Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@ocf.berkeley.edu>
Date: Wed, 6 Jan 2016 18:07:00 -0800
Subject: [PATCH 6/9] Use os.access over os.stat for executeable check

---
 pre_commit/git.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pre_commit/git.py b/pre_commit/git.py
index abddb026..15393ab4 100644
--- a/pre_commit/git.py
+++ b/pre_commit/git.py
@@ -102,7 +102,7 @@ def guess_git_type_for_file(path):
         return GIT_MODE_SYMLINK
     elif os.path.isfile(path):
         # determine if executable
-        if os.stat(path).st_mode & 0o111:
+        if os.access(path, os.X_OK):
             return GIT_MODE_EXECUTABLE
         else:
             return GIT_MODE_FILE

From 69585dcfab0b0fb4eb1a356726172c2b7dfb1a3e Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@ocf.berkeley.edu>
Date: Fri, 8 Jan 2016 15:41:50 -0800
Subject: [PATCH 7/9] Remove unnecessary branch

---
 pre_commit/commands/run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py
index 11fbfbce..e6b62635 100644
--- a/pre_commit/commands/run.py
+++ b/pre_commit/commands/run.py
@@ -84,7 +84,7 @@ def _run_single_hook(hook, repo, args, write, skips=frozenset()):
         args,
         hook['files'],
         hook['exclude'],
-        frozenset(hook['types']) if 'types' in hook else frozenset(['file']),
+        frozenset(hook['types']),
     )
     if hook['id'] in skips:
         _print_user_skipped(hook, write, args)

From 85817b6f4eecdf340035a77228608626e701c0e5 Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@ocf.berkeley.edu>
Date: Fri, 8 Jan 2016 15:42:09 -0800
Subject: [PATCH 8/9] Add and run `fix-encoding-pragma` hook

---
 .pre-commit-config.yaml                                   | 7 ++++---
 pre_commit/__main__.py                                    | 1 +
 pre_commit/clientlib/validate_base.py                     | 1 +
 pre_commit/clientlib/validate_config.py                   | 1 +
 pre_commit/clientlib/validate_manifest.py                 | 1 +
 pre_commit/color.py                                       | 1 +
 pre_commit/commands/autoupdate.py                         | 1 +
 pre_commit/commands/clean.py                              | 1 +
 pre_commit/commands/identify.py                           | 1 +
 pre_commit/commands/install_uninstall.py                  | 1 +
 pre_commit/commands/run.py                                | 1 +
 pre_commit/constants.py                                   | 1 +
 pre_commit/error_handler.py                               | 1 +
 pre_commit/errors.py                                      | 1 +
 pre_commit/file_classifier/classifier.py                  | 2 +-
 pre_commit/file_classifier/extensions.py                  | 1 +
 pre_commit/file_classifier/interpreters.py                | 1 +
 pre_commit/five.py                                        | 1 +
 pre_commit/git.py                                         | 1 +
 pre_commit/jsonschema_extensions.py                       | 1 +
 pre_commit/languages/all.py                               | 1 +
 pre_commit/languages/helpers.py                           | 1 +
 pre_commit/languages/node.py                              | 1 +
 pre_commit/languages/pcre.py                              | 1 +
 pre_commit/languages/python.py                            | 1 +
 pre_commit/languages/ruby.py                              | 1 +
 pre_commit/languages/script.py                            | 1 +
 pre_commit/languages/system.py                            | 1 +
 pre_commit/logging_handler.py                             | 1 +
 pre_commit/main.py                                        | 1 +
 pre_commit/make_archives.py                               | 1 +
 pre_commit/manifest.py                                    | 1 +
 pre_commit/ordereddict.py                                 | 1 +
 pre_commit/output.py                                      | 1 +
 pre_commit/prefixed_command_runner.py                     | 1 +
 pre_commit/repository.py                                  | 1 +
 pre_commit/runner.py                                      | 1 +
 pre_commit/staged_files_only.py                           | 1 +
 pre_commit/store.py                                       | 1 +
 pre_commit/util.py                                        | 1 +
 setup.py                                                  | 1 +
 testing/auto_namedtuple.py                                | 1 +
 testing/fixtures.py                                       | 1 +
 testing/resources/arbitrary_bytes_repo/setup.py           | 1 +
 testing/resources/python3_hooks_repo/python3_hook/main.py | 1 +
 testing/resources/python3_hooks_repo/setup.py             | 1 +
 testing/resources/python_hooks_repo/foo/main.py           | 1 +
 testing/resources/python_hooks_repo/setup.py              | 1 +
 testing/util.py                                           | 1 +
 tests/clientlib/validate_base_test.py                     | 1 +
 tests/clientlib/validate_config_test.py                   | 1 +
 tests/clientlib/validate_manifest_test.py                 | 1 +
 tests/color_test.py                                       | 1 +
 tests/commands/autoupdate_test.py                         | 1 +
 tests/commands/clean_test.py                              | 1 +
 tests/commands/install_uninstall_test.py                  | 1 +
 tests/commands/run_test.py                                | 2 +-
 tests/conftest.py                                         | 1 +
 tests/file_classifier/classifier_test.py                  | 2 +-
 tests/git_test.py                                         | 1 +
 tests/jsonschema_extensions_test.py                       | 1 +
 tests/languages/all_test.py                               | 1 +
 tests/languages/helpers_test.py                           | 1 +
 tests/languages/python_test.py                            | 1 +
 tests/languages/ruby_test.py                              | 1 +
 tests/logging_handler_test.py                             | 1 +
 tests/main_test.py                                        | 1 +
 tests/make_archives_test.py                               | 1 +
 tests/manifest_test.py                                    | 1 +
 tests/output_test.py                                      | 1 +
 tests/prefixed_command_runner_test.py                     | 1 +
 tests/repository_test.py                                  | 1 +
 tests/runner_test.py                                      | 1 +
 tests/staged_files_only_test.py                           | 2 +-
 tests/store_test.py                                       | 1 +
 tests/util_test.py                                        | 1 +
 76 files changed, 79 insertions(+), 7 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ae00f49d..10f52fa0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,5 +1,5 @@
 -   repo: https://github.com/pre-commit/pre-commit-hooks.git
-    sha: cf550fcab3f12015f8676b8278b30e1a5bc10e70
+    sha: 5edf945ca57abe10a8f090f18e575eabb6a9585a
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -11,13 +11,14 @@
     -   id: name-tests-test
     -   id: requirements-txt-fixer
     -   id: flake8
+    -   id: fix-encoding-pragma
 -   repo: https://github.com/pre-commit/pre-commit.git
-    sha: 8dba3281d5051060755459dcf88e28fc26c27526
+    sha: 75aaadd4c455043b0fff3cc22eb480f6a120caaf
     hooks:
     -   id: validate_config
     -   id: validate_manifest
 -   repo: https://github.com/asottile/reorder_python_imports.git
-    sha: 3d86483455ab5bd06cc1069fdd5ac57be5463f10
+    sha: 8b583ac1beb0dd0f14c4bceb0a53bb1023cb3dd7
     hooks:
     -   id: reorder-python-imports
         language_version: python2.7
diff --git a/pre_commit/__main__.py b/pre_commit/__main__.py
index fc424d82..9f1574b0 100644
--- a/pre_commit/__main__.py
+++ b/pre_commit/__main__.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
 from pre_commit.main import main
diff --git a/pre_commit/clientlib/validate_base.py b/pre_commit/clientlib/validate_base.py
index 3d08a6c3..b0ae6d60 100644
--- a/pre_commit/clientlib/validate_base.py
+++ b/pre_commit/clientlib/validate_base.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import print_function
 from __future__ import unicode_literals
 
diff --git a/pre_commit/clientlib/validate_config.py b/pre_commit/clientlib/validate_config.py
index dcc4ebd7..502d1838 100644
--- a/pre_commit/clientlib/validate_config.py
+++ b/pre_commit/clientlib/validate_config.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from pre_commit.clientlib.validate_base import get_run_function
diff --git a/pre_commit/clientlib/validate_manifest.py b/pre_commit/clientlib/validate_manifest.py
index 182a5e1c..d0ff785a 100644
--- a/pre_commit/clientlib/validate_manifest.py
+++ b/pre_commit/clientlib/validate_manifest.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from pre_commit.clientlib.validate_base import get_run_function
diff --git a/pre_commit/color.py b/pre_commit/color.py
index a787821c..977d2ae8 100644
--- a/pre_commit/color.py
+++ b/pre_commit/color.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import sys
diff --git a/pre_commit/commands/autoupdate.py b/pre_commit/commands/autoupdate.py
index a446a172..3b7465a9 100644
--- a/pre_commit/commands/autoupdate.py
+++ b/pre_commit/commands/autoupdate.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import print_function
 from __future__ import unicode_literals
 
diff --git a/pre_commit/commands/clean.py b/pre_commit/commands/clean.py
index e0d307fb..505bb8ce 100644
--- a/pre_commit/commands/clean.py
+++ b/pre_commit/commands/clean.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import print_function
 from __future__ import unicode_literals
 
diff --git a/pre_commit/commands/identify.py b/pre_commit/commands/identify.py
index f1ec0ce6..f43c363d 100644
--- a/pre_commit/commands/identify.py
+++ b/pre_commit/commands/identify.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from pre_commit import git
 from pre_commit.file_classifier.classifier import classify
 
diff --git a/pre_commit/commands/install_uninstall.py b/pre_commit/commands/install_uninstall.py
index 9ab6fc57..0756db16 100644
--- a/pre_commit/commands/install_uninstall.py
+++ b/pre_commit/commands/install_uninstall.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import print_function
 from __future__ import unicode_literals
 
diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py
index e6b62635..b5638582 100644
--- a/pre_commit/commands/run.py
+++ b/pre_commit/commands/run.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import print_function
 from __future__ import unicode_literals
 
diff --git a/pre_commit/constants.py b/pre_commit/constants.py
index b89c29f8..e743d5aa 100644
--- a/pre_commit/constants.py
+++ b/pre_commit/constants.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 
diff --git a/pre_commit/error_handler.py b/pre_commit/error_handler.py
index 85d5602e..5c45c5f5 100644
--- a/pre_commit/error_handler.py
+++ b/pre_commit/error_handler.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import print_function
 from __future__ import unicode_literals
diff --git a/pre_commit/errors.py b/pre_commit/errors.py
index 4dedbfc2..58fd9737 100644
--- a/pre_commit/errors.py
+++ b/pre_commit/errors.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/pre_commit/file_classifier/classifier.py b/pre_commit/file_classifier/classifier.py
index 42481d23..907742ca 100644
--- a/pre_commit/file_classifier/classifier.py
+++ b/pre_commit/file_classifier/classifier.py
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# -*- coding: utf-8 -*-
 import io
 import re
 import string
diff --git a/pre_commit/file_classifier/extensions.py b/pre_commit/file_classifier/extensions.py
index 098a7162..3e5af1f5 100644
--- a/pre_commit/file_classifier/extensions.py
+++ b/pre_commit/file_classifier/extensions.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """List of known filename to file type mappings.
 
 The list consists of tuples of (filename regex, list of types).
diff --git a/pre_commit/file_classifier/interpreters.py b/pre_commit/file_classifier/interpreters.py
index 1434ac25..d44d351d 100644
--- a/pre_commit/file_classifier/interpreters.py
+++ b/pre_commit/file_classifier/interpreters.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 KNOWN_INTERPRETERS = [
     ('^python([23](\.[0-9]+)?)?$', ['python']),
     ('^(ba|da|tc|[ckz])?sh$', ['shell']),
diff --git a/pre_commit/five.py b/pre_commit/five.py
index 8b9a2b54..2df53cba 100644
--- a/pre_commit/five.py
+++ b/pre_commit/five.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 # pylint:disable=invalid-name
diff --git a/pre_commit/git.py b/pre_commit/git.py
index 15393ab4..941e29e8 100644
--- a/pre_commit/git.py
+++ b/pre_commit/git.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import functools
diff --git a/pre_commit/jsonschema_extensions.py b/pre_commit/jsonschema_extensions.py
index 0314e32e..ed271e52 100644
--- a/pre_commit/jsonschema_extensions.py
+++ b/pre_commit/jsonschema_extensions.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import copy
diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py
index 63c1d514..f2ad2893 100644
--- a/pre_commit/languages/all.py
+++ b/pre_commit/languages/all.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from pre_commit.languages import node
diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py
index b0add575..770bac6f 100644
--- a/pre_commit/languages/helpers.py
+++ b/pre_commit/languages/helpers.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import pipes
diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py
index 99b46df7..f33b4f9a 100644
--- a/pre_commit/languages/node.py
+++ b/pre_commit/languages/node.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import contextlib
diff --git a/pre_commit/languages/pcre.py b/pre_commit/languages/pcre.py
index 823e55cb..194efeb3 100644
--- a/pre_commit/languages/pcre.py
+++ b/pre_commit/languages/pcre.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from sys import platform
diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py
index 4c463874..86306698 100644
--- a/pre_commit/languages/python.py
+++ b/pre_commit/languages/python.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import contextlib
diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py
index a23df5d3..a7ac94c4 100644
--- a/pre_commit/languages/ruby.py
+++ b/pre_commit/languages/ruby.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import contextlib
diff --git a/pre_commit/languages/script.py b/pre_commit/languages/script.py
index 5ba871ea..4291f43d 100644
--- a/pre_commit/languages/script.py
+++ b/pre_commit/languages/script.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from pre_commit.languages.helpers import file_args_to_stdin
diff --git a/pre_commit/languages/system.py b/pre_commit/languages/system.py
index 3930422b..6f9ce265 100644
--- a/pre_commit/languages/system.py
+++ b/pre_commit/languages/system.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import shlex
diff --git a/pre_commit/logging_handler.py b/pre_commit/logging_handler.py
index 5dc2d227..fb369660 100644
--- a/pre_commit/logging_handler.py
+++ b/pre_commit/logging_handler.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import logging
diff --git a/pre_commit/main.py b/pre_commit/main.py
index 132e901a..c23880cc 100644
--- a/pre_commit/main.py
+++ b/pre_commit/main.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import argparse
diff --git a/pre_commit/make_archives.py b/pre_commit/make_archives.py
index ff6d3bda..22f909ad 100644
--- a/pre_commit/make_archives.py
+++ b/pre_commit/make_archives.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import print_function
 from __future__ import unicode_literals
diff --git a/pre_commit/manifest.py b/pre_commit/manifest.py
index 0738e5d4..428fab57 100644
--- a/pre_commit/manifest.py
+++ b/pre_commit/manifest.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os.path
diff --git a/pre_commit/ordereddict.py b/pre_commit/ordereddict.py
index 2844cb46..59a6904e 100644
--- a/pre_commit/ordereddict.py
+++ b/pre_commit/ordereddict.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/pre_commit/output.py b/pre_commit/output.py
index 84697ec0..d92e054c 100644
--- a/pre_commit/output.py
+++ b/pre_commit/output.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os
diff --git a/pre_commit/prefixed_command_runner.py b/pre_commit/prefixed_command_runner.py
index 2b1212a2..a5e7466f 100644
--- a/pre_commit/prefixed_command_runner.py
+++ b/pre_commit/prefixed_command_runner.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os
diff --git a/pre_commit/repository.py b/pre_commit/repository.py
index a0c0d01a..9ff14d7e 100644
--- a/pre_commit/repository.py
+++ b/pre_commit/repository.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import io
diff --git a/pre_commit/runner.py b/pre_commit/runner.py
index 88028939..96f4cef4 100644
--- a/pre_commit/runner.py
+++ b/pre_commit/runner.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os
diff --git a/pre_commit/staged_files_only.py b/pre_commit/staged_files_only.py
index a2978b99..93f40815 100644
--- a/pre_commit/staged_files_only.py
+++ b/pre_commit/staged_files_only.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import contextlib
diff --git a/pre_commit/store.py b/pre_commit/store.py
index 608472bf..4058a219 100644
--- a/pre_commit/store.py
+++ b/pre_commit/store.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import contextlib
diff --git a/pre_commit/util.py b/pre_commit/util.py
index 3736d2e5..f10da453 100644
--- a/pre_commit/util.py
+++ b/pre_commit/util.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import contextlib
diff --git a/setup.py b/setup.py
index 56951354..ee83daca 100644
--- a/setup.py
+++ b/setup.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from setuptools import find_packages
 from setuptools import setup
 
diff --git a/testing/auto_namedtuple.py b/testing/auto_namedtuple.py
index 02e08fef..89824658 100644
--- a/testing/auto_namedtuple.py
+++ b/testing/auto_namedtuple.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import collections
diff --git a/testing/fixtures.py b/testing/fixtures.py
index 36c7400c..1f159399 100644
--- a/testing/fixtures.py
+++ b/testing/fixtures.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/testing/resources/arbitrary_bytes_repo/setup.py b/testing/resources/arbitrary_bytes_repo/setup.py
index bf7690c0..827d92ed 100644
--- a/testing/resources/arbitrary_bytes_repo/setup.py
+++ b/testing/resources/arbitrary_bytes_repo/setup.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from setuptools import find_packages
 from setuptools import setup
 
diff --git a/testing/resources/python3_hooks_repo/python3_hook/main.py b/testing/resources/python3_hooks_repo/python3_hook/main.py
index ceeca0c4..18910019 100644
--- a/testing/resources/python3_hooks_repo/python3_hook/main.py
+++ b/testing/resources/python3_hooks_repo/python3_hook/main.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import print_function
 
 import sys
diff --git a/testing/resources/python3_hooks_repo/setup.py b/testing/resources/python3_hooks_repo/setup.py
index bf7690c0..827d92ed 100644
--- a/testing/resources/python3_hooks_repo/setup.py
+++ b/testing/resources/python3_hooks_repo/setup.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from setuptools import find_packages
 from setuptools import setup
 
diff --git a/testing/resources/python_hooks_repo/foo/main.py b/testing/resources/python_hooks_repo/foo/main.py
index 78c2c0f7..d6e087b5 100644
--- a/testing/resources/python_hooks_repo/foo/main.py
+++ b/testing/resources/python_hooks_repo/foo/main.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import print_function
 
 import sys
diff --git a/testing/resources/python_hooks_repo/setup.py b/testing/resources/python_hooks_repo/setup.py
index 556dd8f5..a5773f9a 100644
--- a/testing/resources/python_hooks_repo/setup.py
+++ b/testing/resources/python_hooks_repo/setup.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from setuptools import find_packages
 from setuptools import setup
 
diff --git a/testing/util.py b/testing/util.py
index 40ee389b..865cb778 100644
--- a/testing/util.py
+++ b/testing/util.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os
diff --git a/tests/clientlib/validate_base_test.py b/tests/clientlib/validate_base_test.py
index 5c44ab51..b6123822 100644
--- a/tests/clientlib/validate_base_test.py
+++ b/tests/clientlib/validate_base_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import pytest
diff --git a/tests/clientlib/validate_config_test.py b/tests/clientlib/validate_config_test.py
index 5631adbc..81fa7820 100644
--- a/tests/clientlib/validate_config_test.py
+++ b/tests/clientlib/validate_config_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import jsonschema
diff --git a/tests/clientlib/validate_manifest_test.py b/tests/clientlib/validate_manifest_test.py
index 4e51ade9..607c9b78 100644
--- a/tests/clientlib/validate_manifest_test.py
+++ b/tests/clientlib/validate_manifest_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import pytest
diff --git a/tests/color_test.py b/tests/color_test.py
index 500a9bbc..280377b8 100644
--- a/tests/color_test.py
+++ b/tests/color_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import sys
diff --git a/tests/commands/autoupdate_test.py b/tests/commands/autoupdate_test.py
index bd8fbe80..0b9349d7 100644
--- a/tests/commands/autoupdate_test.py
+++ b/tests/commands/autoupdate_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import shutil
diff --git a/tests/commands/clean_test.py b/tests/commands/clean_test.py
index bdbdc998..ee08fb73 100644
--- a/tests/commands/clean_test.py
+++ b/tests/commands/clean_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os.path
diff --git a/tests/commands/install_uninstall_test.py b/tests/commands/install_uninstall_test.py
index 9f22a788..a0d9aa6e 100644
--- a/tests/commands/install_uninstall_test.py
+++ b/tests/commands/install_uninstall_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/commands/run_test.py b/tests/commands/run_test.py
index 363743fb..10d3c9ef 100644
--- a/tests/commands/run_test.py
+++ b/tests/commands/run_test.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import functools
diff --git a/tests/conftest.py b/tests/conftest.py
index ea50bee6..dad94559 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/file_classifier/classifier_test.py b/tests/file_classifier/classifier_test.py
index a115b482..106e52cb 100644
--- a/tests/file_classifier/classifier_test.py
+++ b/tests/file_classifier/classifier_test.py
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from contextlib import contextmanager
diff --git a/tests/git_test.py b/tests/git_test.py
index 1e832a76..875916fa 100644
--- a/tests/git_test.py
+++ b/tests/git_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/jsonschema_extensions_test.py b/tests/jsonschema_extensions_test.py
index 65948564..6a802003 100644
--- a/tests/jsonschema_extensions_test.py
+++ b/tests/jsonschema_extensions_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import jsonschema.exceptions
diff --git a/tests/languages/all_test.py b/tests/languages/all_test.py
index 2254e388..2d36f0a2 100644
--- a/tests/languages/all_test.py
+++ b/tests/languages/all_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import inspect
diff --git a/tests/languages/helpers_test.py b/tests/languages/helpers_test.py
index 8497ceb0..98384206 100644
--- a/tests/languages/helpers_test.py
+++ b/tests/languages/helpers_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/languages/python_test.py b/tests/languages/python_test.py
index 8715b690..60573a64 100644
--- a/tests/languages/python_test.py
+++ b/tests/languages/python_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/languages/ruby_test.py b/tests/languages/ruby_test.py
index 1eddea1d..cf0fd034 100644
--- a/tests/languages/ruby_test.py
+++ b/tests/languages/ruby_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os.path
diff --git a/tests/logging_handler_test.py b/tests/logging_handler_test.py
index 2273de9b..142a0553 100644
--- a/tests/logging_handler_test.py
+++ b/tests/logging_handler_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import mock
diff --git a/tests/main_test.py b/tests/main_test.py
index 537ff23c..9b84fc83 100644
--- a/tests/main_test.py
+++ b/tests/main_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/make_archives_test.py b/tests/make_archives_test.py
index fc267b63..cf1a7a1b 100644
--- a/tests/make_archives_test.py
+++ b/tests/make_archives_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/manifest_test.py b/tests/manifest_test.py
index eb3b65da..abd58f35 100644
--- a/tests/manifest_test.py
+++ b/tests/manifest_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/output_test.py b/tests/output_test.py
index eca7a3d7..f3a39423 100644
--- a/tests/output_test.py
+++ b/tests/output_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import mock
diff --git a/tests/prefixed_command_runner_test.py b/tests/prefixed_command_runner_test.py
index bafcae72..f0ead97a 100644
--- a/tests/prefixed_command_runner_test.py
+++ b/tests/prefixed_command_runner_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os
diff --git a/tests/repository_test.py b/tests/repository_test.py
index ef870b53..81e763c3 100644
--- a/tests/repository_test.py
+++ b/tests/repository_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/runner_test.py b/tests/runner_test.py
index 782e8d53..48c71ed5 100644
--- a/tests/runner_test.py
+++ b/tests/runner_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/staged_files_only_test.py b/tests/staged_files_only_test.py
index 00f4cca9..1e7ac57a 100644
--- a/tests/staged_files_only_test.py
+++ b/tests/staged_files_only_test.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/store_test.py b/tests/store_test.py
index 21aceb64..25866b43 100644
--- a/tests/store_test.py
+++ b/tests/store_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
diff --git a/tests/util_test.py b/tests/util_test.py
index 1361d639..eaac7e78 100644
--- a/tests/util_test.py
+++ b/tests/util_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import os

From f15fcf0e22c22c15838a3e996645ae2098222cb3 Mon Sep 17 00:00:00 2001
From: Chris Kuehl <ckuehl@ocf.berkeley.edu>
Date: Fri, 8 Jan 2016 16:04:39 -0800
Subject: [PATCH 9/9] Clarify docstring in shebang reader

---
 pre_commit/file_classifier/classifier.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pre_commit/file_classifier/classifier.py b/pre_commit/file_classifier/classifier.py
index 907742ca..162d2b85 100644
--- a/pre_commit/file_classifier/classifier.py
+++ b/pre_commit/file_classifier/classifier.py
@@ -85,9 +85,10 @@ def _guess_types_from_shebang(path):
 def _read_interpreter_from_shebang(path):
     """Read an interpreter from a file's shebang.
 
-    The first line of a script is guaranteed to be ASCII, so we read ASCII
-    until we hit a newline (at which point we check if we read a valid shebang)
-    or a non-ASCII character (at which point we bail).
+    The first line of a script which has a valid shebang is guaranteed to be
+    ASCII, so we read ASCII until we hit a newline (at which point we check if
+    we read a valid shebang) or a non-ASCII character (at which point we bail,
+    because this can't be a valid script-with-shebang).
 
     :param path: path to text file
     :return: interpreter, or None if no shebang could be read