Add utility for parsing shebangs and resolving PATH

pre_commit/commands/install_uninstall.py

@@ -5,10 +5,10 @@ import io
 import logging
 import os
 import os.path
-import stat
 import sys
 
 from pre_commit.logging_handler import LoggingHandler
+from pre_commit.util import make_executable
 from pre_commit.util import mkdirp
 from pre_commit.util import resource_filename
 
@@ -42,14 +42,6 @@ def is_previous_pre_commit(filename):
     return any(hash in contents for hash in PREVIOUS_IDENTIFYING_HASHES)
 
 
-def make_executable(filename):
-    original_mode = os.stat(filename).st_mode
-    os.chmod(
-        filename,
-        original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
-    )
-
-
 def install(runner, overwrite=False, hooks=False, hook_type='pre-commit'):
     """Install the pre-commit hooks."""
     hook_path = runner.get_hook_path(hook_type)

pre_commit/parse_shebang.py

@@ -0,0 +1,95 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import io
+import os.path
+import shlex
+import string
+
+from pre_commit import five
+
+
+printable = frozenset(string.printable)
+
+
+def parse_bytesio(bytesio):
+    """Parse the shebang from a file opened for reading binary."""
+    if bytesio.read(2) != b'#!':
+        return ()
+    first_line = bytesio.readline()
+    try:
+        first_line = first_line.decode('US-ASCII')
+    except UnicodeDecodeError:
+        return ()
+
+    # Require only printable ascii
+    for c in first_line:
+        if c not in printable:
+            return ()
+
+    # shlex.split is horribly broken in py26 on text strings
+    cmd = tuple(shlex.split(five.n(first_line)))
+    if cmd[0] == '/usr/bin/env':
+        cmd = cmd[1:]
+    return cmd
+
+
+def parse_filename(filename):
+    """Parse the shebang given a filename."""
+    if not os.path.exists(filename) or not os.access(filename, os.X_OK):
+        return ()
+
+    with io.open(filename, 'rb') as f:
+        return parse_bytesio(f)
+
+
+def find_executable(exe, _environ=None):
+    exe = os.path.normpath(exe)
+    if os.sep in exe:
+        return exe
+
+    environ = _environ if _environ is not None else os.environ
+
+    if 'PATHEXT' in environ:
+        possible_exe_names = (exe,) + tuple(
+            exe + ext.lower() for ext in environ['PATHEXT'].split(os.pathsep)
+        )
+    else:
+        possible_exe_names = (exe,)
+
+    for path in environ.get('PATH', '').split(os.pathsep):
+        for possible_exe_name in possible_exe_names:
+            joined = os.path.join(path, possible_exe_name)
+            if os.path.isfile(joined) and os.access(joined, os.X_OK):
+                return joined
+    else:
+        return None
+
+
+def normexe(orig_exe):
+    if os.sep not in orig_exe:
+        exe = find_executable(orig_exe)
+        if exe is None:
+            raise OSError('Executable {0} not found'.format(orig_exe))
+        return exe
+    else:
+        return orig_exe
+
+
+def normalize_cmd(cmd):
+    """Fixes for the following issues on windows
+    - http://bugs.python.org/issue8557
+    - windows does not parse shebangs
+
+    This function also makes deep-path shebangs work just fine
+    """
+    # Use PATH to determine the executable
+    exe = normexe(cmd[0])
+
+    # Figure out the shebang from the resulting command
+    cmd = parse_filename(exe) + (exe,) + cmd[1:]
+
+    # This could have given us back another bare executable
+    exe = normexe(cmd[0])
+
+    return (exe,) + cmd[1:]

pre_commit/util.py

@@ -14,6 +14,7 @@ import tempfile
 import pkg_resources
 
 from pre_commit import five
+from pre_commit import parse_shebang
 
 
 @contextlib.contextmanager
@@ -110,6 +111,14 @@ def resource_filename(filename):
     )
 
 
+def make_executable(filename):
+    original_mode = os.stat(filename).st_mode
+    os.chmod(
+        filename,
+        original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
+    )
+
+
 class CalledProcessError(RuntimeError):
     def __init__(self, returncode, cmd, expected_returncode, output=None):
         super(CalledProcessError, self).__init__(
@@ -166,12 +175,14 @@ def cmd_output(*cmd, **kwargs):
     }
 
     # py2/py3 on windows are more strict about the types here
-    cmd = [five.n(arg) for arg in cmd]
+    cmd = tuple(five.n(arg) for arg in cmd)
     kwargs['env'] = dict(
         (five.n(key), five.n(value))
         for key, value in kwargs.pop('env', {}).items()
     ) or None
 
+    cmd = parse_shebang.normalize_cmd(cmd)
+
     popen_kwargs.update(kwargs)
     proc = __popen(cmd, **popen_kwargs)
     stdout, stderr = proc.communicate()

tests/commands/install_uninstall_test.py

@@ -15,12 +15,12 @@ from pre_commit.commands.install_uninstall import IDENTIFYING_HASH
 from pre_commit.commands.install_uninstall import install
 from pre_commit.commands.install_uninstall import is_our_pre_commit
 from pre_commit.commands.install_uninstall import is_previous_pre_commit
-from pre_commit.commands.install_uninstall import make_executable
 from pre_commit.commands.install_uninstall import PREVIOUS_IDENTIFYING_HASHES
 from pre_commit.commands.install_uninstall import uninstall
 from pre_commit.runner import Runner
 from pre_commit.util import cmd_output
 from pre_commit.util import cwd
+from pre_commit.util import make_executable
 from pre_commit.util import mkdirp
 from pre_commit.util import resource_filename
 from testing.fixtures import git_dir
@@ -473,6 +473,8 @@ def test_installed_from_venv(tempdir_factory):
                 'TERM': os.environ.get('TERM', ''),
                 # Windows needs this to import `random`
                 'SYSTEMROOT': os.environ.get('SYSTEMROOT', ''),
+                # Windows needs this to resolve executables
+                'PATHEXT': os.environ.get('PATHEXT', ''),
             },
         )
         assert ret == 0

tests/parse_shebang_test.py

@@ -0,0 +1,154 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import contextlib
+import distutils.spawn
+import io
+import os
+import sys
+
+import pytest
+
+from pre_commit import parse_shebang
+from pre_commit.envcontext import envcontext
+from pre_commit.envcontext import Var
+from pre_commit.util import make_executable
+
+
+@pytest.mark.parametrize(
+    ('s', 'expected'),
+    (
+        (b'', ()),
+        (b'#!/usr/bin/python', ('/usr/bin/python',)),
+        (b'#!/usr/bin/env python', ('python',)),
+        (b'#! /usr/bin/python', ('/usr/bin/python',)),
+        (b'#!/usr/bin/foo  python', ('/usr/bin/foo', 'python')),
+        (b'\xf9\x93\x01\x42\xcd', ()),
+        (b'#!\xf9\x93\x01\x42\xcd', ()),
+        (b'#!\x00\x00\x00\x00', ()),
+    ),
+)
+def test_parse_bytesio(s, expected):
+    assert parse_shebang.parse_bytesio(io.BytesIO(s)) == expected
+
+
+def test_file_doesnt_exist():
+    assert parse_shebang.parse_filename('herp derp derp') == ()
+
+
+@pytest.mark.xfail(
+    sys.platform == 'win32', reason='Windows says everything is X_OK',
+)
+def test_file_not_executable(tmpdir):
+    x = tmpdir.join('f')
+    x.write_text('#!/usr/bin/env python', encoding='UTF-8')
+    assert parse_shebang.parse_filename(x.strpath) == ()
+
+
+def test_simple_case(tmpdir):
+    x = tmpdir.join('f')
+    x.write_text('#!/usr/bin/env python', encoding='UTF-8')
+    make_executable(x.strpath)
+    assert parse_shebang.parse_filename(x.strpath) == ('python',)
+
+
+def test_find_executable_full_path():
+    assert parse_shebang.find_executable(sys.executable) == sys.executable
+
+
+def test_find_executable_on_path():
+    expected = distutils.spawn.find_executable('echo')
+    assert parse_shebang.find_executable('echo') == expected
+
+
+def test_find_executable_not_found_none():
+    assert parse_shebang.find_executable('not-a-real-executable') is None
+
+
+def write_executable(shebang, filename='run'):
+    os.mkdir('bin')
+    path = os.path.join('bin', filename)
+    with io.open(path, 'w') as f:
+        f.write('#!{0}'.format(shebang))
+    make_executable(path)
+    return path
+
+
+@contextlib.contextmanager
+def bin_on_path():
+    bindir = os.path.join(os.getcwd(), 'bin')
+    with envcontext((('PATH', (bindir, os.pathsep, Var('PATH'))),)):
+        yield
+
+
+def test_find_executable_path_added(in_tmpdir):
+    path = os.path.abspath(write_executable('/usr/bin/env sh'))
+    assert parse_shebang.find_executable('run') is None
+    with bin_on_path():
+        assert parse_shebang.find_executable('run') == path
+
+
+def test_find_executable_path_ext(in_tmpdir):
+    """Windows exports PATHEXT as a list of extensions to automatically add
+    to executables when doing PATH searching.
+    """
+    exe_path = os.path.abspath(write_executable(
+        '/usr/bin/env sh', filename='run.myext',
+    ))
+    env_path = {'PATH': os.path.dirname(exe_path)}
+    env_path_ext = dict(env_path, PATHEXT=os.pathsep.join(('.exe', '.myext')))
+    assert parse_shebang.find_executable('run') is None
+    assert parse_shebang.find_executable('run', _environ=env_path) is None
+    ret = parse_shebang.find_executable('run.myext', _environ=env_path)
+    assert ret == exe_path
+    ret = parse_shebang.find_executable('run', _environ=env_path_ext)
+    assert ret == exe_path
+
+
+def test_normexe_does_not_exist():
+    with pytest.raises(OSError) as excinfo:
+        parse_shebang.normexe('i-dont-exist-lol')
+    assert excinfo.value.args == ('Executable i-dont-exist-lol not found',)
+
+
+def test_normexe_already_full_path():
+    assert parse_shebang.normexe(sys.executable) == sys.executable
+
+
+def test_normexe_gives_full_path():
+    expected = distutils.spawn.find_executable('echo')
+    assert parse_shebang.normexe('echo') == expected
+    assert os.sep in expected
+
+
+def test_normalize_cmd_trivial():
+    cmd = (distutils.spawn.find_executable('echo'), 'hi')
+    assert parse_shebang.normalize_cmd(cmd) == cmd
+
+
+def test_normalize_cmd_PATH():
+    cmd = ('python', '--version')
+    expected = (distutils.spawn.find_executable('python'), '--version')
+    assert parse_shebang.normalize_cmd(cmd) == expected
+
+
+def test_normalize_cmd_shebang(in_tmpdir):
+    python = distutils.spawn.find_executable('python')
+    path = write_executable(python.replace(os.sep, '/'))
+    assert parse_shebang.normalize_cmd((path,)) == (python, path)
+
+
+def test_normalize_cmd_PATH_shebang_full_path(in_tmpdir):
+    python = distutils.spawn.find_executable('python')
+    path = write_executable(python.replace(os.sep, '/'))
+    with bin_on_path():
+        ret = parse_shebang.normalize_cmd(('run',))
+        assert ret == (python, os.path.abspath(path))
+
+
+def test_normalize_cmd_PATH_shebang_PATH(in_tmpdir):
+    python = distutils.spawn.find_executable('python')
+    path = write_executable('/usr/bin/env python')
+    with bin_on_path():
+        ret = parse_shebang.normalize_cmd(('run',))
+        assert ret == (python, os.path.abspath(path))

tests/prefixed_command_runner_test.py

@@ -78,7 +78,7 @@ def test_run_substitutes_prefix(popen_mock, makedirs_mock):
     )
     ret = instance.run(['{prefix}bar', 'baz'], retcode=None)
     popen_mock.assert_called_once_with(
-        [five.n(os.path.join('prefix', 'bar')), five.n('baz')],
+        (five.n(os.path.join('prefix', 'bar')), five.n('baz')),
         env=None,
         stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
@@ -132,4 +132,4 @@ def test_raises_on_error(popen_mock, makedirs_mock):
         instance = PrefixedCommandRunner(
             '.', popen=popen_mock, makedirs=makedirs_mock,
         )
-        instance.run(['foo'])
+        instance.run(['echo'])