diff --git a/pre_commit/commands/install_uninstall.py b/pre_commit/commands/install_uninstall.py index 9ab6fc57..a60f7273 100644 --- a/pre_commit/commands/install_uninstall.py +++ b/pre_commit/commands/install_uninstall.py @@ -5,10 +5,10 @@ import io import logging import os import os.path -import stat import sys from pre_commit.logging_handler import LoggingHandler +from pre_commit.util import make_executable from pre_commit.util import mkdirp from pre_commit.util import resource_filename @@ -42,14 +42,6 @@ def is_previous_pre_commit(filename): return any(hash in contents for hash in PREVIOUS_IDENTIFYING_HASHES) -def make_executable(filename): - original_mode = os.stat(filename).st_mode - os.chmod( - filename, - original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, - ) - - def install(runner, overwrite=False, hooks=False, hook_type='pre-commit'): """Install the pre-commit hooks.""" hook_path = runner.get_hook_path(hook_type) diff --git a/pre_commit/parse_shebang.py b/pre_commit/parse_shebang.py new file mode 100644 index 00000000..df10c6d3 --- /dev/null +++ b/pre_commit/parse_shebang.py @@ -0,0 +1,95 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import io +import os.path +import shlex +import string + +from pre_commit import five + + +printable = frozenset(string.printable) + + +def parse_bytesio(bytesio): + """Parse the shebang from a file opened for reading binary.""" + if bytesio.read(2) != b'#!': + return () + first_line = bytesio.readline() + try: + first_line = first_line.decode('US-ASCII') + except UnicodeDecodeError: + return () + + # Require only printable ascii + for c in first_line: + if c not in printable: + return () + + # shlex.split is horribly broken in py26 on text strings + cmd = tuple(shlex.split(five.n(first_line))) + if cmd[0] == '/usr/bin/env': + cmd = cmd[1:] + return cmd + + +def parse_filename(filename): + """Parse the shebang given a filename.""" + if not os.path.exists(filename) or not os.access(filename, os.X_OK): + return () + + with io.open(filename, 'rb') as f: + return parse_bytesio(f) + + +def find_executable(exe, _environ=None): + exe = os.path.normpath(exe) + if os.sep in exe: + return exe + + environ = _environ if _environ is not None else os.environ + + if 'PATHEXT' in environ: + possible_exe_names = (exe,) + tuple( + exe + ext.lower() for ext in environ['PATHEXT'].split(os.pathsep) + ) + else: + possible_exe_names = (exe,) + + for path in environ.get('PATH', '').split(os.pathsep): + for possible_exe_name in possible_exe_names: + joined = os.path.join(path, possible_exe_name) + if os.path.isfile(joined) and os.access(joined, os.X_OK): + return joined + else: + return None + + +def normexe(orig_exe): + if os.sep not in orig_exe: + exe = find_executable(orig_exe) + if exe is None: + raise OSError('Executable {0} not found'.format(orig_exe)) + return exe + else: + return orig_exe + + +def normalize_cmd(cmd): + """Fixes for the following issues on windows + - http://bugs.python.org/issue8557 + - windows does not parse shebangs + + This function also makes deep-path shebangs work just fine + """ + # Use PATH to determine the executable + exe = normexe(cmd[0]) + + # Figure out the shebang from the resulting command + cmd = parse_filename(exe) + (exe,) + cmd[1:] + + # This could have given us back another bare executable + exe = normexe(cmd[0]) + + return (exe,) + cmd[1:] diff --git a/pre_commit/util.py b/pre_commit/util.py index 046cf96e..57303f56 100644 --- a/pre_commit/util.py +++ b/pre_commit/util.py @@ -14,6 +14,7 @@ import tempfile import pkg_resources from pre_commit import five +from pre_commit import parse_shebang @contextlib.contextmanager @@ -110,6 +111,14 @@ def resource_filename(filename): ) +def make_executable(filename): + original_mode = os.stat(filename).st_mode + os.chmod( + filename, + original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, + ) + + class CalledProcessError(RuntimeError): def __init__(self, returncode, cmd, expected_returncode, output=None): super(CalledProcessError, self).__init__( @@ -166,12 +175,14 @@ def cmd_output(*cmd, **kwargs): } # py2/py3 on windows are more strict about the types here - cmd = [five.n(arg) for arg in cmd] + cmd = tuple(five.n(arg) for arg in cmd) kwargs['env'] = dict( (five.n(key), five.n(value)) for key, value in kwargs.pop('env', {}).items() ) or None + cmd = parse_shebang.normalize_cmd(cmd) + popen_kwargs.update(kwargs) proc = __popen(cmd, **popen_kwargs) stdout, stderr = proc.communicate() diff --git a/tests/commands/install_uninstall_test.py b/tests/commands/install_uninstall_test.py index 7717a1f0..331d857f 100644 --- a/tests/commands/install_uninstall_test.py +++ b/tests/commands/install_uninstall_test.py @@ -15,12 +15,12 @@ from pre_commit.commands.install_uninstall import IDENTIFYING_HASH from pre_commit.commands.install_uninstall import install from pre_commit.commands.install_uninstall import is_our_pre_commit from pre_commit.commands.install_uninstall import is_previous_pre_commit -from pre_commit.commands.install_uninstall import make_executable from pre_commit.commands.install_uninstall import PREVIOUS_IDENTIFYING_HASHES from pre_commit.commands.install_uninstall import uninstall from pre_commit.runner import Runner from pre_commit.util import cmd_output from pre_commit.util import cwd +from pre_commit.util import make_executable from pre_commit.util import mkdirp from pre_commit.util import resource_filename from testing.fixtures import git_dir @@ -473,6 +473,8 @@ def test_installed_from_venv(tempdir_factory): 'TERM': os.environ.get('TERM', ''), # Windows needs this to import `random` 'SYSTEMROOT': os.environ.get('SYSTEMROOT', ''), + # Windows needs this to resolve executables + 'PATHEXT': os.environ.get('PATHEXT', ''), }, ) assert ret == 0 diff --git a/tests/parse_shebang_test.py b/tests/parse_shebang_test.py new file mode 100644 index 00000000..c26ff73f --- /dev/null +++ b/tests/parse_shebang_test.py @@ -0,0 +1,154 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import contextlib +import distutils.spawn +import io +import os +import sys + +import pytest + +from pre_commit import parse_shebang +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import Var +from pre_commit.util import make_executable + + +@pytest.mark.parametrize( + ('s', 'expected'), + ( + (b'', ()), + (b'#!/usr/bin/python', ('/usr/bin/python',)), + (b'#!/usr/bin/env python', ('python',)), + (b'#! /usr/bin/python', ('/usr/bin/python',)), + (b'#!/usr/bin/foo python', ('/usr/bin/foo', 'python')), + (b'\xf9\x93\x01\x42\xcd', ()), + (b'#!\xf9\x93\x01\x42\xcd', ()), + (b'#!\x00\x00\x00\x00', ()), + ), +) +def test_parse_bytesio(s, expected): + assert parse_shebang.parse_bytesio(io.BytesIO(s)) == expected + + +def test_file_doesnt_exist(): + assert parse_shebang.parse_filename('herp derp derp') == () + + +@pytest.mark.xfail( + sys.platform == 'win32', reason='Windows says everything is X_OK', +) +def test_file_not_executable(tmpdir): + x = tmpdir.join('f') + x.write_text('#!/usr/bin/env python', encoding='UTF-8') + assert parse_shebang.parse_filename(x.strpath) == () + + +def test_simple_case(tmpdir): + x = tmpdir.join('f') + x.write_text('#!/usr/bin/env python', encoding='UTF-8') + make_executable(x.strpath) + assert parse_shebang.parse_filename(x.strpath) == ('python',) + + +def test_find_executable_full_path(): + assert parse_shebang.find_executable(sys.executable) == sys.executable + + +def test_find_executable_on_path(): + expected = distutils.spawn.find_executable('echo') + assert parse_shebang.find_executable('echo') == expected + + +def test_find_executable_not_found_none(): + assert parse_shebang.find_executable('not-a-real-executable') is None + + +def write_executable(shebang, filename='run'): + os.mkdir('bin') + path = os.path.join('bin', filename) + with io.open(path, 'w') as f: + f.write('#!{0}'.format(shebang)) + make_executable(path) + return path + + +@contextlib.contextmanager +def bin_on_path(): + bindir = os.path.join(os.getcwd(), 'bin') + with envcontext((('PATH', (bindir, os.pathsep, Var('PATH'))),)): + yield + + +def test_find_executable_path_added(in_tmpdir): + path = os.path.abspath(write_executable('/usr/bin/env sh')) + assert parse_shebang.find_executable('run') is None + with bin_on_path(): + assert parse_shebang.find_executable('run') == path + + +def test_find_executable_path_ext(in_tmpdir): + """Windows exports PATHEXT as a list of extensions to automatically add + to executables when doing PATH searching. + """ + exe_path = os.path.abspath(write_executable( + '/usr/bin/env sh', filename='run.myext', + )) + env_path = {'PATH': os.path.dirname(exe_path)} + env_path_ext = dict(env_path, PATHEXT=os.pathsep.join(('.exe', '.myext'))) + assert parse_shebang.find_executable('run') is None + assert parse_shebang.find_executable('run', _environ=env_path) is None + ret = parse_shebang.find_executable('run.myext', _environ=env_path) + assert ret == exe_path + ret = parse_shebang.find_executable('run', _environ=env_path_ext) + assert ret == exe_path + + +def test_normexe_does_not_exist(): + with pytest.raises(OSError) as excinfo: + parse_shebang.normexe('i-dont-exist-lol') + assert excinfo.value.args == ('Executable i-dont-exist-lol not found',) + + +def test_normexe_already_full_path(): + assert parse_shebang.normexe(sys.executable) == sys.executable + + +def test_normexe_gives_full_path(): + expected = distutils.spawn.find_executable('echo') + assert parse_shebang.normexe('echo') == expected + assert os.sep in expected + + +def test_normalize_cmd_trivial(): + cmd = (distutils.spawn.find_executable('echo'), 'hi') + assert parse_shebang.normalize_cmd(cmd) == cmd + + +def test_normalize_cmd_PATH(): + cmd = ('python', '--version') + expected = (distutils.spawn.find_executable('python'), '--version') + assert parse_shebang.normalize_cmd(cmd) == expected + + +def test_normalize_cmd_shebang(in_tmpdir): + python = distutils.spawn.find_executable('python') + path = write_executable(python.replace(os.sep, '/')) + assert parse_shebang.normalize_cmd((path,)) == (python, path) + + +def test_normalize_cmd_PATH_shebang_full_path(in_tmpdir): + python = distutils.spawn.find_executable('python') + path = write_executable(python.replace(os.sep, '/')) + with bin_on_path(): + ret = parse_shebang.normalize_cmd(('run',)) + assert ret == (python, os.path.abspath(path)) + + +def test_normalize_cmd_PATH_shebang_PATH(in_tmpdir): + python = distutils.spawn.find_executable('python') + path = write_executable('/usr/bin/env python') + with bin_on_path(): + ret = parse_shebang.normalize_cmd(('run',)) + assert ret == (python, os.path.abspath(path)) diff --git a/tests/prefixed_command_runner_test.py b/tests/prefixed_command_runner_test.py index 3f691b4b..bb412101 100644 --- a/tests/prefixed_command_runner_test.py +++ b/tests/prefixed_command_runner_test.py @@ -78,7 +78,7 @@ def test_run_substitutes_prefix(popen_mock, makedirs_mock): ) ret = instance.run(['{prefix}bar', 'baz'], retcode=None) popen_mock.assert_called_once_with( - [five.n(os.path.join('prefix', 'bar')), five.n('baz')], + (five.n(os.path.join('prefix', 'bar')), five.n('baz')), env=None, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -132,4 +132,4 @@ def test_raises_on_error(popen_mock, makedirs_mock): instance = PrefixedCommandRunner( '.', popen=popen_mock, makedirs=makedirs_mock, ) - instance.run(['foo']) + instance.run(['echo'])