Implement a simplified xargs in python

pre_commit/commands/run.py

@@ -86,7 +86,7 @@ def _run_single_hook(hook, repo, args, write, skips=frozenset()):
     sys.stdout.flush()
 
     diff_before = cmd_output('git', 'diff', retcode=None, encoding=None)
-    retcode, stdout, stderr = repo.run_hook(hook, filenames)
+    retcode, stdout, stderr = repo.run_hook(hook, tuple(filenames))
     diff_after = cmd_output('git', 'diff', retcode=None, encoding=None)
 
     file_modifications = diff_before != diff_after

pre_commit/languages/helpers.py

@@ -12,18 +12,3 @@ def environment_dir(ENVIRONMENT_DIR, language_version):
         return None
     else:
         return '{0}-{1}'.format(ENVIRONMENT_DIR, language_version)
-
-
-def file_args_to_stdin(file_args):
-    return '\0'.join(list(file_args) + [''])
-
-
-def run_hook(cmd_args, file_args):
-    return cmd_output(
-        # Use -s 4000 (slightly less than posix mandated minimum)
-        # This is to prevent "xargs: ... Bad file number" on windows
-        'xargs', '-0', '-s4000', *cmd_args,
-        stdin=file_args_to_stdin(file_args),
-        retcode=None,
-        encoding=None
-    )

pre_commit/languages/node.py

@@ -8,6 +8,7 @@ from pre_commit.envcontext import envcontext
 from pre_commit.envcontext import Var
 from pre_commit.languages import helpers
 from pre_commit.util import clean_path_on_failure
+from pre_commit.xargs import xargs
 
 
 ENVIRONMENT_DIR = 'node_env'
@@ -63,6 +64,4 @@ def install_environment(
 
 def run_hook(repo_cmd_runner, hook, file_args):
     with in_env(repo_cmd_runner, hook['language_version']):
-        return helpers.run_hook(
-            (hook['entry'],) + tuple(hook['args']), file_args,
-        )
+        return xargs((hook['entry'],) + tuple(hook['args']), file_args)

pre_commit/languages/pcre.py

@@ -2,8 +2,8 @@ from __future__ import unicode_literals
 
 from sys import platform
 
-from pre_commit.languages import helpers
 from pre_commit.util import shell_escape
+from pre_commit.xargs import xargs
 
 
 ENVIRONMENT_DIR = None
@@ -24,7 +24,7 @@ def run_hook(repo_cmd_runner, hook, file_args):
     )
 
     # For PCRE the entry is the regular expression to match
-    return helpers.run_hook(
+    return xargs(
         (
             'sh', '-c',
             # Grep usually returns 0 for matches, and nonzero for non-matches

pre_commit/languages/python.py

@@ -10,6 +10,7 @@ from pre_commit.envcontext import UNSET
 from pre_commit.envcontext import Var
 from pre_commit.languages import helpers
 from pre_commit.util import clean_path_on_failure
+from pre_commit.xargs import xargs
 
 
 ENVIRONMENT_DIR = 'py_env'
@@ -80,6 +81,4 @@ def install_environment(
 
 def run_hook(repo_cmd_runner, hook, file_args):
     with in_env(repo_cmd_runner, hook['language_version']):
-        return helpers.run_hook(
-            (hook['entry'],) + tuple(hook['args']), file_args,
-        )
+        return xargs((hook['entry'],) + tuple(hook['args']), file_args)

pre_commit/languages/ruby.py

@@ -12,6 +12,7 @@ from pre_commit.util import CalledProcessError
 from pre_commit.util import clean_path_on_failure
 from pre_commit.util import resource_filename
 from pre_commit.util import tarfile_open
+from pre_commit.xargs import xargs
 
 
 ENVIRONMENT_DIR = 'rbenv'
@@ -125,6 +126,4 @@ def install_environment(
 
 def run_hook(repo_cmd_runner, hook, file_args):
     with in_env(repo_cmd_runner, hook['language_version']):
-        return helpers.run_hook(
-            (hook['entry'],) + tuple(hook['args']), file_args,
-        )
+        return xargs((hook['entry'],) + tuple(hook['args']), file_args)

pre_commit/languages/script.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from pre_commit.languages import helpers
+from pre_commit.xargs import xargs
 
 
 ENVIRONMENT_DIR = None
@@ -16,7 +16,7 @@ def install_environment(
 
 
 def run_hook(repo_cmd_runner, hook, file_args):
-    return helpers.run_hook(
+    return xargs(
         (repo_cmd_runner.prefix_dir + hook['entry'],) + tuple(hook['args']),
         file_args,
     )

pre_commit/languages/system.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import shlex
 
-from pre_commit.languages import helpers
+from pre_commit.xargs import xargs
 
 
 ENVIRONMENT_DIR = None
@@ -18,6 +18,6 @@ def install_environment(
 
 
 def run_hook(repo_cmd_runner, hook, file_args):
-    return helpers.run_hook(
+    return xargs(
         tuple(shlex.split(hook['entry'])) + tuple(hook['args']), file_args,
     )

pre_commit/util.py

@@ -160,7 +160,6 @@ class CalledProcessError(RuntimeError):
 
 def cmd_output(*cmd, **kwargs):
     retcode = kwargs.pop('retcode', 0)
-    stdin = kwargs.pop('stdin', None)
     encoding = kwargs.pop('encoding', 'UTF-8')
     __popen = kwargs.pop('__popen', subprocess.Popen)
 
@@ -170,9 +169,6 @@ def cmd_output(*cmd, **kwargs):
         'stderr': subprocess.PIPE,
     }
 
-    if stdin is not None:
-        stdin = stdin.encode('UTF-8')
-
     # py2/py3 on windows are more strict about the types here
     cmd = [five.n(arg) for arg in cmd]
     kwargs['env'] = dict(
@@ -182,7 +178,7 @@ def cmd_output(*cmd, **kwargs):
 
     popen_kwargs.update(kwargs)
     proc = __popen(cmd, **popen_kwargs)
-    stdout, stderr = proc.communicate(stdin)
+    stdout, stderr = proc.communicate()
     if encoding is not None and stdout is not None:
         stdout = stdout.decode(encoding)
     if encoding is not None and stderr is not None:

pre_commit/xargs.py

@@ -0,0 +1,59 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+from pre_commit.util import cmd_output
+
+
+# Limit used previously to avoid "xargs ... Bad file number" on windows
+# This is slightly less than the posix mandated minimum
+MAX_LENGTH = 4000
+
+
+class ArgumentTooLongError(RuntimeError):
+    pass
+
+
+def partition(cmd, varargs, _max_length=MAX_LENGTH):
+    cmd = tuple(cmd)
+    ret = []
+
+    ret_cmd = []
+    total_len = len(' '.join(cmd))
+    # Reversed so arguments are in order
+    varargs = list(reversed(varargs))
+
+    while varargs:
+        arg = varargs.pop()
+
+        if total_len + 1 + len(arg) <= _max_length:
+            ret_cmd.append(arg)
+            total_len += len(arg)
+        elif not ret_cmd:
+            raise ArgumentTooLongError(arg)
+        else:
+            # We've exceeded the length, yield a command
+            ret.append(cmd + tuple(ret_cmd))
+            ret_cmd = []
+            total_len = len(' '.join(cmd))
+            varargs.append(arg)
+
+    ret.append(cmd + tuple(ret_cmd))
+
+    return tuple(ret)
+
+
+def xargs(cmd, varargs):
+    """A simplified implementation of xargs."""
+    retcode = 0
+    stdout = b''
+    stderr = b''
+
+    for run_cmd in partition(cmd, varargs):
+        proc_retcode, proc_out, proc_err = cmd_output(
+            *run_cmd, encoding=None, retcode=None
+        )
+        retcode |= proc_retcode
+        stdout += proc_out
+        stderr += proc_err
+
+    return retcode, stdout, stderr