Merge pull request #1616 from pre-commit/zipapp

add zipapp support

requirements-dev.txt

@@ -1,5 +1,6 @@
 covdefaults
 coverage
+distlib
 pytest
 pytest-env
 re-assert

testing/zipapp/Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:bionic
+RUN : \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        python3 \
+        python3-distutils \
+        python3-venv \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV LANG=C.UTF-8 PATH=/venv/bin:$PATH
+RUN : \
+    && python3.6 -mvenv /venv \
+    && pip install --no-cache-dir pip setuptools wheel no-manylinux --upgrade

testing/zipapp/entry

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import os.path
+import shutil
+import stat
+import sys
+import tempfile
+import zipfile
+
+from pre_commit.file_lock import lock
+
+CACHE_DIR = os.path.expanduser('~/.cache/pre-commit-zipapp')
+
+
+def _make_executable(filename: str) -> None:
+    os.chmod(filename, os.stat(filename).st_mode | stat.S_IXUSR)
+
+
+def _ensure_cache(zipf: zipfile.ZipFile, cache_key: str) -> str:
+    os.makedirs(CACHE_DIR, exist_ok=True)
+
+    cache_dest = os.path.join(CACHE_DIR, cache_key)
+    lock_filename = os.path.join(CACHE_DIR, f'{cache_key}.lock')
+
+    if os.path.exists(cache_dest):
+        return cache_dest
+
+    with lock(lock_filename, blocked_cb=lambda: None):
+        # another process may have completed this work
+        if os.path.exists(cache_dest):
+            return cache_dest
+
+        tmpdir = tempfile.mkdtemp(prefix=os.path.join(CACHE_DIR, ''))
+        try:
+            zipf.extractall(tmpdir)
+            # zip doesn't maintain permissions
+            _make_executable(os.path.join(tmpdir, 'python'))
+            _make_executable(os.path.join(tmpdir, 'python.exe'))
+            os.rename(tmpdir, cache_dest)
+        except BaseException:
+            shutil.rmtree(tmpdir)
+            raise
+
+        return cache_dest
+
+
+def main() -> int:
+    with zipfile.ZipFile(os.path.dirname(__file__)) as zipf:
+        with zipf.open('CACHE_KEY') as f:
+            cache_key = f.read().decode().strip()
+
+        cache_dest = _ensure_cache(zipf, cache_key)
+
+    if sys.platform != 'win32':
+        exe = os.path.join(cache_dest, 'python')
+    else:
+        exe = os.path.join(cache_dest, 'python.exe')
+
+    cmd = (exe, '-mpre_commit', *sys.argv[1:])
+    if sys.platform == 'win32':  # https://bugs.python.org/issue19124
+        import subprocess
+
+        if sys.version_info < (3, 7):  # https://bugs.python.org/issue25942
+            return subprocess.Popen(cmd).wait()
+        else:
+            return subprocess.call(cmd)
+    else:
+        os.execvp(cmd[0], cmd)
+
+
+if __name__ == '__main__':
+    exit(main())

testing/zipapp/make

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+import argparse
+import base64
+import hashlib
+import importlib.resources
+import io
+import os.path
+import shutil
+import subprocess
+import tempfile
+import zipapp
+import zipfile
+
+HERE = os.path.dirname(os.path.realpath(__file__))
+IMG = 'make-pre-commit-zipapp'
+
+
+def _msg(s: str) -> None:
+    print(f'\033[7m{s}\033[m')
+
+
+def _exit_if_retv(*cmd: str) -> None:
+    if subprocess.call(cmd):
+        raise SystemExit(1)
+
+
+def _check_no_shared_objects(wheeldir: str) -> None:
+    for zip_filename in os.listdir(wheeldir):
+        with zipfile.ZipFile(os.path.join(wheeldir, zip_filename)) as zipf:
+            for filename in zipf.namelist():
+                if filename.endswith('.so') or '.so.' in filename:
+                    raise AssertionError(zip_filename, filename)
+
+
+def _add_shim(dest: str) -> None:
+    shim = os.path.join(HERE, 'python')
+    shutil.copy(shim, dest)
+
+    bio = io.BytesIO()
+    with zipfile.ZipFile(bio, 'w') as zipf:
+        zipf.write(shim, arcname='__main__.py')
+
+    with open(os.path.join(dest, 'python.exe'), 'wb') as f:
+        f.write(importlib.resources.read_binary('distlib', 't32.exe'))
+        f.write(b'#!py.exe -3\n')
+        f.write(bio.getvalue())
+
+
+def _write_cache_key(version: str, wheeldir: str, dest: str) -> None:
+    cache_hash = hashlib.sha256(f'{version}\n'.encode())
+    for filename in sorted(os.listdir(wheeldir)):
+        cache_hash.update(f'{filename}\n'.encode())
+    with open(os.path.join(HERE, 'python'), 'rb') as f:
+        cache_hash.update(f.read())
+    with open(os.path.join(dest, 'CACHE_KEY'), 'wb') as f:
+        f.write(base64.urlsafe_b64encode(cache_hash.digest()).rstrip(b'='))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('version')
+    args = parser.parse_args()
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        wheeldir = os.path.join(tmpdir, 'wheels')
+        os.mkdir(wheeldir)
+
+        _msg('building podman image...')
+        _exit_if_retv('podman', 'build', '-q', '-t', IMG, HERE)
+
+        _msg('populating wheels...')
+        _exit_if_retv(
+            'podman', 'run', '--rm', '--volume', f'{wheeldir}:/wheels:rw', IMG,
+            'pip', 'wheel', f'pre_commit=={args.version}',
+            '--wheel-dir', '/wheels',
+        )
+
+        _msg('validating wheels...')
+        _check_no_shared_objects(wheeldir)
+
+        _msg('adding __main__.py...')
+        mainfile = os.path.join(tmpdir, '__main__.py')
+        shutil.copy(os.path.join(HERE, 'entry'), mainfile)
+
+        _msg('adding shim...')
+        _add_shim(tmpdir)
+
+        _msg('copying file_lock.py...')
+        file_lock_py = os.path.join(HERE, '../../pre_commit/file_lock.py')
+        file_lock_py_dest = os.path.join(tmpdir, 'pre_commit/file_lock.py')
+        os.makedirs(os.path.dirname(file_lock_py_dest))
+        shutil.copy(file_lock_py, file_lock_py_dest)
+
+        _msg('writing CACHE_KEY...')
+        _write_cache_key(args.version, wheeldir, tmpdir)
+
+        filename = f'pre-commit-{args.version}.pyz'
+        _msg(f'writing {filename}...')
+        shebang = '/usr/bin/env python3'
+        zipapp.create_archive(tmpdir, filename, interpreter=shebang)
+
+    return 0
+
+
+if __name__ == '__main__':
+    exit(main())

testing/zipapp/python

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""A shim executable to put dependencies on sys.path"""
+import argparse
+import os.path
+import runpy
+import sys
+
+# an exe-zipapp will have a __file__ of shim.exe/__main__.py
+EXE = __file__ if os.path.isfile(__file__) else os.path.dirname(__file__)
+EXE = os.path.realpath(EXE)
+HERE = os.path.dirname(EXE)
+WHEELDIR = os.path.join(HERE, 'wheels')
+SITE_DIRS = frozenset(('dist-packages', 'site-packages'))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument('-m')
+    args, rest = parser.parse_known_args()
+
+    if args.m:
+        # try and remove site-packages from sys.path so our packages win
+        sys.path[:] = [
+            p for p in sys.path
+            if os.path.split(p)[1] not in SITE_DIRS
+        ]
+        for wheel in sorted(os.listdir(WHEELDIR)):
+            sys.path.append(os.path.join(WHEELDIR, wheel))
+        if args.m == 'pre_commit' or args.m.startswith('pre_commit.'):
+            sys.executable = EXE
+        sys.argv[1:] = rest
+        runpy.run_module(args.m, run_name='__main__', alter_sys=True)
+        return 0
+    else:
+        cmd = (sys.executable, *sys.argv[1:])
+        if sys.platform == 'win32':  # https://bugs.python.org/issue19124
+            import subprocess
+
+            if sys.version_info < (3, 7):  # https://bugs.python.org/issue25942
+                return subprocess.Popen(cmd).wait()
+            else:
+                return subprocess.call(cmd)
+        else:
+            os.execvp(cmd[0], cmd)
+
+
+if __name__ == '__main__':
+    exit(main())