pre-commit gc

pre_commit/commands/gc.py

@@ -0,0 +1,83 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os.path
+
+import pre_commit.constants as C
+from pre_commit import output
+from pre_commit.clientlib import InvalidConfigError
+from pre_commit.clientlib import InvalidManifestError
+from pre_commit.clientlib import is_local_repo
+from pre_commit.clientlib import is_meta_repo
+from pre_commit.clientlib import load_config
+from pre_commit.clientlib import load_manifest
+
+
+def _mark_used_repos(store, all_repos, unused_repos, repo):
+    if is_meta_repo(repo):
+        return
+    elif is_local_repo(repo):
+        for hook in repo['hooks']:
+            deps = hook.get('additional_dependencies')
+            unused_repos.discard((
+                store.db_repo_name(repo['repo'], deps), C.LOCAL_REPO_VERSION,
+            ))
+    else:
+        key = (repo['repo'], repo['rev'])
+        path = all_repos.get(key)
+        # can't inspect manifest if it isn't cloned
+        if path is None:
+            return
+
+        try:
+            manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE))
+        except InvalidManifestError:
+            return
+        else:
+            unused_repos.discard(key)
+            by_id = {hook['id']: hook for hook in manifest}
+
+        for hook in repo['hooks']:
+            if hook['id'] not in by_id:
+                continue
+
+            deps = hook.get(
+                'additional_dependencies',
+                by_id[hook['id']]['additional_dependencies'],
+            )
+            unused_repos.discard((
+                store.db_repo_name(repo['repo'], deps), repo['rev'],
+            ))
+
+
+def _gc_repos(store):
+    configs = store.select_all_configs()
+    repos = store.select_all_repos()
+
+    # delete config paths which do not exist
+    dead_configs = [p for p in configs if not os.path.exists(p)]
+    live_configs = [p for p in configs if os.path.exists(p)]
+
+    all_repos = {(repo, ref): path for repo, ref, path in repos}
+    unused_repos = set(all_repos)
+    for config_path in live_configs:
+        try:
+            config = load_config(config_path)
+        except InvalidConfigError:
+            dead_configs.append(config_path)
+            continue
+        else:
+            for repo in config['repos']:
+                _mark_used_repos(store, all_repos, unused_repos, repo)
+
+    store.delete_configs(dead_configs)
+    for db_repo_name, ref in unused_repos:
+        store.delete_repo(db_repo_name, ref, all_repos[(db_repo_name, ref)])
+    return len(unused_repos)
+
+
+def gc(store):
+    with store.exclusive_lock():
+        repos_removed = _gc_repos(store)
+    output.write_line('{} repo(s) removed.'.format(repos_removed))
+    return 0

pre_commit/error_handler.py

@@ -32,7 +32,6 @@ def _log_and_exit(msg, exc, formatted):
     ))
     output.write(error_msg)
     store = Store()
-    store.require_created()
     log_path = os.path.join(store.directory, 'pre-commit.log')
     output.write_line('Check the log at {}'.format(log_path))
     with open(log_path, 'wb') as log:

pre_commit/main.py

@@ -11,6 +11,7 @@ from pre_commit import five
 from pre_commit import git
 from pre_commit.commands.autoupdate import autoupdate
 from pre_commit.commands.clean import clean
+from pre_commit.commands.gc import gc
 from pre_commit.commands.install_uninstall import install
 from pre_commit.commands.install_uninstall import install_hooks
 from pre_commit.commands.install_uninstall import uninstall
@@ -176,6 +177,11 @@ def main(argv=None):
     )
     _add_color_option(clean_parser)
     _add_config_option(clean_parser)
+
+    gc_parser = subparsers.add_parser('gc', help='Clean unused cached repos.')
+    _add_color_option(gc_parser)
+    _add_config_option(gc_parser)
+
     autoupdate_parser = subparsers.add_parser(
         'autoupdate',
         help="Auto-update pre-commit config to the latest repos' versions.",
@@ -251,9 +257,11 @@ def main(argv=None):
     with error_handler(), logging_handler(args.color):
         _adjust_args_and_chdir(args)
 
-        store = Store()
         git.check_for_cygwin_mismatch()
 
+        store = Store()
+        store.mark_config_used(args.config)
+
         if args.command == 'install':
             return install(
                 args.config, store,
@@ -267,6 +275,8 @@ def main(argv=None):
             return uninstall(hook_type=args.hook_type)
         elif args.command == 'clean':
             return clean(store)
+        elif args.command == 'gc':
+            return gc(store)
         elif args.command == 'autoupdate':
             if args.tags_only:
                 logger.warning('--tags-only is the default')

pre_commit/store.py

@@ -13,6 +13,7 @@ from pre_commit import git
 from pre_commit.util import clean_path_on_failure
 from pre_commit.util import cmd_output
 from pre_commit.util import resource_text
+from pre_commit.util import rmtree
 
 
 logger = logging.getLogger('pre_commit')
@@ -33,10 +34,43 @@ def _get_default_directory():
 
 class Store(object):
     get_default_directory = staticmethod(_get_default_directory)
-    __created = False
 
     def __init__(self, directory=None):
         self.directory = directory or Store.get_default_directory()
+        self.db_path = os.path.join(self.directory, 'db.db')
+
+        if not os.path.exists(self.directory):
+            os.makedirs(self.directory)
+            with io.open(os.path.join(self.directory, 'README'), 'w') as f:
+                f.write(
+                    'This directory is maintained by the pre-commit project.\n'
+                    'Learn more: https://github.com/pre-commit/pre-commit\n',
+                )
+
+        if os.path.exists(self.db_path):
+            return
+        with self.exclusive_lock():
+            # Another process may have already completed this work
+            if os.path.exists(self.db_path):  # pragma: no cover (race)
+                return
+            # To avoid a race where someone ^Cs between db creation and
+            # execution of the CREATE TABLE statement
+            fd, tmpfile = tempfile.mkstemp(dir=self.directory)
+            # We'll be managing this file ourselves
+            os.close(fd)
+            with self.connect(db_path=tmpfile) as db:
+                db.executescript(
+                    'CREATE TABLE repos ('
+                    '    repo TEXT NOT NULL,'
+                    '    ref TEXT NOT NULL,'
+                    '    path TEXT NOT NULL,'
+                    '    PRIMARY KEY (repo, ref)'
+                    ');',
+                )
+                self._create_config_table_if_not_exists(db)
+
+            # Atomic file move
+            os.rename(tmpfile, self.db_path)
 
     @contextlib.contextmanager
     def exclusive_lock(self):
@@ -46,62 +80,30 @@ class Store(object):
         with file_lock.lock(os.path.join(self.directory, '.lock'), blocked_cb):
             yield
 
-    def _write_readme(self):
-        with io.open(os.path.join(self.directory, 'README'), 'w') as readme:
-            readme.write(
-                'This directory is maintained by the pre-commit project.\n'
-                'Learn more: https://github.com/pre-commit/pre-commit\n',
-            )
-
-    def _write_sqlite_db(self):
-        # To avoid a race where someone ^Cs between db creation and execution
-        # of the CREATE TABLE statement
-        fd, tmpfile = tempfile.mkstemp(dir=self.directory)
-        # We'll be managing this file ourselves
-        os.close(fd)
+    @contextlib.contextmanager
+    def connect(self, db_path=None):
+        db_path = db_path or self.db_path
         # sqlite doesn't close its fd with its contextmanager >.<
         # contextlib.closing fixes this.
         # See: https://stackoverflow.com/a/28032829/812183
-        with contextlib.closing(sqlite3.connect(tmpfile)) as db:
-            db.executescript(
-                'CREATE TABLE repos ('
-                '    repo TEXT NOT NULL,'
-                '    ref TEXT NOT NULL,'
-                '    path TEXT NOT NULL,'
-                '    PRIMARY KEY (repo, ref)'
-                ');',
-            )
+        with contextlib.closing(sqlite3.connect(db_path)) as db:
+            # this creates a transaction
+            with db:
+                yield db
 
-        # Atomic file move
-        os.rename(tmpfile, self.db_path)
-
-    def _create(self):
-        if not os.path.exists(self.directory):
-            os.makedirs(self.directory)
-            self._write_readme()
-
-        if os.path.exists(self.db_path):
-            return
-        with self.exclusive_lock():
-            # Another process may have already completed this work
-            if os.path.exists(self.db_path):  # pragma: no cover (race)
-                return
-            self._write_sqlite_db()
-
-    def require_created(self):
-        """Require the pre-commit file store to be created."""
-        if not self.__created:
-            self._create()
-            self.__created = True
+    @classmethod
+    def db_repo_name(cls, repo, deps):
+        if deps:
+            return '{}:{}'.format(repo, ','.join(sorted(deps)))
+        else:
+            return repo
 
     def _new_repo(self, repo, ref, deps, make_strategy):
-        self.require_created()
-        if deps:
-            repo = '{}:{}'.format(repo, ','.join(sorted(deps)))
+        repo = self.db_repo_name(repo, deps)
 
         def _get_result():
             # Check if we already exist
-            with sqlite3.connect(self.db_path) as db:
+            with self.connect() as db:
                 result = db.execute(
                     'SELECT path FROM repos WHERE repo = ? AND ref = ?',
                     (repo, ref),
@@ -125,7 +127,7 @@ class Store(object):
                 make_strategy(directory)
 
             # Update our db with the created repo
-            with sqlite3.connect(self.db_path) as db:
+            with self.connect() as db:
                 db.execute(
                     'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
                     [repo, ref, directory],
@@ -175,6 +177,43 @@ class Store(object):
             'local', C.LOCAL_REPO_VERSION, deps, make_local_strategy,
         )
 
-    @property
-    def db_path(self):
-        return os.path.join(self.directory, 'db.db')
+    def _create_config_table_if_not_exists(self, db):
+        db.executescript(
+            'CREATE TABLE IF NOT EXISTS configs ('
+            '   path TEXT NOT NULL,'
+            '   PRIMARY KEY (path)'
+            ');',
+        )
+
+    def mark_config_used(self, path):
+        path = os.path.realpath(path)
+        # don't insert config files that do not exist
+        if not os.path.exists(path):
+            return
+        with self.connect() as db:
+            # TODO: eventually remove this and only create in _create
+            self._create_config_table_if_not_exists(db)
+            db.execute('INSERT OR IGNORE INTO configs VALUES (?)', (path,))
+
+    def select_all_configs(self):
+        with self.connect() as db:
+            self._create_config_table_if_not_exists(db)
+            rows = db.execute('SELECT path FROM configs').fetchall()
+            return [path for path, in rows]
+
+    def delete_configs(self, configs):
+        with self.connect() as db:
+            rows = [(path,) for path in configs]
+            db.executemany('DELETE FROM configs WHERE path = ?', rows)
+
+    def select_all_repos(self):
+        with self.connect() as db:
+            return db.execute('SELECT repo, ref, path from repos').fetchall()
+
+    def delete_repo(self, db_repo_name, ref, path):
+        with self.connect() as db:
+            db.execute(
+                'DELETE FROM repos WHERE repo = ? and ref = ?',
+                (db_repo_name, ref),
+            )
+        rmtree(path)