Merge pull request #2768 from pre-commit/per-language-tests

test languages only when they are changed

.github/actions/pre-test/action.yml

@@ -5,36 +5,5 @@ inputs:
 runs:
   using: composite
   steps:
-  - name: setup (windows)
-    shell: bash
-    if: runner.os == 'Windows'
-    run: |
-      set -x
-
-      echo 'TEMP=C:\TEMP' >> "$GITHUB_ENV"
-
-      echo "$CONDA\Scripts" >> "$GITHUB_PATH"
-
-      echo 'C:\Strawberry\perl\bin' >> "$GITHUB_PATH"
-      echo 'C:\Strawberry\perl\site\bin' >> "$GITHUB_PATH"
-      echo 'C:\Strawberry\c\bin' >> "$GITHUB_PATH"
-
-      testing/get-coursier.sh
-      testing/get-dart.sh
-  - name: setup (linux)
-    shell: bash
-    if: runner.os == 'Linux'
-    run: |
-      set -x
-
-      sudo apt-get update
-      sudo apt-get install -y --no-install-recommends \
-        lua5.3 \
-        liblua5.3-dev \
-        luarocks
-
-      testing/get-coursier.sh
-      testing/get-dart.sh
-      testing/get-swift.sh
   - uses: asottile/workflows/.github/actions/latest-git@v1.4.0
     if: inputs.env == 'py38' && runner.os == 'Linux'

.github/workflows/languages.yaml

@@ -0,0 +1,82 @@
+name: languages
+
+on:
+  push:
+    branches: [main, test-me-*]
+    tags:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  vars:
+    runs-on: ubuntu-latest
+    outputs:
+      languages: ${{ steps.vars.outputs.languages }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+    - name: install deps
+      run: python -mpip install -e . -r requirements-dev.txt
+    - name: vars
+      run: testing/languages ${{ github.event_name == 'push' && '--all' || '' }}
+      id: vars
+  language:
+    needs: [vars]
+    runs-on: ${{ matrix.os }}
+    if: needs.vars.outputs.languages != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.vars.outputs.languages) }}
+    steps:
+    - uses: asottile/workflows/.github/actions/fast-checkout@v1.4.0
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+
+    - run: echo "$CONDA\Scripts" >> "$GITHUB_PATH"
+      shell: bash
+      if: matrix.os == 'windows-latest' && matrix.language == 'conda'
+    - run: testing/get-coursier.sh
+      shell: bash
+      if: matrix.language == 'coursier'
+    - run: testing/get-dart.sh
+      shell: bash
+      if: matrix.language == 'dart'
+    - run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends \
+          lua5.3 \
+          liblua5.3-dev \
+          luarocks
+      if: matrix.os == 'ubuntu-latest' && matrix.language == 'lua'
+    - run: |
+        echo 'C:\Strawberry\perl\bin' >> "$GITHUB_PATH"
+        echo 'C:\Strawberry\perl\site\bin' >> "$GITHUB_PATH"
+        echo 'C:\Strawberry\c\bin' >> "$GITHUB_PATH"
+      shell: bash
+      if: matrix.os == 'windows-latest' && matrix.language == 'perl'
+    - run: testing/get-swift.sh
+      if: matrix.os == 'ubuntu-latest' && matrix.language == 'swift'
+
+    - name: install deps
+      run: python -mpip install -e . -r requirements-dev.txt
+    - name: run tests
+      run: coverage run -m pytest tests/languages/${{ matrix.language }}_test.py
+    - name: check coverage
+      run: coverage report --include pre_commit/languages/${{ matrix.language }}.py,tests/languages/${{ matrix.language }}_test.py
+  collector:
+    needs: [language]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+    - name: check for failures
+      if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+      run: echo job failed && exit 1

pre_commit/lang_base.py

@@ -43,12 +43,7 @@ class Language(Protocol):
         ...
 
     # modify the environment for hook execution
-    def in_env(
+    def in_env(self, prefix: Prefix, version: str) -> ContextManager[None]: ...
-            self,
-            prefix: Prefix,
-            version: str,
-    ) -> ContextManager[None]:
-        ...
 
     # execute a hook and return the exit code and output
     def run_hook(

testing/languages

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import json
+import os.path
+import subprocess
+import sys
+
+EXCLUDED = frozenset((
+    ('windows-latest', 'docker'),
+    ('windows-latest', 'docker_image'),
+    ('windows-latest', 'lua'),
+    ('windows-latest', 'swift'),
+))
+
+
+def _lang_files(lang: str) -> frozenset[str]:
+    prog = f'''\
+import json
+import os.path
+import sys
+
+import pre_commit.languages.{lang}
+import tests.languages.{lang}_test
+
+modules = sorted(
+    os.path.relpath(v.__file__)
+    for k, v in sys.modules.items()
+    if k.startswith(('pre_commit.', 'tests.', 'testing.'))
+)
+print(json.dumps(modules))
+'''
+    out = json.loads(subprocess.check_output((sys.executable, '-c', prog)))
+    return frozenset(out)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--all', action='store_true')
+    args = parser.parse_args()
+
+    langs = [
+        os.path.splitext(fname)[0]
+        for fname in sorted(os.listdir('pre_commit/languages'))
+        if fname.endswith('.py') and fname != '__init__.py'
+    ]
+
+    if not args.all:
+        with concurrent.futures.ThreadPoolExecutor(os.cpu_count()) as exe:
+            by_lang = {
+                lang: files
+                for lang, files in zip(langs, exe.map(_lang_files, langs))
+            }
+
+        diff_cmd = ('git', 'diff', '--name-only', 'origin/main...HEAD')
+        files = set(subprocess.check_output(diff_cmd).decode().splitlines())
+
+        langs = [
+            lang
+            for lang, lang_files in by_lang.items()
+            if lang_files & files
+        ]
+
+    matched = [
+        {'os': os, 'language': lang}
+        for os in ('windows-latest', 'ubuntu-latest')
+        for lang in langs
+        if (os, lang) not in EXCLUDED
+    ]
+
+    with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+        f.write(f'languages={json.dumps(matched)}\n')
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

tox.ini

@@ -6,8 +6,8 @@ deps = -rrequirements-dev.txt
 passenv = *
 commands =
     coverage erase
-    coverage run -m pytest {posargs:tests}
+    coverage run -m pytest {posargs:tests} --ignore=tests/languages
-    coverage report
+    coverage report --omit=pre_commit/languages/*,tests/languages/*
 
 [testenv:pre-commit]
 skip_install = true