Merge pull request #2534 from Holzhaus/rust-as-1st-class-language

Rust as 1st class language

CONTRIBUTING.md

@@ -65,9 +65,9 @@ to implement.  The current implemented languages are at varying levels:
 - 0th class - pre-commit does not require any dependencies for these languages
   as they're not actually languages (current examples: fail, pygrep)
 - 1st class - pre-commit will bootstrap a full interpreter requiring nothing to
-  be installed globally (current examples: node, ruby)
+  be installed globally (current examples: node, ruby, rust)
 - 2nd class - pre-commit requires the user to install the language globally but
-  will install tools in an isolated fashion (current examples: python, go, rust,
+  will install tools in an isolated fashion (current examples: python, go,
   swift, docker).
 - 3rd class - pre-commit requires the user to install both the tool and the
   language globally (current examples: script, system)

azure-pipelines.yml

@@ -17,6 +17,8 @@ jobs:
   parameters:
     toxenvs: [py37]
     os: windows
+    additional_variables:
+      TEMP: C:\Temp
     pre_test:
     - task: UseRubyVersion@0
     - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"

pre_commit/languages/rust.py

@@ -1,13 +1,20 @@
 from __future__ import annotations
 
 import contextlib
+import functools
 import os.path
+import platform
+import shutil
+import sys
+import tempfile
+import urllib.request
 from typing import Generator
 from typing import Sequence
 
 import toml
 
 import pre_commit.constants as C
+from pre_commit import parse_shebang
 from pre_commit.envcontext import envcontext
 from pre_commit.envcontext import PatchesT
 from pre_commit.envcontext import Var
@@ -16,24 +23,61 @@ from pre_commit.languages import helpers
 from pre_commit.prefix import Prefix
 from pre_commit.util import clean_path_on_failure
 from pre_commit.util import cmd_output_b
+from pre_commit.util import make_executable
+from pre_commit.util import win_exe
 
 ENVIRONMENT_DIR = 'rustenv'
-get_default_version = helpers.basic_get_default_version
 health_check = helpers.basic_health_check
 
 
-def get_env_patch(target_dir: str) -> PatchesT:
+@functools.lru_cache(maxsize=1)
+def get_default_version() -> str:
+    # If rust is already installed, we can save a bunch of setup time by
+    # using the installed version.
+    #
+    # Just detecting the executable does not suffice, because if rustup is
+    # installed but no toolchain is available, then `cargo` exists but
+    # cannot be used without installing a toolchain first.
+    if cmd_output_b('cargo', '--version', retcode=None)[0] == 0:
+        return 'system'
+    else:
+        return C.DEFAULT
+
+
+def _rust_toolchain(language_version: str) -> str:
+    """Transform the language version into a rust toolchain version."""
+    if language_version == C.DEFAULT:
+        return 'stable'
+    else:
+        return language_version
+
+
+def _envdir(prefix: Prefix, version: str) -> str:
+    directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
+    return prefix.path(directory)
+
+
+def get_env_patch(target_dir: str, version: str) -> PatchesT:
     return (
+        ('CARGO_HOME', target_dir),
         ('PATH', (os.path.join(target_dir, 'bin'), os.pathsep, Var('PATH'))),
+        # Only set RUSTUP_TOOLCHAIN if we don't want use the system's default
+        # toolchain
+        *(
+            (('RUSTUP_TOOLCHAIN', _rust_toolchain(version)),)
+            if version != 'system' else ()
+        ),
     )
 
 
 @contextlib.contextmanager
-def in_env(prefix: Prefix) -> Generator[None, None, None]:
+def in_env(
-    target_dir = prefix.path(
+    prefix: Prefix,
-        helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
+    language_version: str,
-    )
+) -> Generator[None, None, None]:
-    with envcontext(get_env_patch(target_dir)):
+    with envcontext(
+        get_env_patch(_envdir(prefix, language_version), language_version),
+    ):
         yield
 
 
@@ -52,15 +96,45 @@ def _add_dependencies(
         f.truncate()
 
 
+def install_rust_with_toolchain(toolchain: str) -> None:
+    with tempfile.TemporaryDirectory() as rustup_dir:
+        with envcontext((('RUSTUP_HOME', rustup_dir),)):
+            # acquire `rustup` if not present
+            if parse_shebang.find_executable('rustup') is None:
+                # We did not detect rustup and need to download it first.
+                if sys.platform == 'win32':  # pragma: win32 cover
+                    if platform.machine() == 'x86_64':
+                        url = 'https://win.rustup.rs/x86_64'
+                    else:
+                        url = 'https://win.rustup.rs/i686'
+                else:  # pragma: win32 no cover
+                    url = 'https://sh.rustup.rs'
+
+                resp = urllib.request.urlopen(url)
+
+                rustup_init = os.path.join(rustup_dir, win_exe('rustup-init'))
+                with open(rustup_init, 'wb') as f:
+                    shutil.copyfileobj(resp, f)
+                make_executable(rustup_init)
+
+                # install rustup into `$CARGO_HOME/bin`
+                cmd_output_b(
+                    rustup_init, '-y', '--quiet', '--no-modify-path',
+                    '--default-toolchain', 'none',
+                )
+
+            cmd_output_b(
+                'rustup', 'toolchain', 'install', '--no-self-update',
+                toolchain,
+            )
+
+
 def install_environment(
         prefix: Prefix,
         version: str,
         additional_dependencies: Sequence[str],
 ) -> None:
-    helpers.assert_version_default('rust', version)
+    directory = _envdir(prefix, version)
-    directory = prefix.path(
-        helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
-    )
 
     # There are two cases where we might want to specify more dependencies:
     # as dependencies for the library being built, and as binary packages
@@ -84,17 +158,21 @@ def install_environment(
         packages_to_install: set[tuple[str, ...]] = {('--path', '.')}
         for cli_dep in cli_deps:
             cli_dep = cli_dep[len('cli:'):]
-            package, _, version = cli_dep.partition(':')
+            package, _, crate_version = cli_dep.partition(':')
-            if version != '':
+            if crate_version != '':
-                packages_to_install.add((package, '--version', version))
+                packages_to_install.add((package, '--version', crate_version))
             else:
                 packages_to_install.add((package,))
 
-        for args in packages_to_install:
+        with in_env(prefix, version):
-            cmd_output_b(
+            if version != 'system':
-                'cargo', 'install', '--bins', '--root', directory, *args,
+                install_rust_with_toolchain(_rust_toolchain(version))
-                cwd=prefix.prefix_dir,
+
-            )
+            for args in packages_to_install:
+                cmd_output_b(
+                    'cargo', 'install', '--bins', '--root', directory, *args,
+                    cwd=prefix.prefix_dir,
+                )
 
 
 def run_hook(
@@ -102,5 +180,5 @@ def run_hook(
         file_args: Sequence[str],
         color: bool,
 ) -> tuple[int, bytes]:
-    with in_env(hook.prefix):
+    with in_env(hook.prefix, hook.language_version):
         return helpers.run_xargs(hook, hook.cmd, file_args, color=color)

tests/languages/rust_test.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from unittest import mock
+
+import pytest
+
+import pre_commit.constants as C
+from pre_commit import parse_shebang
+from pre_commit.languages import rust
+from pre_commit.prefix import Prefix
+from pre_commit.util import cmd_output
+
+ACTUAL_GET_DEFAULT_VERSION = rust.get_default_version.__wrapped__
+
+
+@pytest.fixture
+def cmd_output_b_mck():
+    with mock.patch.object(rust, 'cmd_output_b') as mck:
+        yield mck
+
+
+def test_sets_system_when_rust_is_available(cmd_output_b_mck):
+    cmd_output_b_mck.return_value = (0, b'', b'')
+    assert ACTUAL_GET_DEFAULT_VERSION() == 'system'
+
+
+def test_uses_default_when_rust_is_not_available(cmd_output_b_mck):
+    cmd_output_b_mck.return_value = (127, b'', b'error: not found')
+    assert ACTUAL_GET_DEFAULT_VERSION() == C.DEFAULT
+
+
+@pytest.mark.parametrize('language_version', (C.DEFAULT, '1.56.0'))
+def test_installs_with_bootstrapped_rustup(tmpdir, language_version):
+    tmpdir.join('src', 'main.rs').ensure().write(
+        'fn main() {\n'
+        '    println!("Hello, world!");\n'
+        '}\n',
+    )
+    tmpdir.join('Cargo.toml').ensure().write(
+        '[package]\n'
+        'name = "hello_world"\n'
+        'version = "0.1.0"\n'
+        'edition = "2021"\n',
+    )
+    prefix = Prefix(str(tmpdir))
+
+    find_executable_exes = []
+
+    original_find_executable = parse_shebang.find_executable
+
+    def mocked_find_executable(exe: str) -> str | None:
+        """
+        Return `None` the first time `find_executable` is called to ensure
+        that the bootstrapping code is executed, then just let the function
+        work as normal.
+
+        Also log the arguments to ensure that everything works as expected.
+        """
+        find_executable_exes.append(exe)
+        if len(find_executable_exes) == 1:
+            return None
+        return original_find_executable(exe)
+
+    with mock.patch.object(parse_shebang, 'find_executable') as find_exe_mck:
+        find_exe_mck.side_effect = mocked_find_executable
+        rust.install_environment(prefix, language_version, ())
+        assert find_executable_exes == ['rustup', 'rustup', 'cargo']
+
+    with rust.in_env(prefix, language_version):
+        assert cmd_output('hello_world')[1] == 'Hello, world!\n'

tests/repository_test.py

@@ -471,7 +471,7 @@ def test_additional_rust_cli_dependencies_installed(
     hook = _get_hook(config, store, 'rust-hook')
     binaries = os.listdir(
         hook.prefix.path(
-            helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin',
+            helpers.environment_dir(rust.ENVIRONMENT_DIR, 'system'), 'bin',
         ),
     )
     # normalize for windows
@@ -490,7 +490,7 @@ def test_additional_rust_lib_dependencies_installed(
     hook = _get_hook(config, store, 'rust-hook')
     binaries = os.listdir(
         hook.prefix.path(
-            helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin',
+            helpers.environment_dir(rust.ENVIRONMENT_DIR, 'system'), 'bin',
         ),
     )
     # normalize for windows