From 5cfdaa3aee90206f7f9fb713a01348b2bffe4c79 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 10 Feb 2022 17:30:22 +0000 Subject: [PATCH] Use /proc/1/sched --- pre_commit/languages/docker.py | 109 ++++++++++++++++++++++- tests/languages/docker_test.py | 155 ++++++++++++++++++++++++++------- 2 files changed, 229 insertions(+), 35 deletions(-) diff --git a/pre_commit/languages/docker.py b/pre_commit/languages/docker.py index fd7a00cb..893dd095 100644 --- a/pre_commit/languages/docker.py +++ b/pre_commit/languages/docker.py @@ -19,7 +19,7 @@ get_default_version = helpers.basic_get_default_version healthy = helpers.basic_healthy -def _is_in_docker() -> bool: +def _is_in_docker_cgroup() -> bool: try: with open('/proc/1/cgroup', 'rb') as f: for line in f.readlines(): @@ -37,7 +37,27 @@ def _is_in_docker() -> bool: return False -def _get_container_id() -> str: +def _is_in_docker_sched() -> bool: + try: + with open('/proc/1/sched', 'rb') as f: + line = f.readline() + + if line.startswith(b'systemd ') or line.startswith(b'init '): + return False + + return True + except FileNotFoundError: + return False + + +def _is_in_docker() -> bool: + if _is_in_docker_cgroup() or _is_in_docker_sched(): + return True + + return False + + +def _get_container_id_cgroup() -> str: # It's assumed that we already check /proc/1/cgroup in _is_in_docker. The # cpuset cgroup controller existed since cgroups were introduced so this # way of getting the container ID is pretty reliable. @@ -45,7 +65,90 @@ def _get_container_id() -> str: for line in f.readlines(): if line.split(b':')[1] == b'cpuset': return os.path.basename(line.split(b':')[2]).strip().decode() - raise RuntimeError('Failed to find the container ID in /proc/1/cgroup.') + + return '' + + +def _get_container_id_sched() -> str: + # The idea here is to try to match the the workdir option found in the + # overlay mount with the GraphDriver.Data.WorkDir from the docker describe. + + # Get details for the overlay mount type + try: + _, out, _ = cmd_output_b('mount', '-t', 'overlay') + except CalledProcessError: + # No mount command available or the -t option is not supported + return '' + + lines = out.decode().strip().split('\n') + + # There is always only one overlay mount inside the container + if len(lines) > 1 or lines[0] == '' or '(' not in lines[0]: + return '' + + _, all_opts = lines[0].strip(')').split('(') + opts = all_opts.split(',') + + # Search for workdir option + for opt in opts: + if '=' in opt: + k, v = opt.split('=') + + if k == 'workdir': + # We have found workdir + workdir = v + + break + else: + # No workdir was found + return '' + + # Get list IDs for all running containers + try: + _, out, _ = cmd_output_b('docker', 'ps', '--format', '{{ .ID }}') + except CalledProcessError: + # There is probably no docker command + return '' + + container_ids = out.decode().strip().split('\n') + + # Check if there are any container IDs + if len(container_ids) == 1 and container_ids[0] == '': + return '' + + # Search for a container that has the workdir we got from the mount command + for container_id in container_ids: + try: + _, out, _ = cmd_output_b('docker', 'inspect', container_id) + except CalledProcessError: + # Container probably doesn't exist anymore + return '' + + container, = json.loads(out) + + if ( + 'GraphDriver' in container and + 'Data' in container['GraphDriver'] and + 'WorkDir' in container['GraphDriver']['Data'] and + container['GraphDriver']['Data']['WorkDir'] == workdir + ): + # We have found matching container! + return container_id + else: + # No matching container found + return '' + + +def _get_container_id() -> str: + container_id = _get_container_id_cgroup() + + if container_id == '': + container_id = _get_container_id_sched() + + if container_id == '': + raise RuntimeError('Failed to find the container ID.') + + return container_id def _get_docker_path(path: str) -> str: diff --git a/tests/languages/docker_test.py b/tests/languages/docker_test.py index af692126..da599605 100644 --- a/tests/languages/docker_test.py +++ b/tests/languages/docker_test.py @@ -12,7 +12,7 @@ import pytest from pre_commit.languages import docker from pre_commit.util import CalledProcessError -DOCKER_CGROUP_EXAMPLE1 = b'''\ +DOCKER_CGROUP_EXAMPLE = b'''\ 12:hugetlb:/docker/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 11:blkio:/docker/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 10:freezer:/docker/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 @@ -28,23 +28,7 @@ DOCKER_CGROUP_EXAMPLE1 = b'''\ 0::/system.slice/containerd.service ''' # noqa: E501 -DOCKER_CGROUP_EXAMPLE2 = b'''\ -12:blkio:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -11:memory:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -10:hugetlb:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -9:devices:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -8:freezer:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -7:cpu,cpuacct:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -6:cpuset:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -5:pids:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -4:net_cls,net_prio:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -3:perf_event:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -2:rdma:/ -1:name=systemd:/actions_job/c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7 -0::/system.slice/containerd.service -''' # noqa: E501 - -# The ID should match the above cgroup examples. +# The ID should match the above cgroup example. CONTAINER_ID = 'c33988ec7651ebc867cb24755eaf637a6734088bc7eef59d5799293a9e5450f7' # noqa: E501 NON_DOCKER_CGROUP_EXAMPLE = b'''\ @@ -63,6 +47,108 @@ NON_DOCKER_CGROUP_EXAMPLE = b'''\ 0::/init.scope ''' +DOCKER_SCHED_EXAMPLE = b'''\ +sh (1, #threads: 1) +------------------------------------------------------------------- +se.exec_start : 349708379.385019 +se.vruntime : 32.684454 +se.sum_exec_runtime : 33.860082 +se.nr_migrations : 18 +nr_switches : 77 +nr_voluntary_switches : 64 +nr_involuntary_switches : 13 +se.load.weight : 1048576 +se.avg.load_sum : 198 +se.avg.runnable_sum : 203813 +se.avg.util_sum : 200802 +se.avg.load_avg : 4 +se.avg.runnable_avg : 4 +se.avg.util_avg : 4 +se.avg.last_update_time : 349700175852544 +se.avg.util_est.ewma : 8 +se.avg.util_est.enqueued : 0 +uclamp.min : 0 +uclamp.max : 1024 +effective uclamp.min : 0 +effective uclamp.max : 1024 +policy : 0 +prio : 120 +clock-delta : 27 +mm->numa_scan_seq : 0 +numa_pages_migrated : 0 +numa_preferred_nid : -1 +total_numa_faults : 0 +current_node=0, numa_group_id=0 +numa_faults node=0 task_private=0 task_shared=0 group_private=0 group_shared=0 +''' # noqa: E501 + +SYSTEMD_SCHED_EXAMPLE = b'''\ +systemd (1, #threads: 1) +------------------------------------------------------------------- +se.exec_start : 349429286.842224 +se.vruntime : 2759.778972 +se.sum_exec_runtime : 9858.995771 +se.nr_migrations : 11801 +nr_switches : 80235 +nr_voluntary_switches : 78822 +nr_involuntary_switches : 1413 +se.load.weight : 1048576 +se.avg.load_sum : 4221 +se.avg.runnable_sum : 4325517 +se.avg.util_sum : 3881112 +se.avg.load_avg : 91 +se.avg.runnable_avg : 91 +se.avg.util_avg : 81 +se.avg.last_update_time : 349418325520384 +se.avg.util_est.ewma : 81 +se.avg.util_est.enqueued : 81 +uclamp.min : 0 +uclamp.max : 1024 +effective uclamp.min : 0 +effective uclamp.max : 1024 +policy : 0 +prio : 120 +clock-delta : 78 +mm->numa_scan_seq : 0 +numa_pages_migrated : 0 +numa_preferred_nid : -1 +total_numa_faults : 0 +current_node=0, numa_group_id=0 +numa_faults node=0 task_private=0 task_shared=0 group_private=0 group_shared=0 +''' # noqa: E501 + +INIT_SCHED_EXAMPLE = b'''\ +init (1, #threads: 1) +------------------------------------------------------------------- +se.exec_start : 45362.204529 +se.vruntime : 14424.583092 +se.sum_exec_runtime : 636.475090 +se.nr_migrations : 0 +nr_switches : 409 +nr_voluntary_switches : 206 +nr_involuntary_switches : 203 +se.load.weight : 1048576 +se.runnable_weight : 1048576 +se.avg.load_sum : 48 +se.avg.runnable_load_sum : 48 +se.avg.util_sum : 37888 +se.avg.load_avg : 0 +se.avg.runnable_load_avg : 0 +se.avg.util_avg : 0 +se.avg.last_update_time : 45362203648 +se.avg.util_est.ewma : 9 +se.avg.util_est.enqueued : 0 +policy : 0 +prio : 120 +clock-delta : 396 +mm->numa_scan_seq : 0 +numa_pages_migrated : 0 +numa_preferred_nid : -1 +total_numa_faults : 0 +current_node=0, numa_group_id=0 +numa_faults node=0 task_private=0 task_shared=0 group_private=0 group_shared=0 +''' # noqa: E501 + def test_docker_fallback_user(): def invalid_attribute(): @@ -90,28 +176,33 @@ def _mock_open(data): ) -def test_in_docker_docker_in_file1(): - with _mock_open(DOCKER_CGROUP_EXAMPLE1): - assert docker._is_in_docker() is True - - -def test_in_docker_docker_in_file2(): - with _mock_open(DOCKER_CGROUP_EXAMPLE2): - assert docker._is_in_docker() is True +def test_in_docker_docker_in_file(): + with _mock_open(DOCKER_CGROUP_EXAMPLE): + assert docker._is_in_docker_cgroup() is True def test_in_docker_docker_not_in_file(): with _mock_open(NON_DOCKER_CGROUP_EXAMPLE): - assert docker._is_in_docker() is False + assert docker._is_in_docker_cgroup() is False -def test_get_container_id1(): - with _mock_open(DOCKER_CGROUP_EXAMPLE1): - assert docker._get_container_id() == CONTAINER_ID +def test_in_docker_inside_container(): + with _mock_open(DOCKER_SCHED_EXAMPLE): + assert docker._is_in_docker_sched() is True -def test_get_container_id2(): - with _mock_open(DOCKER_CGROUP_EXAMPLE2): +def test_in_docker_outside_container_systemd(): + with _mock_open(SYSTEMD_SCHED_EXAMPLE): + assert docker._is_in_docker_sched() is False + + +def test_in_docker_outside_container_init(): + with _mock_open(INIT_SCHED_EXAMPLE): + assert docker._is_in_docker_sched() is False + + +def test_get_container_id(): + with _mock_open(DOCKER_CGROUP_EXAMPLE): assert docker._get_container_id() == CONTAINER_ID