From e016d63e8c34f99654536ac5652cfe8f96f95470 Mon Sep 17 00:00:00 2001 From: Omair Majid Date: Tue, 7 Apr 2020 20:00:22 -0400 Subject: [PATCH] Sync cgroup v2 in libraries with coreclr This commit brings in two changes from coreclr to libraries: 1. https://github.com/dotnet/runtime/pull/980 "Fix named cgroup handling in docker" This fixes getting cgroup information for named cgroups inside containers. 2. https://github.com/dotnet/runtime/pull/34334 "Add cgroup v2 support to coreclr" This is essentially the same change pushed to corefx (now libraries) to add cgroupv2 support, but this newer coreclr change has one major difference: it determines whether the system is using cgroup v1 or cgroup v2 once, and then explicitly uses that (only). This avoids issues on systems where both cgroup v1 and v2 are enabled, (but only one is being used by default). --- .../Interop/Linux/cgroups/Interop.cgroups.cs | 189 ++++++++++++------ .../tests/Tests/Interop/cgroupsTests.cs | 67 ++++--- .../src/System.Diagnostics.Process.csproj | 3 +- .../tests/DescriptionNameTests.cs | 1 + 4 files changed, 176 insertions(+), 84 deletions(-) diff --git a/src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs b/src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs index 287fc21ebedf0b..abb7baa63a673d 100644 --- a/src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs +++ b/src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs @@ -16,17 +16,23 @@ internal static partial class cgroups { // For cgroup v1, see https://www.kernel.org/doc/Documentation/cgroup-v1/ // For cgroup v2, see https://www.kernel.org/doc/Documentation/cgroup-v2.txt + // For disambiguation, see https://systemd.io/CGROUP_DELEGATION/#three-different-tree-setups- - /// The version of cgroup that's being used + /// The supported versions of cgroup. internal enum CGroupVersion { None, CGroup1, CGroup2 }; + /// Path to cgroup filesystem that tells us which version of cgroup is in use. + private const string SysFsCgroupFileSystemPath = "/sys/fs/cgroup"; /// Path to mountinfo file in procfs for the current process. private const string ProcMountInfoFilePath = "/proc/self/mountinfo"; /// Path to cgroup directory in procfs for the current process. private const string ProcCGroupFilePath = "/proc/self/cgroup"; + /// The version of cgroup that's being used. Mutated by tests only. + internal static readonly CGroupVersion s_cgroupVersion = FindCGroupVersion(); + /// Path to the found cgroup memory limit path, or null if it couldn't be found. - internal static readonly string? s_cgroupMemoryLimitPath = FindCGroupMemoryLimitPath(); + internal static readonly string? s_cgroupMemoryLimitPath = FindCGroupMemoryLimitPath(s_cgroupVersion); /// Tries to read the memory limit from the cgroup memory location. /// The read limit, or 0 if it couldn't be read. @@ -102,19 +108,39 @@ internal static bool TryReadMemoryValueFromFile(string path, out ulong result) return false; } + /// Find the cgroup version in use on the system. + /// The cgroup version. + private static CGroupVersion FindCGroupVersion() + { + try + { + return new DriveInfo(SysFsCgroupFileSystemPath).DriveFormat switch + { + "cgroup2fs" => CGroupVersion.CGroup2, + "tmpfs" => CGroupVersion.CGroup1, + _ => CGroupVersion.None, + }; + } + catch (Exception ex) when (ex is DriveNotFoundException || ex is ArgumentException) + { + return CGroupVersion.None; + } + } + /// Find the cgroup memory limit path. + /// The cgroup version currently in use on the system. /// The limit path if found; otherwise, null. - private static string? FindCGroupMemoryLimitPath() + private static string? FindCGroupMemoryLimitPath(CGroupVersion cgroupVersion) { - string? cgroupMemoryPath = FindCGroupPath("memory", out CGroupVersion version); + string? cgroupMemoryPath = FindCGroupPath(cgroupVersion, "memory"); if (cgroupMemoryPath != null) { - if (version == CGroupVersion.CGroup1) + if (cgroupVersion == CGroupVersion.CGroup1) { return cgroupMemoryPath + "/memory.limit_in_bytes"; } - if (version == CGroupVersion.CGroup2) + if (cgroupVersion == CGroupVersion.CGroup2) { // 'memory.high' is a soft limit; the process may get throttled // 'memory.max' is where OOM killer kicks in @@ -126,34 +152,71 @@ internal static bool TryReadMemoryValueFromFile(string path, out ulong result) } /// Find the cgroup path for the specified subsystem. + /// The cgroup version currently in use on the system. /// The subsystem, e.g. "memory". /// The cgroup path if found; otherwise, null. - private static string? FindCGroupPath(string subsystem, out CGroupVersion version) + private static string? FindCGroupPath(CGroupVersion cgroupVersion, string subsystem) { - if (TryFindHierarchyMount(subsystem, out version, out string? hierarchyRoot, out string? hierarchyMount) && - TryFindCGroupPathForSubsystem(subsystem, out string? cgroupPathRelativeToMount)) + if (cgroupVersion == CGroupVersion.None) + { + return null; + } + + if (TryFindHierarchyMount(cgroupVersion, subsystem, out string? hierarchyRoot, out string? hierarchyMount) && + TryFindCGroupPathForSubsystem(cgroupVersion, subsystem, out string? cgroupPathRelativeToMount)) { - // For a host cgroup, we need to append the relative path. - // In a docker container, the root and relative path are the same and we don't need to append. - return (hierarchyRoot != cgroupPathRelativeToMount) ? - hierarchyMount + cgroupPathRelativeToMount : - hierarchyMount; + return FindCGroupPath(hierarchyRoot, hierarchyMount, cgroupPathRelativeToMount); } return null; } + internal static string FindCGroupPath(string hierarchyRoot, string hierarchyMount, string cgroupPathRelativeToMount) + { + // For a host cgroup, we need to append the relative path. + // The root and cgroup path can share a common prefix of the path that should not be appended. + // Example 1 (docker): + // hierarchyMount: /sys/fs/cgroup/cpu + // hierarchyRoot: /docker/87ee2de57e51bc75175a4d2e81b71d162811b179d549d6601ed70b58cad83578 + // cgroupPathRelativeToMount: /docker/87ee2de57e51bc75175a4d2e81b71d162811b179d549d6601ed70b58cad83578/my_named_cgroup + // append to the cgroupPath: /my_named_cgroup + // final cgroupPath: /sys/fs/cgroup/cpu/my_named_cgroup + // + // Example 2 (out of docker) + // hierarchyMount: /sys/fs/cgroup/cpu + // hierarchyRoot: / + // cgroupPathRelativeToMount: /my_named_cgroup + // append to the cgroupPath: /my_named_cgroup + // final cgroupPath: /sys/fs/cgroup/cpu/my_named_cgroup + + int commonPathPrefixLength = hierarchyRoot.Length; + if ((commonPathPrefixLength == 1) || !cgroupPathRelativeToMount.StartsWith(hierarchyRoot, StringComparison.Ordinal)) + { + commonPathPrefixLength = 0; + } + + return string.Concat(hierarchyMount, cgroupPathRelativeToMount.AsSpan(commonPathPrefixLength)); + } + /// Find the cgroup mount information for the specified subsystem. + /// The cgroup version currently in use on the system. /// The subsystem, e.g. "memory". /// The path of the directory in the filesystem which forms the root of this mount; null if not found. /// The path of the mount point relative to the process's root directory; null if not found. /// true if the mount was found; otherwise, null. - private static bool TryFindHierarchyMount(string subsystem, out CGroupVersion version, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path) + private static bool TryFindHierarchyMount(CGroupVersion cgroupVersion, string subsystem, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path) { - return TryFindHierarchyMount(ProcMountInfoFilePath, subsystem, out version, out root, out path); + return TryFindHierarchyMount(cgroupVersion, ProcMountInfoFilePath, subsystem, out root, out path); } - internal static bool TryFindHierarchyMount(string mountInfoFilePath, string subsystem, out CGroupVersion version, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path) + /// Find the cgroup mount information for the specified subsystem. + /// The cgroup version currently in use on the system. + /// The path to the /mountinfo file. Useful for tests. + /// The subsystem, e.g. "memory". + /// The path of the directory in the filesystem which forms the root of this mount; null if not found. + /// The path of the mount point relative to the process's root directory; null if not found. + /// true if the mount was found; otherwise, null. + internal static bool TryFindHierarchyMount(CGroupVersion cgroupVersion, string mountInfoFilePath, string subsystem, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path) { if (File.Exists(mountInfoFilePath)) { @@ -188,31 +251,30 @@ internal static bool TryFindHierarchyMount(string mountInfoFilePath, string subs continue; } - bool validCGroup1Entry = ((postSeparatorlineParts[0] == "cgroup") && - (Array.IndexOf(postSeparatorlineParts[2].Split(','), subsystem) >= 0)); - bool validCGroup2Entry = postSeparatorlineParts[0] == "cgroup2"; - - if (!validCGroup1Entry && !validCGroup2Entry) + if (cgroupVersion == CGroupVersion.CGroup1) { - // Not the relevant entry. - continue; + bool validCGroup1Entry = ((postSeparatorlineParts[0] == "cgroup") && + (Array.IndexOf(postSeparatorlineParts[2].Split(','), subsystem) >= 0)); + if (!validCGroup1Entry) + { + continue; + } } + else if (cgroupVersion == CGroupVersion.CGroup2) + { + bool validCGroup2Entry = postSeparatorlineParts[0] == "cgroup2"; + if (!validCGroup2Entry) + { + continue; + } - // Found the relevant entry. Extract the cgroup version, mount root and path. - switch (postSeparatorlineParts[0]) + } + else { - case "cgroup": - version = CGroupVersion.CGroup1; - break; - case "cgroup2": - version = CGroupVersion.CGroup2; - break; - default: - version = CGroupVersion.None; - Debug.Fail($"invalid value for CGroupVersion \"{postSeparatorlineParts[0]}\""); - break; + Debug.Fail($"Unexpected cgroup version \"{cgroupVersion}\""); } + string[] lineParts = line.Substring(0, endOfOptionalFields).Split(' '); root = lineParts[3]; path = lineParts[4]; @@ -227,22 +289,27 @@ internal static bool TryFindHierarchyMount(string mountInfoFilePath, string subs } } - version = CGroupVersion.None; root = null; path = null; return false; } /// Find the cgroup relative path for the specified subsystem. + /// The cgroup version currently in use on the system. /// The subsystem, e.g. "memory". /// The found path, or null if it couldn't be found. - /// - private static bool TryFindCGroupPathForSubsystem(string subsystem, [NotNullWhen(true)] out string? path) + /// true if a cgroup path for the subsystem is found. + private static bool TryFindCGroupPathForSubsystem(CGroupVersion cgroupVersion, string subsystem, [NotNullWhen(true)] out string? path) { - return TryFindCGroupPathForSubsystem(ProcCGroupFilePath, subsystem, out path); + return TryFindCGroupPathForSubsystem(cgroupVersion, ProcCGroupFilePath, subsystem, out path); } - internal static bool TryFindCGroupPathForSubsystem(string procCGroupFilePath, string subsystem, [NotNullWhen(true)] out string? path) + /// Find the cgroup relative path for the specified subsystem. + /// The cgroup version currently in use on the system. + /// The subsystem, e.g. "memory". + /// The found path, or null if it couldn't be found. + /// true if a cgroup path for the subsystem is found. + internal static bool TryFindCGroupPathForSubsystem(CGroupVersion cgroupVersion, string procCGroupFilePath, string subsystem, [NotNullWhen(true)] out string? path) { if (File.Exists(procCGroupFilePath)) { @@ -261,28 +328,36 @@ internal static bool TryFindCGroupPathForSubsystem(string procCGroupFilePath, st continue; } - // cgroup v2: Find the first entry that matches the cgroup v2 hierarchy: - // 0::$PATH - - if ((lineParts[0] == "0") && (string.Empty == lineParts[1])) + if (cgroupVersion == CGroupVersion.CGroup1) { + // cgroup v1: Find the first entry that has the subsystem listed in its controller + // list. See man page for cgroups for /proc/[pid]/cgroups format, e.g: + // hierarchy-ID:controller-list:cgroup-path + // 5:cpuacct,cpu,cpuset:/daemons + if (Array.IndexOf(lineParts[1].Split(','), subsystem) < 0) + { + // Not the relevant entry. + continue; + } + path = lineParts[2]; return true; } - - // cgroup v1: Find the first entry that has the subsystem listed in its controller - // list. See man page for cgroups for /proc/[pid]/cgroups format, e.g: - // hierarchy-ID:controller-list:cgroup-path - // 5:cpuacct,cpu,cpuset:/daemons - - if (Array.IndexOf(lineParts[1].Split(','), subsystem) < 0) + else if (cgroupVersion == CGroupVersion.CGroup2) { - // Not the relevant entry. - continue; + // cgroup v2: Find the first entry that matches the cgroup v2 hierarchy: + // 0::$PATH + + if ((lineParts[0] == "0") && (lineParts[1] == string.Empty)) + { + path = lineParts[2]; + return true; + } + } + else + { + Debug.Fail($"Unexpected cgroup version: \"{cgroupVersion}\""); } - - path = lineParts[2]; - return true; } } } diff --git a/src/libraries/Common/tests/Tests/Interop/cgroupsTests.cs b/src/libraries/Common/tests/Tests/Interop/cgroupsTests.cs index fc6ab5c9753ce4..a40713ed36899a 100644 --- a/src/libraries/Common/tests/Tests/Interop/cgroupsTests.cs +++ b/src/libraries/Common/tests/Tests/Interop/cgroupsTests.cs @@ -9,6 +9,12 @@ namespace Common.Tests { public class cgroupsTests : FileCleanupTestBase { + [Fact] + public void ValidateFindCGroupVersion() + { + Assert.InRange((int)Interop.cgroups.s_cgroupVersion, 0, 2); + } + [Theory] [InlineData(true, "0", 0)] [InlineData(false, "max", 0)] @@ -27,48 +33,57 @@ public void ValidateTryReadMemoryValue(bool expectedResult, string valueText, ul } [Theory] - [InlineData(false, "0 0 0:0 / /foo ignore ignore - overlay overlay ignore", "ignore", 0, "/", "/")] - [InlineData(true, "0 0 0:0 / /foo ignore ignore - cgroup2 cgroup2 ignore", "ignore", 2, "/", "/foo")] - [InlineData(true, "0 0 0:0 / /foo ignore ignore - cgroup2 cgroup2 ignore", "memory", 2, "/", "/foo")] - [InlineData(true, "0 0 0:0 / /foo ignore ignore - cgroup2 cgroup2 ignore", "cpu", 2, "/", "/foo")] - [InlineData(true, "0 0 0:0 / /foo ignore - cgroup2 cgroup2 ignore", "cpu", 2, "/", "/foo")] - [InlineData(true, "0 0 0:0 / /foo ignore ignore ignore - cgroup2 cgroup2 ignore", "cpu", 2, "/", "/foo")] - [InlineData(true, "0 0 0:0 / /foo-with-dashes ignore ignore - cgroup2 cgroup2 ignore", "ignore", 2, "/", "/foo-with-dashes")] - [InlineData(true, "0 0 0:0 / /foo ignore ignore - cgroup cgroup memory", "memory", 1, "/", "/foo")] - [InlineData(true, "0 0 0:0 / /foo-with-dashes ignore ignore - cgroup cgroup memory", "memory", 1, "/", "/foo-with-dashes")] - [InlineData(true, "0 0 0:0 / /foo ignore ignore - cgroup cgroup cpu,memory", "memory", 1, "/", "/foo")] - [InlineData(true, "0 0 0:0 / /foo ignore ignore - cgroup cgroup memory,cpu", "memory", 1, "/", "/foo")] - [InlineData(false, "0 0 0:0 / /foo ignore ignore - cgroup cgroup cpu", "memory", 0, "/", "/foo")] - public void ParseValidateMountInfo(bool expectedFound, string procSelfMountInfoText, string subsystem, int expectedVersion, string expectedRoot, string expectedMount) + [InlineData("/sys/fs/cgroup/cpu/my_cgroup", "/docker/1234", "/sys/fs/cgroup/cpu", "/docker/1234/my_cgroup")] + [InlineData("/sys/fs/cgroup/cpu/my_cgroup", "/", "/sys/fs/cgroup/cpu", "/my_cgroup")] + public void ValidateFindCGroupPath(string expectedResult, string hierarchyRoot, string hierarchyMount, string cgroupPathRelativeToMount) + { + Assert.Equal(expectedResult, Interop.cgroups.FindCGroupPath(hierarchyRoot, hierarchyMount, cgroupPathRelativeToMount)); + } + + [Theory] + [InlineData(true, 2, "0 0 0:0 / /foo ignore ignore - cgroup2 cgroup2 ignore", "ignore", "/", "/foo")] + [InlineData(true, 2, "0 0 0:0 / /foo ignore ignore - cgroup2 cgroup2 ignore", "memory", "/", "/foo")] + [InlineData(true, 2, "0 0 0:0 / /foo ignore ignore - cgroup2 cgroup2 ignore", "cpu", "/", "/foo")] + [InlineData(true, 2, "0 0 0:0 / /foo ignore - cgroup2 cgroup2 ignore", "cpu", "/", "/foo")] + [InlineData(true, 2, "0 0 0:0 / /foo ignore ignore ignore - cgroup2 cgroup2 ignore", "cpu", "/", "/foo")] + [InlineData(true, 2, "0 0 0:0 / /foo-with-dashes ignore ignore - cgroup2 cgroup2 ignore", "ignore", "/", "/foo-with-dashes")] + [InlineData(true, 1, "0 0 0:0 / /foo ignore ignore - cgroup cgroup memory", "memory", "/", "/foo")] + [InlineData(true, 1, "0 0 0:0 / /foo-with-dashes ignore ignore - cgroup cgroup memory", "memory", "/", "/foo-with-dashes")] + [InlineData(true, 1, "0 0 0:0 / /foo ignore ignore - cgroup cgroup cpu,memory", "memory", "/", "/foo")] + [InlineData(true, 1, "0 0 0:0 / /foo ignore ignore - cgroup cgroup memory,cpu", "memory", "/", "/foo")] + public void ParseValidateMountInfo(bool expectedFound, int cgroupVersion, string procSelfMountInfoText, string subsystem, string expectedRoot, string expectedMount) { string path = GetTestFilePath(); File.WriteAllText(path, procSelfMountInfoText); - Assert.Equal(expectedFound, Interop.cgroups.TryFindHierarchyMount(path, subsystem, out Interop.cgroups.CGroupVersion version, out string root, out string mount)); + Assert.Equal(expectedFound, Interop.cgroups.TryFindHierarchyMount((Interop.cgroups.CGroupVersion) cgroupVersion, + path, subsystem, out string root, out string mount)); if (expectedFound) { - Assert.Equal(expectedVersion, (int)version); Assert.Equal(expectedRoot, root); Assert.Equal(expectedMount, mount); } } [Theory] - [InlineData(true, "0::/foo", "ignore", "/foo")] - [InlineData(true, "0::/bar", "ignore", "/bar")] - [InlineData(true, "0::frob", "ignore", "frob")] - [InlineData(false, "1::frob", "ignore", "ignore")] - [InlineData(true, "1:foo:bar", "foo", "bar")] - [InlineData(true, "2:foo:bar", "foo", "bar")] - [InlineData(false, "2:foo:bar", "bar", "ignore")] - [InlineData(true, "1:foo:bar\n2:eggs:spam", "foo", "bar")] - [InlineData(true, "1:foo:bar\n2:eggs:spam", "eggs", "spam")] - public void ParseValidateProcCGroup(bool expectedFound, string procSelfCgroupText, string subsystem, string expectedMountPath) + [InlineData(true, 2, "0::/foo", "ignore", "/foo")] + [InlineData(true, 2, "0::/bar", "ignore", "/bar")] + [InlineData(true, 2, "0::frob", "ignore", "frob")] + [InlineData(false, 1, "1::frob", "ignore", "ignore")] + [InlineData(true, 1, "1:foo:bar", "foo", "bar")] + [InlineData(true, 1, "0::baz\n1:foo:bar", "foo", "bar")] + [InlineData(true, 1, "2:foo:bar", "foo", "bar")] + [InlineData(false, 1, "2:foo:bar", "bar", "ignore")] + [InlineData(true, 1, "1:foo:bar\n2:eggs:spam", "foo", "bar")] + [InlineData(true, 1, "1:foo:bar\n2:eggs:spam", "eggs", "spam")] + [InlineData(true, 1, "2:eggs:spam\n0:foo:bar", "eggs", "spam")] + public void ParseValidateProcCGroup(bool expectedFound, int cgroupVersion, string procSelfCgroupText, string subsystem, string expectedMountPath) { string path = GetTestFilePath(); File.WriteAllText(path, procSelfCgroupText); - Assert.Equal(expectedFound, Interop.cgroups.TryFindCGroupPathForSubsystem(path, subsystem, out string mountPath)); + Assert.Equal(expectedFound, Interop.cgroups.TryFindCGroupPathForSubsystem((Interop.cgroups.CGroupVersion) cgroupVersion, + path, subsystem, out string mountPath)); if (expectedFound) { Assert.Equal(expectedMountPath, mountPath); diff --git a/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj b/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj index ce8806c4e3f093..70a003c2012a33 100644 --- a/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj +++ b/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj @@ -458,6 +458,7 @@ + @@ -468,4 +469,4 @@ - \ No newline at end of file + diff --git a/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs b/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs index bbc28bce1546ba..1647c62cccadec 100644 --- a/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs +++ b/src/libraries/System.Runtime.InteropServices.RuntimeInformation/tests/DescriptionNameTests.cs @@ -41,6 +41,7 @@ public void DumpRuntimeInformationToConsole() Console.WriteLine($"### CURRENT DIRECTORY: {Environment.CurrentDirectory}"); + Console.WriteLine($"### CGROUPS VERSION: {Interop.cgroups.s_cgroupVersion}"); string cgroupsLocation = Interop.cgroups.s_cgroupMemoryLimitPath; if (cgroupsLocation != null) {