From 0905479bd175bf95fd508978e1b5318bbb2b40a8 Mon Sep 17 00:00:00 2001 From: Ivan Duplenskikh <115665590+ivanduplenskikh@users.noreply.github.com> Date: Tue, 20 Feb 2024 19:52:33 +0100 Subject: [PATCH] Update Git to v2.43.0 (#4618) * Update Git to a new version Add feature flag to use different Git versions in Worker and Listener Separate Git config and Git usage --- src/Agent.Plugins/GitCliManager.cs | 59 ++++++-- src/Agent.Plugins/GitSourceProvider.cs | 40 ++++-- src/Agent.Sdk/Knob/AgentKnobs.cs | 21 +++ src/Agent.Worker/Build/GitCommandManager.cs | 44 +++++- src/Agent.Worker/Build/GitSourceProvider.cs | 28 ++++ .../Constants.cs | 8 ++ src/Misc/externals.sh | 2 +- .../TestGitCliManager/TestGitCliManagerL0.cs | 132 +++++++++--------- .../GitSourceProviderL0.cs | 53 +++++++ .../MockAgentTaskPluginExecutionContext.cs | 9 ++ .../L0/Worker/Build/GitCommandManagerL0.cs | 66 +++++++++ .../L0/Worker/Build/GitSourceProviderL0.cs | 48 +++++++ 12 files changed, 417 insertions(+), 93 deletions(-) create mode 100644 src/Test/L0/Plugin/TestGitSourceProvider/GitSourceProviderL0.cs create mode 100644 src/Test/L0/Plugin/TestGitSourceProvider/MockAgentTaskPluginExecutionContext.cs create mode 100644 src/Test/L0/Worker/Build/GitCommandManagerL0.cs diff --git a/src/Agent.Plugins/GitCliManager.cs b/src/Agent.Plugins/GitCliManager.cs index ff26e269d1..95b9c542f7 100644 --- a/src/Agent.Plugins/GitCliManager.cs +++ b/src/Agent.Plugins/GitCliManager.cs @@ -18,7 +18,12 @@ namespace Agent.Plugins.Repository { - public class GitCliManager + public interface IGitCliManager + { + Task GitConfig(AgentTaskPluginExecutionContext context, string repositoryPath, string configKey, string configValue); + } + + public class GitCliManager : IGitCliManager { private static Encoding _encoding { @@ -77,27 +82,57 @@ public bool EnsureGitLFSVersion(Version requiredVersion, bool throwOnNotMatch) return gitLfsVersion >= requiredVersion; } + public (string gitPath, string gitLfsPath) GetInternalGitPaths( + AgentTaskPluginExecutionContext context, + bool useLatestGitVersion) + { + string agentHomeDir = context.Variables.GetValueOrDefault("agent.homedirectory")?.Value; + ArgUtil.NotNullOrEmpty(agentHomeDir, nameof(agentHomeDir)); + + string gitPath; + + if (useLatestGitVersion) + { + gitPath = Path.Combine(agentHomeDir, "externals", "ff_git", "cmd", $"git.exe"); + } + else + { + gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe"); + } + + context.Debug($@"The useLatestGitVersion property is set to ""{useLatestGitVersion}"" therefore the Git path is ""{gitPath}"""); + + string gitLfsPath; + + if (PlatformUtil.BuiltOnX86) + { + gitLfsPath = Path.Combine(agentHomeDir, "externals", "git", "mingw32", "bin", "git-lfs.exe"); + } + else + { + gitLfsPath = Path.Combine(agentHomeDir, "externals", "git", "mingw64", "bin", "git-lfs.exe"); + } + + return (gitPath, gitLfsPath); + } + public virtual async Task LoadGitExecutionInfo(AgentTaskPluginExecutionContext context, bool useBuiltInGit) { // There is no built-in git for OSX/Linux gitPath = null; + gitLfsPath = null; // Resolve the location of git. if (useBuiltInGit && PlatformUtil.RunningOnWindows) { - string agentHomeDir = context.Variables.GetValueOrDefault("agent.homedirectory")?.Value; - ArgUtil.NotNullOrEmpty(agentHomeDir, nameof(agentHomeDir)); + context.Debug("Git paths are resolving from internal dependencies"); - if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(context).AsBoolean()) - { - gitPath = Path.Combine(agentHomeDir, "externals", "ff_git", "cmd", $"git.exe"); - } - else - { - gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe"); - } + var (resolvedGitPath, resolvedGitLfsPath) = GetInternalGitPaths( + context, + AgentKnobs.UseLatestGitVersion.GetValue(context).AsBoolean()); - gitLfsPath = Path.Combine(agentHomeDir, "externals", "git", PlatformUtil.BuiltOnX86 ? "mingw32" : "mingw64", "bin", "git-lfs.exe"); + gitPath = resolvedGitPath; + gitLfsPath = resolvedGitLfsPath; // Prepend the PATH. context.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", "Path", Path.GetFileName(gitPath))); diff --git a/src/Agent.Plugins/GitSourceProvider.cs b/src/Agent.Plugins/GitSourceProvider.cs index 0ae32f094a..1d5563b6a8 100644 --- a/src/Agent.Plugins/GitSourceProvider.cs +++ b/src/Agent.Plugins/GitSourceProvider.cs @@ -404,7 +404,7 @@ public async Task GetSourceAsync( // Make sure the build machine met all requirements for the git repository // For now, the requirement we have are: - // 1. git version greater than 2.9 and git-lfs version greater than 2.1 for on-prem tfsgit + // 1. git version greater than 2.9 and git-lfs version greater than 2.1 for on-prem tfsgit // 2. git version greater than 2.14.2 if use SChannel for SSL backend (Windows only) RequirementCheck(executionContext, repository, gitCommandManager); string username = string.Empty; @@ -684,17 +684,7 @@ public async Task GetSourceAsync( executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performance of fetching."); } - if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(executionContext).AsBoolean()) - { - await gitCommandManager.GitConfig(executionContext, targetPath, "pack.threads", "1"); - await gitCommandManager.GitConfig(executionContext, targetPath, "http.postBuffer", "524288000"); - await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitwindowsize", "256m"); - await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitlimit", "256m"); - await gitCommandManager.GitConfig(executionContext, targetPath, "pack.windowmemory", "256m"); - await gitCommandManager.GitConfig(executionContext, targetPath, "pack.deltaCacheSize", "256m"); - await gitCommandManager.GitConfig(executionContext, targetPath, "pack.packSizeLimit", "256m"); - await gitCommandManager.GitConfig(executionContext, targetPath, "core.longpaths", "true"); - } + SetGitFeatureFlagsConfiguration(executionContext, gitCommandManager, targetPath); // always remove any possible left extraheader setting from git config. if (await gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader")) @@ -1315,6 +1305,32 @@ public async Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionC } } + public async void SetGitFeatureFlagsConfiguration( + AgentTaskPluginExecutionContext executionContext, + IGitCliManager gitCommandManager, + string targetPath) + { + if (AgentKnobs.UseGitSingleThread.GetValue(executionContext).AsBoolean()) + { + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.threads", "1"); + } + + if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(executionContext).AsBoolean()) + { + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.windowmemory", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.deltaCacheSize", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.packSizeLimit", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "http.postBuffer", "524288000"); + await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitwindowsize", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitlimit", "256m"); + } + + if (AgentKnobs.UseGitLongPaths.GetValue(executionContext).AsBoolean()) + { + await gitCommandManager.GitConfig(executionContext, targetPath, "core.longpaths", "true"); + } + } + protected virtual GitCliManager GetCliManager(Dictionary gitEnv = null) { return new GitCliManager(gitEnv); diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs index 9fb7f91f7d..6b3f01f6a4 100644 --- a/src/Agent.Sdk/Knob/AgentKnobs.cs +++ b/src/Agent.Sdk/Knob/AgentKnobs.cs @@ -128,6 +128,27 @@ public class AgentKnobs new EnvironmentKnobSource("FIX_POSSIBLE_GIT_OUT_OF_MEMORY_PROBLEM"), new BuiltInDefaultKnobSource("false")); + public static readonly Knob UseGitLongPaths = new Knob( + nameof(UseGitLongPaths), + "When true, set core.longpaths to true", + new RuntimeKnobSource("USE_GIT_LONG_PATHS"), + new EnvironmentKnobSource("USE_GIT_LONG_PATHS"), + new BuiltInDefaultKnobSource("false")); + + public static readonly Knob UseGitSingleThread = new Knob( + nameof(UseGitSingleThread), + "When true, spawn only one thread searching for best delta matches", + new RuntimeKnobSource("USE_GIT_SINGLE_THREAD"), + new EnvironmentKnobSource("USE_GIT_SINGLE_THREAD"), + new BuiltInDefaultKnobSource("false")); + + public static readonly Knob UseLatestGitVersion = new Knob( + nameof(UseLatestGitVersion), + "When true, set path to the latest git version", + new RuntimeKnobSource("USE_LATEST_GIT_VERSION"), + new EnvironmentKnobSource("USE_LATEST_GIT_VERSION"), + new BuiltInDefaultKnobSource("false")); + public static readonly Knob TfVCUseSecureParameterPassing = new Knob( nameof(TfVCUseSecureParameterPassing), "If true, don't pass auth token in TFVC parameters", diff --git a/src/Agent.Worker/Build/GitCommandManager.cs b/src/Agent.Worker/Build/GitCommandManager.cs index da6637fae8..6540379fab 100644 --- a/src/Agent.Worker/Build/GitCommandManager.cs +++ b/src/Agent.Worker/Build/GitCommandManager.cs @@ -22,6 +22,8 @@ public interface IGitCommandManager : IAgentService bool EnsureGitLFSVersion(Version requiredVersion, bool throwOnNotMatch); + (string resolvedGitPath, string resolvedGitLfsPath) GetInternalGitPaths(IExecutionContext context, bool useLatestGitVersion); + // setup git execution info, git location, version, useragent, execpath Task LoadGitExecutionInfo(IExecutionContext context, bool useBuiltInGit, Dictionary gitEnv = null); @@ -160,6 +162,38 @@ public bool EnsureGitLFSVersion(Version requiredVersion, bool throwOnNotMatch) return _gitLfsVersion >= requiredVersion; } + public (string resolvedGitPath, string resolvedGitLfsPath) GetInternalGitPaths(IExecutionContext context, bool useLatestGitVersion) + { + string externalsDirectoryPath = HostContext.GetDirectory(WellKnownDirectory.Externals); + ArgUtil.NotNullOrEmpty(externalsDirectoryPath, nameof(WellKnownDirectory.Externals)); + + string gitPath; + + if (useLatestGitVersion) + { + gitPath = Path.Combine(externalsDirectoryPath, "ff_git", "cmd", $"git.exe"); + } + else + { + gitPath = Path.Combine(externalsDirectoryPath, "git", "cmd", $"git.exe"); + } + + context.Debug($@"The useLatestGitVersion property is set to ""{useLatestGitVersion}"" therefore the Git path is ""{gitPath}"""); + + string gitLfsPath; + + if (PlatformUtil.BuiltOnX86) + { + gitLfsPath = Path.Combine(externalsDirectoryPath, "git", "mingw32", "bin", $"git-lfs.exe"); + } + else + { + gitLfsPath = Path.Combine(externalsDirectoryPath, "git", "mingw64", "bin", $"git-lfs.exe"); + } + + return (gitPath, gitLfsPath); + } + public async Task LoadGitExecutionInfo(IExecutionContext context, bool useBuiltInGit, Dictionary gitEnv = null) { if (gitEnv != null) @@ -181,8 +215,14 @@ public async Task LoadGitExecutionInfo(IExecutionContext context, bool useBuiltI // The Windows agent ships a copy of Git if (PlatformUtil.RunningOnWindows) { - _gitPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "git", "cmd", $"git{IOUtil.ExeExtension}"); - _gitLfsPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "git", PlatformUtil.BuiltOnX86 ? "mingw32" : "mingw64", "bin", "git-lfs.exe"); + context.Debug("Git paths are resolving from internal dependencies"); + + var (resolvedGitPath, resolvedGitLfsPath) = GetInternalGitPaths( + context, + AgentKnobs.UseLatestGitVersion.GetValue(context).AsBoolean()); + + _gitPath = resolvedGitPath; + _gitLfsPath = resolvedGitLfsPath; // Prepend the PATH. context.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", Constants.PathVariable, Path.GetFileName(_gitPath))); diff --git a/src/Agent.Worker/Build/GitSourceProvider.cs b/src/Agent.Worker/Build/GitSourceProvider.cs index 670a74db5b..9744a95e24 100644 --- a/src/Agent.Worker/Build/GitSourceProvider.cs +++ b/src/Agent.Worker/Build/GitSourceProvider.cs @@ -657,6 +657,8 @@ public async Task GetSourceAsync( executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performance of fetching."); } + SetGitFeatureFlagsConfiguration(executionContext, _gitCommandManager, targetPath); + // always remove any possible left extraheader setting from git config. if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader")) { @@ -1229,6 +1231,32 @@ public override async Task RunMaintenanceOperations(IExecutionContext executionC } } + public async void SetGitFeatureFlagsConfiguration( + IExecutionContext executionContext, + IGitCommandManager gitCommandManager, + string targetPath) + { + if (AgentKnobs.UseGitSingleThread.GetValue(executionContext).AsBoolean()) + { + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.threads", "1"); + } + + if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(executionContext).AsBoolean()) + { + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.windowmemory", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.deltaCacheSize", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.packSizeLimit", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "http.postBuffer", "524288000"); + await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitwindowsize", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitlimit", "256m"); + } + + if (AgentKnobs.UseGitLongPaths.GetValue(executionContext).AsBoolean()) + { + await gitCommandManager.GitConfig(executionContext, targetPath, "core.longpaths", "true"); + } + } + public override void SetVariablesInEndpoint(IExecutionContext executionContext, ServiceEndpoint endpoint) { ArgUtil.NotNull(executionContext, nameof(executionContext)); diff --git a/src/Microsoft.VisualStudio.Services.Agent/Constants.cs b/src/Microsoft.VisualStudio.Services.Agent/Constants.cs index fcc5e2b777..2707bdaaee 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/Constants.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/Constants.cs @@ -322,6 +322,7 @@ public static class Agent public static readonly string ContainerMapping = "agent.containermapping"; public static readonly string ContainerNetwork = "agent.containernetwork"; public static readonly string Diagnostic = "agent.diagnostic"; + public static readonly string FixPossibleGitOutOfMemoryProblem = "FIX_POSSIBLE_GIT_OUT_OF_MEMORY_PROBLEM"; public static readonly string HomeDirectory = "agent.homedirectory"; public static readonly string Id = "agent.id"; public static readonly string IsSelfHosted = "agent.isselfhosted"; @@ -351,6 +352,9 @@ public static class Agent public static readonly string SslSkipCertValidation = "agent.skipcertvalidation"; public static readonly string TempDirectory = "agent.TempDirectory"; public static readonly string ToolsDirectory = "agent.ToolsDirectory"; + public static readonly string UseGitLongPaths = "USE_GIT_LONG_PATHS"; + public static readonly string UseGitSingleThread = "USE_GIT_SINGLE_THREAD"; + public static readonly string UseLatestGitVersion = "USE_LATEST_GIT_VERSION"; public static readonly string Version = "agent.version"; public static readonly string WorkFolder = "agent.workfolder"; public static readonly string WorkingDirectory = "agent.WorkingDirectory"; @@ -522,6 +526,7 @@ public static class Task Agent.ContainerMapping, Agent.ContainerNetwork, Agent.Diagnostic, + Agent.FixPossibleGitOutOfMemoryProblem, Agent.GitUseSChannel, Agent.HomeDirectory, Agent.Id, @@ -551,6 +556,9 @@ public static class Task Agent.SslSkipCertValidation, Agent.TempDirectory, Agent.ToolsDirectory, + Agent.UseGitLongPaths, + Agent.UseGitSingleThread, + Agent.UseLatestGitVersion, Agent.Version, Agent.WorkFolder, Agent.WorkingDirectory, diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index a9a01f61db..4f5494c928 100644 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -19,7 +19,7 @@ NODE10_VERSION="10.24.1" NODE16_VERSION="16.20.2" NODE20_VERSION="20.9.0" MINGIT_VERSION="2.39.1" -FF_MINGIT_VERSION="2.42.0.2" +FF_MINGIT_VERSION="2.43.0" LFS_VERSION="3.3.0" get_abs_path() { diff --git a/src/Test/L0/Plugin/TestGitCliManager/TestGitCliManagerL0.cs b/src/Test/L0/Plugin/TestGitCliManager/TestGitCliManagerL0.cs index f9fbca3299..7ce40131cb 100644 --- a/src/Test/L0/Plugin/TestGitCliManager/TestGitCliManagerL0.cs +++ b/src/Test/L0/Plugin/TestGitCliManager/TestGitCliManagerL0.cs @@ -1,12 +1,9 @@ -using Agent.Plugins.Repository; -using Agent.Sdk; -using Microsoft.VisualStudio.Services.Agent.Tests; +using Microsoft.VisualStudio.Services.Agent.Tests; using Microsoft.VisualStudio.Services.Agent.Util; using Moq; using System; using System.Collections.Generic; using System.IO; -using System.Text; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -15,13 +12,20 @@ namespace Test.L0.Plugin.TestGitCliManager { public class TestGitCliManagerL0 { + private readonly string gitPath = Path.Combine("agenthomedirectory", "externals", "git", "cmd", "git.exe"); + private readonly string ffGitPath = Path.Combine("agenthomedirectory", "externals", "ff_git", "cmd", "git.exe"); + private Tuple, MockAgentTaskPluginExecutionContext> setupMocksForGitLfsFetchTests(TestHostContext hostContext) { - Mock argUtilInstanced = new Mock(); - argUtilInstanced.CallBase = true; - argUtilInstanced.Setup(x => x.File(Path.Combine("agenthomedirectory", "externals", "git", "cmd", $"git.exe"), "gitPath")).Callback(() => { }); - argUtilInstanced.Setup(x => x.File(Path.Combine("agenthomedirectory", "externals", "ff_git", "cmd", $"git.exe"), "gitPath")).Callback(() => { }); - argUtilInstanced.Setup(x => x.Directory("agentworkfolder", "agent.workfolder")).Callback(() => { }); + Mock argUtilInstanced = new Mock() + { + CallBase = true + }; + + argUtilInstanced.Setup(x => x.File(gitPath, "gitPath")).Callback(() => { }); + argUtilInstanced.Setup(x => x.File(ffGitPath, "gitPath")).Callback(() => { }); + argUtilInstanced.Setup(x => x.Directory("agentworkfolder", "agent.workfolder")); + var context = new MockAgentTaskPluginExecutionContext(hostContext.GetTrace()); context.Variables.Add("agent.homedirectory", "agenthomedirectory"); context.Variables.Add("agent.workfolder", "agentworkfolder"); @@ -29,86 +33,82 @@ private Tuple, MockAgentTaskPluginExecutionContext> setup return Tuple.Create(argUtilInstanced, context); } - [Fact] + public static IEnumerable UseNewGitVersionFeatureFlagsData => new List + { + new object[] { true }, + new object[] { false }, + }; + + [Theory] [Trait("Level", "L0")] [Trait("Category", "Plugin")] - public async Task TestGitLfsFetchLfsConfigExistsAsync() + [Trait("SkipOn", "darwin")] + [Trait("SkipOn", "linux")] + [MemberData(nameof(UseNewGitVersionFeatureFlagsData))] + public void TestGetInternalGitPaths(bool gitFeatureFlagStatus) { - using (var hostContext = new TestHostContext(this)) - { - // Setup - var originalArgUtilInstance = ArgUtil.ArgUtilInstance; - var mocks = this.setupMocksForGitLfsFetchTests(hostContext); - var argUtilInstanced = mocks.Item1; - var mockAgentTaskPluginExecutionContext = mocks.Item2; - - try - { - ArgUtil.ArgUtilInstance = argUtilInstanced.Object; - - var gitCliManagerMock = new MockGitCliManager(); - - gitCliManagerMock.IsLfsConfigExistsing = true; - await gitCliManagerMock.LoadGitExecutionInfo(mockAgentTaskPluginExecutionContext, true); + using var hostContext = new TestHostContext(this, $"GitFeatureFlagStatus_{gitFeatureFlagStatus}"); - ArgUtil.NotNull(gitCliManagerMock, ""); + // Setup + var originalArgUtilInstance = ArgUtil.ArgUtilInstance; + var mocks = setupMocksForGitLfsFetchTests(hostContext); + var argUtilInstanced = mocks.Item1; + var mockAgentTaskPluginExecutionContext = mocks.Item2; - // Action - await gitCliManagerMock.GitLFSFetch(mockAgentTaskPluginExecutionContext, "repositoryPath", "remoteName", "refSpec", "additionalCmdLine", CancellationToken.None); + ArgUtil.ArgUtilInstance = argUtilInstanced.Object; + MockGitCliManager gitCliManagerMock = new(); + var (resolvedGitPath, resolvedGitLfsPath) = gitCliManagerMock.GetInternalGitPaths( + mockAgentTaskPluginExecutionContext, + gitFeatureFlagStatus); - // Assert - Assert.Equal(2, gitCliManagerMock.GitCommandCallsOptions.Count); - - Assert.True(gitCliManagerMock.GitCommandCallsOptions.Contains("repositoryPath,checkout,refSpec -- .lfsconfig,additionalCmdLine"), "ExecuteGitCommandAsync should pass arguments properly to 'git checkout .lfsconfig' command"); - Assert.True(gitCliManagerMock.GitCommandCallsOptions.Contains("repositoryPath,lfs,fetch origin refSpec,additionalCmdLine"), "ExecuteGitCommandAsync should pass arguments properly to 'git lfs fetch' command"); - - } - finally - { - ArgUtil.ArgUtilInstance = originalArgUtilInstance; - } + if (gitFeatureFlagStatus) + { + Assert.Equal(resolvedGitPath, ffGitPath); + } + else + { + Assert.Equal(resolvedGitPath, gitPath); } } - [Fact] [Trait("Level", "L0")] [Trait("Category", "Plugin")] public async Task TestGitLfsFetchLfsConfigDoesNotExist() { - using (var hostContext = new TestHostContext(this)) + using var hostContext = new TestHostContext(this); + // Setup + var originalArgUtilInstance = ArgUtil.ArgUtilInstance; + var mocks = setupMocksForGitLfsFetchTests(hostContext); + var argUtilInstanced = mocks.Item1; + var mockAgentTaskPluginExecutionContext = mocks.Item2; + + try { - // Setup - var originalArgUtilInstance = ArgUtil.ArgUtilInstance; - var mocks = this.setupMocksForGitLfsFetchTests(hostContext); - var argUtilInstanced = mocks.Item1; - var mockAgentTaskPluginExecutionContext = mocks.Item2; + ArgUtil.ArgUtilInstance = argUtilInstanced.Object; - try + var gitCliManagerMock = new MockGitCliManager() { - ArgUtil.ArgUtilInstance = argUtilInstanced.Object; - - var gitCliManagerMock = new MockGitCliManager(); + IsLfsConfigExistsing = false + }; - gitCliManagerMock.IsLfsConfigExistsing = false; - await gitCliManagerMock.LoadGitExecutionInfo(mockAgentTaskPluginExecutionContext, true); + await gitCliManagerMock.LoadGitExecutionInfo(mockAgentTaskPluginExecutionContext, true); - ArgUtil.NotNull(gitCliManagerMock, ""); + ArgUtil.NotNull(gitCliManagerMock, ""); - // Action - await gitCliManagerMock.GitLFSFetch(mockAgentTaskPluginExecutionContext, "repositoryPath", "remoteName", "refSpec", "additionalCmdLine", CancellationToken.None); + // Action + await gitCliManagerMock.GitLFSFetch(mockAgentTaskPluginExecutionContext, "repositoryPath", "remoteName", "refSpec", "additionalCmdLine", CancellationToken.None); - // Assert - Assert.Equal(2, gitCliManagerMock.GitCommandCallsOptions.Count); + // Assert + Assert.Equal(2, gitCliManagerMock.GitCommandCallsOptions.Count); - Assert.True(gitCliManagerMock.GitCommandCallsOptions.Contains("repositoryPath,checkout,refSpec -- .lfsconfig,additionalCmdLine"), "ExecuteGitCommandAsync should pass arguments properly to 'git checkout .lfsconfig' command"); - Assert.True(gitCliManagerMock.GitCommandCallsOptions.Contains("repositoryPath,lfs,fetch origin refSpec,additionalCmdLine"), "ExecuteGitCommandAsync should pass arguments properly to 'git lfs fetch' command"); + Assert.True(gitCliManagerMock.GitCommandCallsOptions.Contains("repositoryPath,checkout,refSpec -- .lfsconfig,additionalCmdLine"), "ExecuteGitCommandAsync should pass arguments properly to 'git checkout .lfsconfig' command"); + Assert.True(gitCliManagerMock.GitCommandCallsOptions.Contains("repositoryPath,lfs,fetch origin refSpec,additionalCmdLine"), "ExecuteGitCommandAsync should pass arguments properly to 'git lfs fetch' command"); - } - finally - { - ArgUtil.ArgUtilInstance = originalArgUtilInstance; - } + } + finally + { + ArgUtil.ArgUtilInstance = originalArgUtilInstance; } } } diff --git a/src/Test/L0/Plugin/TestGitSourceProvider/GitSourceProviderL0.cs b/src/Test/L0/Plugin/TestGitSourceProvider/GitSourceProviderL0.cs new file mode 100644 index 0000000000..1e87c01c03 --- /dev/null +++ b/src/Test/L0/Plugin/TestGitSourceProvider/GitSourceProviderL0.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.VisualStudio.Services.Agent; +using Microsoft.VisualStudio.Services.Agent.Tests; +using Xunit; +using System.IO; +using System; +using Moq; +using Agent.Plugins.Repository; +using System.Collections.Generic; + +namespace Test.L0.Plugin.TestGitSourceProvider; + +public sealed class TestPluginGitSourceProviderL0 +{ + private readonly Func getWorkFolder = hc => hc.GetDirectory(WellKnownDirectory.Work); + + public static IEnumerable FeatureFlagsStatusData => new List + { + new object[] { true }, + new object[] { false }, + }; + + [Theory] + [Trait("Level", "L0")] + [Trait("Category", "Plugin")] + [Trait("SkipOn", "darwin")] + [Trait("SkipOn", "linux")] + [MemberData(nameof(FeatureFlagsStatusData))] + public void TestSetGitConfiguration(bool featureFlagsStatus) + { + using TestHostContext hc = new(this, $"FeatureFlagsStatus_{featureFlagsStatus}"); + MockAgentTaskPluginExecutionContext tc = new(hc.GetTrace()); + var gitCliManagerMock = new Mock(); + + var repositoryPath = Path.Combine(getWorkFolder(hc), "1", "testrepo"); + var featureFlagStatusString = featureFlagsStatus.ToString(); + var invocation = featureFlagsStatus ? Times.Once() : Times.Never(); + + tc.Variables.Add("USE_GIT_SINGLE_THREAD", featureFlagStatusString); + tc.Variables.Add("USE_GIT_LONG_PATHS", featureFlagStatusString); + tc.Variables.Add("FIX_POSSIBLE_GIT_OUT_OF_MEMORY_PROBLEM", featureFlagStatusString); + + GitSourceProvider gitSourceProvider = new ExternalGitSourceProvider(); + gitSourceProvider.SetGitFeatureFlagsConfiguration(tc, gitCliManagerMock.Object, repositoryPath); + + // Assert. + gitCliManagerMock.Verify(x => x.GitConfig(tc, repositoryPath, "pack.threads", "1"), invocation); + gitCliManagerMock.Verify(x => x.GitConfig(tc, repositoryPath, "core.longpaths", "true"), invocation); + gitCliManagerMock.Verify(x => x.GitConfig(tc, repositoryPath, "http.postBuffer", "524288000"), invocation); + } +} diff --git a/src/Test/L0/Plugin/TestGitSourceProvider/MockAgentTaskPluginExecutionContext.cs b/src/Test/L0/Plugin/TestGitSourceProvider/MockAgentTaskPluginExecutionContext.cs new file mode 100644 index 0000000000..f3126bceaa --- /dev/null +++ b/src/Test/L0/Plugin/TestGitSourceProvider/MockAgentTaskPluginExecutionContext.cs @@ -0,0 +1,9 @@ +using Agent.Sdk; + +namespace Test.L0.Plugin.TestGitSourceProvider +{ + public class MockAgentTaskPluginExecutionContext : AgentTaskPluginExecutionContext + { + public MockAgentTaskPluginExecutionContext(ITraceWriter trace) : base(trace) { } + } +} diff --git a/src/Test/L0/Worker/Build/GitCommandManagerL0.cs b/src/Test/L0/Worker/Build/GitCommandManagerL0.cs new file mode 100644 index 0000000000..4a4ca5dc24 --- /dev/null +++ b/src/Test/L0/Worker/Build/GitCommandManagerL0.cs @@ -0,0 +1,66 @@ +using Agent.Sdk; +using Microsoft.VisualStudio.Services.Agent.Worker; +using Microsoft.VisualStudio.Services.Agent.Worker.Build; +using Moq; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using Xunit; + +namespace Microsoft.VisualStudio.Services.Agent.Tests.Worker.Build; + +public class TestGitCommandManagerL0 +{ + public static IEnumerable UseNewGitVersionFeatureFlagsData => new List + { + new object[] { true }, + new object[] { false }, + }; + + [Theory] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + [Trait("SkipOn", "darwin")] + [Trait("SkipOn", "linux")] + [MemberData(nameof(UseNewGitVersionFeatureFlagsData))] + public void TestGetInternalGitPaths(bool gitFeatureFlagStatus) + { + using var tc = new TestHostContext(this, $"GitFeatureFlagStatus_{gitFeatureFlagStatus}"); + var trace = tc.GetTrace(); + var executionContext = new Mock(); + + GitCommandManager gitCliManager = new(); + gitCliManager.Initialize(tc); + var (resolvedGitPath, resolvedGitLfsPath) = gitCliManager.GetInternalGitPaths( + executionContext.Object, + gitFeatureFlagStatus); + + string gitPath; + + if (gitFeatureFlagStatus) + { + gitPath = Path.Combine(tc.GetDirectory(WellKnownDirectory.Externals), "ff_git", "cmd", "git.exe"); + } + else + { + gitPath = Path.Combine(tc.GetDirectory(WellKnownDirectory.Externals), "git", "cmd", "git.exe"); + } + + var binPath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); + var rootPath = new DirectoryInfo(binPath).Parent.FullName; + var externalsDirectoryPath = Path.Combine(rootPath, Constants.Path.ExternalsDirectory); + string gitLfsPath; + + if (PlatformUtil.BuiltOnX86) + { + gitLfsPath = Path.Combine(externalsDirectoryPath, "git", "mingw32", "bin", $"git-lfs.exe"); + } + else + { + gitLfsPath = Path.Combine(externalsDirectoryPath, "git", "mingw64", "bin", $"git-lfs.exe"); + } + + Assert.Equal(resolvedGitPath, gitPath); + Assert.Equal(resolvedGitLfsPath, gitLfsPath); + } +} diff --git a/src/Test/L0/Worker/Build/GitSourceProviderL0.cs b/src/Test/L0/Worker/Build/GitSourceProviderL0.cs index e341cbc612..000c879f62 100644 --- a/src/Test/L0/Worker/Build/GitSourceProviderL0.cs +++ b/src/Test/L0/Worker/Build/GitSourceProviderL0.cs @@ -162,6 +162,54 @@ public void GetSourceGitClone() } } + public static IEnumerable FeatureFlagsStatusData => new List + { + new object[] { true }, + new object[] { false }, + }; + + [Theory] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + [Trait("SkipOn", "darwin")] + [Trait("SkipOn", "linux")] + [MemberData(nameof(FeatureFlagsStatusData))] + public void TestSetGitConfiguration(bool featureFlagsStatus) + { + var featureFlagStatusString = featureFlagsStatus.ToString(); + var invocation = featureFlagsStatus ? Times.Once() : Times.Never(); + + using TestHostContext tc = new TestHostContext(this, $"GitFeatureFlagStatus_{featureFlagStatusString}"); + using var trace = tc.GetTrace(); + + // Arrange. + var sourceProviderL0Path = Path.Combine(tc.GetDirectory(WellKnownDirectory.Bin), "SourceProviderL0"); + + var executionContext = GetTestExecutionContext(tc, sourceProviderL0Path, "master", "a596e13f5db8869f44574be0392fb8fe1e790ce4", false); + + var environment = new LocalEnvironment(); + + executionContext + .Setup(x => x.GetScopedEnvironment()) + .Returns(environment); + + environment.SetEnvironmentVariable(Constants.Variables.Agent.UseGitLongPaths, featureFlagStatusString); + environment.SetEnvironmentVariable(Constants.Variables.Agent.UseGitSingleThread, featureFlagStatusString); + environment.SetEnvironmentVariable(Constants.Variables.Agent.FixPossibleGitOutOfMemoryProblem, featureFlagStatusString); + + var gitCommandManager = GetDefaultGitCommandMock(); + + GitSourceProvider gitSourceProvider = new ExternalGitSourceProvider(); + + // Act. + gitSourceProvider.SetGitFeatureFlagsConfiguration(executionContext.Object, gitCommandManager.Object, sourceProviderL0Path); + + // Assert. + gitCommandManager.Verify(x => x.GitConfig(executionContext.Object, sourceProviderL0Path, "pack.threads", "1"), invocation); + gitCommandManager.Verify(x => x.GitConfig(executionContext.Object, sourceProviderL0Path, "core.packedgitlimit", "256m"), invocation); + gitCommandManager.Verify(x => x.GitConfig(executionContext.Object, sourceProviderL0Path, "core.longpaths", "true"), invocation); + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")]