Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Code exploration. #5056

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Code exploration. #5056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ Written for .NET Core in C#.
|![Win-x86](docs/res/win_med.png) **Windows x86**|[![Build & Test][win-x86-build-badge]][build]|
|![Win-arm64](docs/res/win_med.png) **Windows ARM64**|[![Build & Test][win-arm64-build-badge]][build]|
|![macOS](docs/res/apple_med.png) **macOS**|[![Build & Test][macOS-build-badge]][build]|
|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|
|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|

[win-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x64)
[win-x86-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x86)
[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(ARM64)
[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(arm64)
[macOS-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=macOS%20(x64)
[linux-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(x64)
[linux-arm-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(ARM)
Expand Down
10 changes: 9 additions & 1 deletion src/Agent.Sdk/Agent.Sdk.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,12 @@
<PackageReference Include="System.Security.Cryptography.Xml" Version="6.0.1" />
<PackageReference Include="Azure.Core" Version="1.44.1" />
</ItemGroup>
</Project>

<ItemGroup>
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\*.cs" />
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PotentialSecurityKeys\*.cs" />
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PreciselyClassifiedSecurityKeys\*.cs" />
</ItemGroup>


</Project>
11 changes: 7 additions & 4 deletions src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//using Microsoft.TeamFoundation.DistributedTask.Logging;
using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
using System;
using Microsoft.Security.Utilities;

using ISecretMaskerTfs = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;

namespace Agent.Sdk.SecretMasking
{
/// <summary>
/// Extended ISecretMasker interface that is adding support of logging secret masker methods
/// </summary>
public interface ILoggedSecretMasker : ISecretMasker
public interface ILoggedSecretMasker : ISecretMasker, ISecretMaskerTfs
{
static int MinSecretLengthLimit { get; }
static int MinimumSecretLength { get; }

void AddRegex(String pattern, string origin);
void AddValue(String value, string origin);
void AddValueEncoder(ValueEncoder encoder, string origin);
void AddValueEncoder(LiteralEncoder encoder, string origin);
void SetTrace(ITraceWriter trace);
new string MaskSecrets(string input);
}
}
60 changes: 60 additions & 0 deletions src/Agent.Sdk/SecretMasking/LiteralEncoders.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//using Microsoft.TeamFoundation.DistributedTask.Logging;
using Newtonsoft.Json;

using System;
using System.Text;

namespace Agent.Sdk.SecretMasking
{
public static class LiteralEncoders
{
public static String JsonStringEscape(String value)
{
// Convert to a JSON string and then remove the leading/trailing double-quote.
String jsonString = JsonConvert.ToString(value);
String jsonEscapedValue = jsonString.Substring(startIndex: 1, length: jsonString.Length - 2);
return jsonEscapedValue;
}

public static String BackslashEscape(String value)
{
return value.Replace(@"\\", @"\").Replace(@"\'", @"'").Replace(@"\""", @"""").Replace(@"\t", "\t");
}

public static String UriDataEscape(String value)
{
return UriDataEscape(value, 65519);
}

internal static String UriDataEscape(
String value,
Int32 maxSegmentSize)
{
if (value.Length <= maxSegmentSize)
{
return Uri.EscapeDataString(value);
}

// Workaround size limitation in Uri.EscapeDataString
var result = new StringBuilder();
var i = 0;
do
{
var length = Math.Min(value.Length - i, maxSegmentSize);

if (Char.IsHighSurrogate(value[i + length - 1]) && length > 1)
{
length--;
}

result.Append(Uri.EscapeDataString(value.Substring(i, length)));
i += length;
}
while (i < value.Length);

return result.ToString();
}
}
}
57 changes: 43 additions & 14 deletions src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
using Microsoft.Security.Utilities;
using System.Collections.Generic;
using System;

namespace Agent.Sdk.SecretMasking
{
Expand All @@ -12,7 +13,7 @@ namespace Agent.Sdk.SecretMasking
/// </summary>
public class LoggedSecretMasker : ILoggedSecretMasker
{
private ISecretMasker _secretMasker;
private SecretMasker _secretMasker;
private ITraceWriter _trace;


Expand All @@ -21,7 +22,7 @@ private void Trace(string msg)
this._trace?.Info(msg);
}

public LoggedSecretMasker(ISecretMasker secretMasker)
public LoggedSecretMasker(SecretMasker secretMasker)
{
this._secretMasker = secretMasker;
}
Expand Down Expand Up @@ -54,7 +55,6 @@ public void AddValue(string value, string origin)
}
public void AddRegex(string pattern)
{
this._secretMasker.AddRegex(pattern);
}

/// <summary>
Expand Down Expand Up @@ -82,30 +82,30 @@ public int MinSecretLength
{
get
{
return _secretMasker.MinSecretLength;
return _secretMasker.MinimumSecretLength;
}
set
{
if (value > MinSecretLengthLimit)
{
_secretMasker.MinSecretLength = MinSecretLengthLimit;
_secretMasker.MinimumSecretLength = MinSecretLengthLimit;
}
else
{
_secretMasker.MinSecretLength = value;
_secretMasker.MinimumSecretLength = value;
}
}
}

public void RemoveShortSecretsFromDictionary()
{
this._trace?.Info("Removing short secrets from masking dictionary");
_secretMasker.RemoveShortSecretsFromDictionary();
_secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
}

public void AddValueEncoder(ValueEncoder encoder)
public void AddValueEncoder(LiteralEncoder encoder)
{
this._secretMasker.AddValueEncoder(encoder);
this._secretMasker.AddLiteralEncoder(encoder);
}


Expand All @@ -114,7 +114,7 @@ public void AddValueEncoder(ValueEncoder encoder)
/// </summary>
/// <param name="encoder"></param>
/// <param name="origin"></param>
public void AddValueEncoder(ValueEncoder encoder, string origin)
public void AddValueEncoder(LiteralEncoder encoder, string origin)
{
this.Trace($"Setting up value for origin: {origin}");
this.Trace($"Length: {encoder.ToString().Length}.");
Expand All @@ -129,14 +129,43 @@ public void AddValueEncoder(ValueEncoder encoder, string origin)

public ISecretMasker Clone()
{
return new LoggedSecretMasker(this._secretMasker.Clone());
return null;
//return new LoggedSecretMasker(this._secretMasker.Clone());
}

public string MaskSecrets(string input)
{
return this._secretMasker.MaskSecrets(input);
}

ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone();
public IEnumerable<Detection> DetectSecrets(string input)
{
return this._secretMasker.DetectSecrets(input);
}

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
this._secretMasker?.Dispose();
this._secretMasker = null;
}
}

public void AddValueEncoder(ValueEncoder encoder)
{
//return this.AddValueEncoder((LiteralEncoder)encoder)
}

Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker.Clone()
{
// WRONG: should do this return new LoggedSecretMasker(this._secretMasker.Clone());
return new LoggedSecretMasker(this._secretMasker);
}
}
}
2 changes: 1 addition & 1 deletion src/Agent.Worker/ContainerOperationProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ private async Task<string> GetAcrPasswordFromAADToken(IExecutionContext executio
}

// Mark retrieved password as secret
HostContext.SecretMasker.AddValue(AcrPassword);
HostContext.SecretMasker.AddValue(AcrPassword, origin: null);

return AcrPassword;
}
Expand Down
8 changes: 0 additions & 8 deletions src/Agent.Worker/ExecutionContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,14 +567,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation
PrependPath = new List<string>();

var minSecretLen = AgentKnobs.MaskedSecretMinLength.GetValue(this).AsInt();
HostContext.SecretMasker.MinSecretLength = minSecretLen;

if (HostContext.SecretMasker.MinSecretLength < minSecretLen)
{
warnings.Add(StringUtil.Loc("MinSecretsLengtLimitWarning", HostContext.SecretMasker.MinSecretLength));
}

HostContext.SecretMasker.RemoveShortSecretsFromDictionary();

// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
Expand Down
10 changes: 6 additions & 4 deletions src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public class LegacyTestRunDataPublisher : AgentService, ILegacyTestRunDataPublis
private int _runCounter = 0;
private IFeatureFlagService _featureFlagService;
private bool _calculateTestRunSummary;
private bool _isFlakyCheckEnabled;
private string _testRunner;
private ITestResultsServer _testResultsServer;
private TestRunDataPublisherHelper _testRunPublisherHelper;
Expand All @@ -55,7 +54,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
_testResultsServer = HostContext.GetService<ITestResultsServer>();
_testResultsServer.InitializeServer(connection, _executionContext);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
_isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, null, _testRunPublisher, _testResultsServer);
Trace.Leaving();
}
Expand Down Expand Up @@ -210,7 +208,9 @@ private async Task<bool> PublishAllTestResultsToSingleTestRunAsync(List<string>

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
IList<TestRun> publishedRuns = new List<TestRun>();
publishedRuns.Add(updatedRun);
Expand Down Expand Up @@ -313,7 +313,9 @@ private async Task<bool> PublishToNewTestRunPerTestResultFileAsync(List<string>

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
Expand Down
15 changes: 10 additions & 5 deletions src/Agent.Worker/TestResults/TestDataPublisher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ public sealed class TestDataPublisher : AgentService, ITestDataPublisher
private IFeatureFlagService _featureFlagService;
private string _testRunner;
private bool _calculateTestRunSummary;
private bool _isFlakyCheckEnabled;
private TestRunDataPublisherHelper _testRunPublisherHelper;
private ITestResultsServer _testResultsServer;

Expand All @@ -59,8 +58,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
var extensionManager = HostContext.GetService<IExtensionManager>();
_featureFlagService = HostContext.GetService<IFeatureFlagService>();
_featureFlagService.InitializeFeatureService(_executionContext, connection);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
_isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); ;
_parser = (extensionManager.GetExtensions<IParser>()).FirstOrDefault(x => _testRunner.Equals(x.Name, StringComparison.OrdinalIgnoreCase));
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, _testRunPublisher, null, _testResultsServer);
Trace.Leaving();
Expand Down Expand Up @@ -89,6 +86,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

IList<TestRun> publishedRuns = publishtestRunDataTask.Result;

_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);

var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);

// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
Expand All @@ -99,7 +98,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
Expand Down Expand Up @@ -187,6 +188,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

IList<TestRun> publishedRuns = publishtestRunDataTask.Result;

_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);

var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);

// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
Expand All @@ -197,7 +200,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
Expand Down
Loading
Loading