public async Task <bool> PublishAsync(TestRunContext runContext, List <string> testResultFiles, PublishOptions publishOptions, CancellationToken cancellationToken = default(CancellationToken))
{
try
{
TestDataProvider testDataProvider = ParseTestResultsFile(runContext, testResultFiles);
var publishTasks = new List <Task>();
if (testDataProvider != null)
{
var testRunData = testDataProvider.GetTestRunData();
//publishing run level attachment
Task <IList <TestRun> > publishtestRunDataTask = Task.Run(() => _testRunPublisher.PublishTestRunDataAsync(runContext, _projectName, testRunData, publishOptions, cancellationToken));
Task uploadBuildDataAttachmentTask = Task.Run(() => UploadBuildDataAttachment(runContext, testDataProvider.GetBuildData(), cancellationToken));
publishTasks.Add(publishtestRunDataTask);
//publishing build level attachment
publishTasks.Add(uploadBuildDataAttachmentTask);
await Task.WhenAll(publishTasks);
IList <TestRun> publishedRuns = publishtestRunDataTask.Result;
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);
// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
if (_calculateTestRunSummary)
{
TestResultUtils.StoreTestRunSummaryInEnvVar(_executionContext, testRunSummary, _testRunner, "PublishTestResults");
}
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
{
isTestRunOutcomeFailed = runOutcome.Value;
}
}
return(isTestRunOutcomeFailed);
}
return(false);
}
catch (Exception ex)
{
_executionContext.Warning("Failed to publish test run data: " + ex.ToString());
}
return(false);
}
public void InitializePublisher(IExecutionContext context, string projectName, VssConnection connection, string testRunner, bool publishRunLevelAttachments)
{
Trace.Entering();
_executionContext = context;
_projectName = projectName;
_testRunner = testRunner;
_resultReader = GetTestResultReader(_testRunner, publishRunLevelAttachments);
_testRunPublisher = HostContext.GetService <ITestRunPublisher>();
_featureFlagService = HostContext.GetService <IFeatureFlagService>();
_testRunPublisher.InitializePublisher(_executionContext, connection, projectName, _resultReader);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
Trace.Leaving();
}
public async Task <bool> PublishAsync(TestRunContext runContext, List <string> testResultFiles, PublishOptions publishOptions, CancellationToken cancellationToken = default(CancellationToken))
{
try
{
TestDataProvider testDataProvider = ParseTestResultsFile(runContext, testResultFiles);
var publishTasks = new List <Task>();
if (testDataProvider != null)
{
var testRunData = testDataProvider.GetTestRunData();
//publishing run level attachment
publishTasks.Add(Task.Run(() => _testRunPublisher.PublishTestRunDataAsync(runContext, _projectName, testRunData, publishOptions, cancellationToken)));
//publishing build level attachment
publishTasks.Add(Task.Run(() => UploadBuildDataAttachment(runContext, testDataProvider.GetBuildData(), cancellationToken)));
await Task.WhenAll(publishTasks);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
var runOutcome = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);
// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
if (_calculateTestRunSummary)
{
TestResultUtils.StoreTestRunSummaryInEnvVar(_executionContext, testRunSummary, _testRunner, "PublishTestResults");
}
return(runOutcome);
}
return(false);
}
catch (Exception ex)
{
_executionContext.Warning("Failed to publish test run data: " + ex.ToString());
}
return(false);
}
/// <summary>
/// Publish single test run
/// </summary>
private async Task <bool> PublishAllTestResultsToSingleTestRunAsync(List <string> resultFiles, ITestRunPublisher publisher, TestRunContext runContext, string resultReader, string runTitle, int?buildId, CancellationToken cancellationToken)
{
bool isTestRunOutcomeFailed = false;
try
{
//use local time since TestRunData defaults to local times
DateTime minStartDate = DateTime.MaxValue;
DateTime maxCompleteDate = DateTime.MinValue;
DateTime presentTime = DateTime.UtcNow;
bool dateFormatError = false;
TimeSpan totalTestCaseDuration = TimeSpan.Zero;
List <string> runAttachments = new List <string>();
List <TestCaseResultData> runResults = new List <TestCaseResultData>();
TestRunSummary testRunSummary = new TestRunSummary();
//read results from each file
foreach (string resultFile in resultFiles)
{
cancellationToken.ThrowIfCancellationRequested();
//test case results
_executionContext.Debug(StringUtil.Format("Reading test results from file '{0}'", resultFile));
TestRunData resultFileRunData = publisher.ReadResultsFromFile(runContext, resultFile);
isTestRunOutcomeFailed = isTestRunOutcomeFailed || GetTestRunOutcome(resultFileRunData, testRunSummary);
if (resultFileRunData != null)
{
if (resultFileRunData.Results != null && resultFileRunData.Results.Length > 0)
{
try
{
if (string.IsNullOrEmpty(resultFileRunData.StartDate) || string.IsNullOrEmpty(resultFileRunData.CompleteDate))
{
dateFormatError = true;
}
//As per discussion with Manoj(refer bug 565487): Test Run duration time should be minimum Start Time to maximum Completed Time when merging
if (!string.IsNullOrEmpty(resultFileRunData.StartDate))
{
DateTime startDate = DateTime.Parse(resultFileRunData.StartDate, null, DateTimeStyles.RoundtripKind);
minStartDate = minStartDate > startDate ? startDate : minStartDate;
if (!string.IsNullOrEmpty(resultFileRunData.CompleteDate))
{
DateTime endDate = DateTime.Parse(resultFileRunData.CompleteDate, null, DateTimeStyles.RoundtripKind);
maxCompleteDate = maxCompleteDate < endDate ? endDate : maxCompleteDate;
}
}
}
catch (FormatException)
{
_executionContext.Warning(StringUtil.Loc("InvalidDateFormat", resultFile, resultFileRunData.StartDate, resultFileRunData.CompleteDate));
dateFormatError = true;
}
//continue to calculate duration as a fallback for case: if there is issue with format or dates are null or empty
foreach (TestCaseResultData tcResult in resultFileRunData.Results)
{
int durationInMs = Convert.ToInt32(tcResult.DurationInMs);
totalTestCaseDuration = totalTestCaseDuration.Add(TimeSpan.FromMilliseconds(durationInMs));
}
runResults.AddRange(resultFileRunData.Results);
//run attachments
if (resultFileRunData.Attachments != null)
{
runAttachments.AddRange(resultFileRunData.Attachments);
}
}
else
{
_executionContext.Output(StringUtil.Loc("NoResultFound", resultFile));
}
}
else
{
_executionContext.Warning(StringUtil.Loc("InvalidResultFiles", resultFile, resultReader));
}
}
//publish run if there are results.
if (runResults.Count > 0)
{
string runName = string.IsNullOrWhiteSpace(runTitle)
? StringUtil.Format("{0}_TestResults_{1}", _resultReader.Name, buildId)
: runTitle;
if (DateTime.Compare(minStartDate, maxCompleteDate) > 0)
{
_executionContext.Warning(StringUtil.Loc("InvalidCompletedDate", maxCompleteDate, minStartDate));
dateFormatError = true;
}
minStartDate = DateTime.Equals(minStartDate, DateTime.MaxValue) ? presentTime : minStartDate;
maxCompleteDate = dateFormatError || DateTime.Equals(maxCompleteDate, DateTime.MinValue) ? minStartDate.Add(totalTestCaseDuration) : maxCompleteDate;
// create test run
TestRunData testRunData = new TestRunData(
name: runName,
startedDate: minStartDate.ToString("o"),
completedDate: maxCompleteDate.ToString("o"),
state: "InProgress",
isAutomated: true,
buildId: runContext != null ? runContext.BuildId : 0,
buildFlavor: runContext != null ? runContext.Configuration : string.Empty,
buildPlatform: runContext != null ? runContext.Platform : string.Empty,
releaseUri: runContext != null ? runContext.ReleaseUri : null,
releaseEnvironmentUri: runContext != null ? runContext.ReleaseEnvironmentUri : null
);
testRunData.PipelineReference = runContext.PipelineReference;
testRunData.Attachments = runAttachments.ToArray();
testRunData.AddCustomField(_testRunSystemCustomFieldName, runContext.TestRunSystem);
AddTargetBranchInfoToRunCreateModel(testRunData, runContext.TargetBranchName);
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);
TestRun updatedRun = await publisher.EndTestRunAsync(testRunData, testRun.Id, true, _executionContext.CancellationToken);
// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
IList <TestRun> publishedRuns = new List <TestRun>();
publishedRuns.Add(updatedRun);
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
{
isTestRunOutcomeFailed = runOutcome.Value;
}
}
StoreTestRunSummaryInEnvVar(testRunSummary);
}
}
catch (Exception ex) when(!(ex is OperationCanceledException && _executionContext.CancellationToken.IsCancellationRequested))
{
// Not catching all the operationcancelled exceptions, as the pipeline cancellation should cancel the command as well.
// Do not fail the task.
LogPublishTestResultsFailureWarning(ex);
}
return(isTestRunOutcomeFailed);
}