/// <summary>
/// Construct a WorkItem for a particular test.
/// </summary>
/// <param name="test">The test that the WorkItem will run</param>
public WorkItem(Test test, FinallyDelegate finallyDelegate)
{
_test = test;
testResult = test.MakeTestResult();
_state = WorkItemState.Ready;
finD = finallyDelegate;
}
/// <summary>
/// Construct a WorkItem for a particular test.
/// </summary>
/// <param name="test">The test that the WorkItem will run</param>
/// <param name="context">The TestExecutionContext in which it will run</param>
public WorkItem(Test test, TestExecutionContext context)
{
_test = test;
_testResult = test.MakeTestResult();
_state = WorkItemState.Ready;
_context = context;
}
public void Init()
{
base.InitBase();
project = new MockCSharpProject();
MockBuildProjectBeforeTestRun buildProject = new MockBuildProjectBeforeTestRun();
context.MockBuildProjectFactory.AddBuildProjectBeforeTestRun(buildProject);
classToTest = MockClass.CreateMockClassWithoutAnyAttributes();
classToTest.SetDotNetName("MyTestClass");
treeView = new MockTestTreeView();
treeView.SelectedProject = project;
treeView.SelectedClass = classToTest;
runTestCommand.Owner = treeView;
runTestCommand.Run();
buildProject.FireBuildCompleteEvent();
TestResult result = new TestResult("MyTestClass");
result.ResultType = TestResultType.Success;
context.MockTestResultsMonitor.FireTestFinishedEvent(result);
runTestCommand.CallTestsCompleted();
}
public void TestFinished(TestResult test)
{
foreach (var unitTestRunnerCallback in m_CallbackList)
{
unitTestRunnerCallback.TestFinished(test);
}
}
private static TestResult RunTest(object instance, MethodBase method)
{
var result = new TestResult { Name = method.Name };
var sw = new Stopwatch();
sw.Start();
try
{
method.Invoke(instance, null);
result.WasSuccessful = true;
}
catch (Exception ex)
{
result.WasSuccessful = false;
result.Message = ex.ToString();
}
finally
{
sw.Stop();
result.Duration = sw.ElapsedMilliseconds;
}
return result;
}
public void OnSpecificationEnd(SpecificationInfo specification, Result result)
{
var formatter = resultFormatterFactory.GetResultFormatterFor(result);
testListener.WriteLine(formatter.FormatResult(specification), Category.Output);
var testResult = new TestResult();
testResult.Name = specification.Name;
if (result.Passed)
testResult.State = TestState.Passed;
else if (result.Status == Status.Ignored)
{
testResult.State = TestState.Ignored;
testResult.Message = "Ignored";
}
else if (result.Status == Status.NotImplemented)
{
testResult.State = TestState.Ignored;
testResult.Message = "Not Implemented";
}
else
{
testResult.State = TestState.Failed;
if (result.Exception != null)
testResult.StackTrace = result.Exception.ToString();
}
testResults.Add(testResult);
}
public void TestFinished(TestResult result)
{
_testRunInfo.Running = false;
_testRunInfo.Passed = result.IsSuccess;
_testRunInfo.TestResult = result;
RunOnUiThread(() => _detailsView.Description = DefaultDescription);
}
private bool CanAccessUrl(string url)
{
var key = GetHostFromUrl(url);
lock (_resultLock)
{
TestResult result;
if (_results.TryGetValue(url, out result))
{
var timespan = DateTime.UtcNow - result.Date;
if (timespan <= TimeSpan.FromMinutes(120))
{
return result.Success;
}
}
}
var canAccess = CanAccessUrlInternal(url);
lock (_resultLock)
{
_results[key] = new TestResult
{
Success = canAccess,
Date = DateTime.UtcNow
};
}
return canAccess;
}
public void TestPacketTimeouts()
{
try
{
BrainCloudClient.Get().Initialize(_serverUrl + "unitTestFail", _secret, _appId, _version);
BrainCloudClient.Get ().EnableLogging(true);
BrainCloudClient.Get ().SetPacketTimeouts(new List<int> {3, 3, 3});
DateTime timeStart = DateTime.Now;
TestResult tr = new TestResult();
tr.SetTimeToWaitSecs(120);
BrainCloudClient.Get().AuthenticationService.AuthenticateUniversal("abc", "abc", true, tr.ApiSuccess, tr.ApiError);
tr.RunExpectFail(StatusCodes.CLIENT_NETWORK_ERROR, ReasonCodes.CLIENT_NETWORK_ERROR_TIMEOUT);
DateTime timeEnd = DateTime.Now;
TimeSpan delta = timeEnd.Subtract(timeStart);
if (delta < TimeSpan.FromSeconds (8) && delta > TimeSpan.FromSeconds(15))
{
Console.WriteLine("Failed timing check - took " + delta.TotalSeconds + " seconds");
Assert.Fail ();
}
}
finally
{
// reset to defaults
BrainCloudClient.Get ().SetPacketTimeoutsToDefault();
}
}
void EventListener.TestFinished(TestResult result)
{
try {
var node = suite.FindNodeByOriginalName(result.FullName);
switch(result.ResultState) {
case ResultState.Success:
node.RunState = UniTestRunState.Passed;
break;
case ResultState.Skipped:
case ResultState.Ignored:
case ResultState.Inconclusive:
node.RunState = UniTestRunState.Pending;
break;
case ResultState.Failure:
case ResultState.Error:
node.RunState = UniTestRunState.Failed;
break;
}
}
catch(System.Exception e) {
Debug.LogError("Error with " + result.FullName);
Debug.LogError(e.Message);
Debug.LogError(e.StackTrace);
}
// Debug.Log("Test finished " + result.FullName);
// Debug.Log("Result of finished test: " + result.ResultState);
}
public void TestStarted (TestResult test)
{
foreach (var unitTestRunnerCallback in callbackList)
{
unitTestRunnerCallback.TestStarted(test);
}
}
public static MarkValue UpdateToTestResult(MarkValue current, TestResult result)
{
MarkValue updated = current;
switch (result)
{
case TestResult.Passed:
if (current == MarkValue.ToLearn)
updated = MarkValue.AlmostLearned;
else
updated = Upgrade(current);
break;
case TestResult.PassedWithHints:
updated = MarkValue.WithHints;
break;
case TestResult.Failed:
updated = MarkValue.ToLearn;
break;
default:
Debug.WriteLine("Unexpected test result {0} in Mark::updateToTestResult", result);
break;
}
Debug.WriteLine("Mark changed from {0} to {1}", current, updated);
return updated;
}
public static void SubmitResult(Uri testVaultServer, string session, string project, string buildname, string testgroup, string testname, TestOutcome outcome, bool personal)
{
try
{
using ( var client = new WebClient() ){
client.BaseAddress = testVaultServer.ToString();
client.CachePolicy = new System.Net.Cache.RequestCachePolicy( System.Net.Cache.RequestCacheLevel.NoCacheNoStore );
client.Headers.Add("Content-Type", "text/xml");
var result = new TestResult()
{
Group = new TestGroup() { Name = testgroup, Project = new TestProject() { Project = project } },
Name = testname,
Outcome = outcome,
TestSession = session,
IsPersonal = personal,
BuildID = buildname,
};
var xc = new XmlSerializer(result.GetType());
var io = new System.IO.MemoryStream();
xc.Serialize( io, result );
client.UploadData( testVaultServer.ToString(), io.ToArray() );
}
} catch ( Exception e )
{
Console.Error.WriteLine( e );
}
}
internal TestResult ProcessSingleTestNode(XmlNode theResultNode)
{
TestResult testResult = new TestResult();
//Get the name and outcome of the test
Regex match = new Regex("Environment='[A-Za-z]*'");
Match environmentName = match.Match(theResultNode.InnerText);
if (environmentName != Match.Empty)
{
string[] splitString = environmentName.ToString().Split('=');
if (splitString.Length == 2)
{
testResult.ExecutionEnvironment = splitString[1].Replace("'", "");
}
}
testResult.TestName = theResultNode.Attributes["testName"].Value;
testResult.setOutcome(theResultNode.Attributes["outcome"].Value);
//Capture the end time and duration
testResult.ExecutionCompletionDateTime = Convert.ToDateTime(theResultNode.Attributes["endTime"].Value);
testResult.ExecutionDuration = TimeSpan.Parse(theResultNode.Attributes["duration"].Value);
//Build the first part of the comment
StringBuilder comment = new StringBuilder();
comment.Append("\r\n");
comment.Append("Test " + theResultNode.Attributes["outcome"].Value.ToUpper());
comment.Append(" on " + theResultNode.Attributes["computerName"].Value);
comment.Append(" at " + testResult.ExecutionCompletionDateTime.ToShortDateString() + " " + testResult.ExecutionCompletionDateTime.ToShortTimeString());
comment.Append("\r\n");
comment.Append("Duration: " + testResult.ExecutionDuration.Minutes + " min " + testResult.ExecutionDuration.Seconds + " sec");
testResult.ExecutionComments = comment.ToString();
testResult.ComputerName = theResultNode.Attributes["computerName"].Value;
return testResult;
}
private void Visit(TestResult result)
{
if (result.Test is TestSuite)
{
if (result.Results != null)
foreach (TestResult r in result.Results)
Visit(r);
return;
}
// We only count non-suites
testCount++;
switch (result.ResultState)
{
case ResultState.NotRun:
notRunCount++;
break;
case ResultState.Error:
errorCount++;
break;
case ResultState.Failure:
failureCount++;
break;
default:
break;
}
}
public static TestResult ToVsTestResult(this Models.TestCase test)
{
var testCase = test.ToVsTestCase();
TestResult result;
if (test.Passed)
{
result = new TestResult(testCase)
{
Duration = TimeSpan.FromMilliseconds(test.TimeTaken),
DisplayName = testCase.DisplayName,
Outcome = ToVsTestOutcome(true)
};
}
else
{
var failureResult = test.TestResults.FirstOrDefault(x => !x.Passed);
result = new TestResult(testCase)
{
Duration = TimeSpan.FromMilliseconds(test.TimeTaken),
DisplayName = testCase.DisplayName,
ErrorMessage = GetTestFailureMessage(failureResult),
Outcome = ToVsTestOutcome(false)
};
}
return result;
}
public void AddResult (Test test, TestResult result)
{
XmlElement el = document.CreateElement ("Test");
string result_file = GetFilePath (test.ResultFile);
string master_file = GetFilePath (test.MasterFile);
CopyImageToRunDirectory (test_run_dir,result_file);
el.SetAttribute ("Id", test.Id);
el.SetAttribute ("InputFile", test.InputFile);
el.SetAttribute ("ResultFile", MakeRelativePath (result_file));
el.SetAttribute ("MasterFile", Path.GetFullPath (master_file));
el.SetAttribute ("TestResult", result.ToString ());
foreach (string category in test.Categories) {
XmlElement cat = document.CreateElement ("Category");
cat.SetAttribute ("Name", category);
el.AppendChild (cat);
}
runs_node.AppendChild (el);
UpdateRunsNode ();
SaveDocument ();
}
public JsonFeatureWithMetaInfo(FeatureNode featureNodeTreeNode, TestResult result)
{
var jsonMapper = new JsonMapper();
this.Feature = jsonMapper.Map(featureNodeTreeNode.Feature);
this.RelativeFolder = featureNodeTreeNode.RelativePathFromRoot;
this.Result = jsonMapper.Map(result);
}
/**
* Outputs the results of the unit tests.
* Completely overrides the base class method of the same name.
*
* @param result, the result containing the failures to display.
*/
public override void LogResults(TestResult result)
{
// Set results
m_result = result;
// Create xml file
m_file = new System.IO.StreamWriter(m_fileName, false);
if (null != m_file)
{
// Begin xml log
m_file.WriteLine("<?xml version=\"1.0\" encoding=\"utf-8\"?>");
m_file.WriteLine("<SharpUnit>");
// Log summary
LogSummary();
// If results valid
if (null != m_result)
{
// For each failure
foreach (Exception error in m_result.ErrorList)
{
// Log the failure
LogFailure(error);
}
}
// End xml log
m_file.WriteLine("</SharpUnit>");
// Close file
m_file.Close();
m_file = null;
}
}
private static SpecificationResult GenerateSpecificationResult(IAmSpecification instance)
{
instance.Setup();
var specificationResult = new SpecificationResult()
{
Given = instance.GetGiven().ToString(),
When = instance.When().ToString(),
Results = new List<TestResult>()
};
var tests =
instance.GetType().GetMethods().Where(
y => y.GetCustomAttribute<TestAttribute>() != null);
foreach (var test in tests)
{
var testResult = new TestResult() { Description = test.Name.Replace("_", " "), Passed = true };
try
{
test.Invoke(instance, null);
}
catch (Exception)
{
testResult.Passed = false;
}
specificationResult.Results.Add(testResult);
}
return specificationResult;
}
public static TestResult Run(string filePath)
{
var result = new TestResult();
try
{
Process p = new Process();
ProcessStartInfo psi = new ProcessStartInfo(ExecutorPath, filePath);
psi.UseShellExecute = false;
psi.CreateNoWindow = true;
psi.RedirectStandardOutput = true;
p.StartInfo = psi;
p.Start();
var output = p.StandardOutput.ReadToEnd();
p.WaitForExit();
result.ExitCode = p.ExitCode;
result.Output = output;
}
catch (Exception ex)
{
result.ExitCode = -1;
result.Output = ex.StackTrace;
}
return result;
}
public void EndTest(string testId, TestResult testResult)
{
foreach (ILogger logger in Loggers)
{
logger.EndTest(testId, testResult);
}
}
public void Init()
{
base.InitBase();
project = new MockCSharpProject();
MockBuildProjectBeforeTestRun buildProject = new MockBuildProjectBeforeTestRun();
context.MockBuildProjectFactory.AddBuildProjectBeforeTestRun(buildProject);
methodToTest = MockMethod.CreateMockMethodWithoutAnyAttributes();
methodToTest.FullyQualifiedName = "MyTests.MyTestClass.MyTestMethod";
classToTest = methodToTest.DeclaringType as MockClass;
classToTest.SetDotNetName("MyTests.MyTestClass");
treeView = new MockTestTreeView();
treeView.SelectedProject = project;
treeView.SelectedMember = methodToTest;
runTestCommand.Owner = treeView;
runTestCommand.Run();
buildProject.FireBuildCompleteEvent();
context.MockUnitTestWorkbench.MakeSafeThreadAsyncMethodCallsWithArguments = true;
context.MockBuildOptions.ShowErrorListAfterBuild = false;
TestResult result = new TestResult("MyTests.MyTestClass.MyTestMethod");
result.ResultType = TestResultType.Failure;
context.MockTestResultsMonitor.FireTestFinishedEvent(result);
runTestCommand.CallTestsCompleted();
}
public static void PrintSingleTest()
{
var directories = GetBuildDirectories();
var tests = new List<TestResult>();
foreach (var dir in directories)
{
var files = Directory.GetFiles(dir, "junitResult.xml");
if (files.Any())
{
var fileName = files[0];
var iterator = GetTestCasesWithErrors("UserAcceptanceTests.Features.ProductFeature","RefreshOnProductAfterAddingCommentShouldNOTResultInErrorBugFix",fileName);
while (iterator.MoveNext())
{
var failingTest = GetFailingTestName(iterator);
var testResult = new TestResult {BuildName = GetBuildName(dir),TestName = failingTest};
tests.Add(testResult);
}
}
}
foreach (var failingTest in tests.ToList().OrderBy(x=>x.BuildName).Reverse())
{
Console.WriteLine(failingTest);
}
}
public void EWMTestMapping()
{
var testResult = new TestResult()
{
TestID = 1,
ProdName = "Thomson Reuters Eikon for Wealth Management",
ProdVers = "1",
TestCases = new List<TestCase>()
};
testResult.TestCases.Add(new TestCase()
{
ID = "1",
Title = "Browser Version:"
});
testResult.TestCases.Add(new TestCase()
{
ID = "2",
Title = "Operating System:"
});
var list = new List<TestResult> {testResult};
DataMapper.MapStatCodeEWM(testResult);
Assert.IsNotNull(testResult);
}
internal static void LogStackTrace()
{
if (_context != null && _context.Result.Status == TestStatus.Failed)
{
var testname = _context.Test.FullName.Split('.')[2].Split('<')[0];
var methodname = _context.Test.Name;
var _stackFilePath = string.Format("{0}{1}.cs__{2}()__{3}__StackTrace.txt",
_stacktraceDir,
testname,
methodname,
_browser);
var stackTrace = new StackTrace(true);
using (var outfile = new StreamWriter(_stackFilePath, false))
{
outfile.WriteLine(_result.Message);
outfile.WriteLine("");
outfile.WriteLine(stackTrace.ToString());
}
}
_stackTrace = null;
_context = null;
_browser = null;
_result = null;
}
public void AddTestResult(ContentFile contentFile, TestResult testResult)
{
var checksum = contentFile.Checksum;
var storagePath = Path.Combine(_storagePath, checksum);
try
{
if (!FileUtil.EnsureDirectory(storagePath))
{
return;
}
Write(checksum, StorageKind.ExitCode, testResult.ExitCode.ToString());
Write(checksum, StorageKind.AssemblyPath, testResult.AssemblyPath);
Write(checksum, StorageKind.StandardOutput, testResult.StandardOutput);
Write(checksum, StorageKind.ErrorOutput, testResult.ErrorOutput);
Write(checksum, StorageKind.CommandLine, testResult.CommandLine);
Write(checksum, StorageKind.Content, contentFile.Content);
if (!string.IsNullOrEmpty(testResult.ResultsFilePath))
{
File.Copy(testResult.ResultsFilePath, GetStoragePath(checksum, StorageKind.ResultsFile));
}
}
catch (Exception e)
{
// I/O errors are expected and okay here.
Logger.Log($"Failed to log {checksum} {e.Message}");
FileUtil.DeleteDirectory(storagePath);
}
}
private string BuildTitle(TestResult successful)
{
var sb = new StringBuilder();
switch (successful)
{
default:
case TestResult.Inconclusive:
sb.AppendFormat("{0}", "Inconclusive");
break;
case TestResult.Passed:
sb.AppendFormat("{0}", "Successful");
break;
case TestResult.Failed:
sb.AppendFormat("{0}", "Failed");
break;
}
if (!string.IsNullOrEmpty(this.configuration.SystemUnderTestName) &&
!string.IsNullOrEmpty(this.configuration.SystemUnderTestVersion))
{
sb.AppendFormat(" with {0} version {1}", this.configuration.SystemUnderTestName, this.configuration.SystemUnderTestVersion);
}
else if (!string.IsNullOrEmpty(this.configuration.SystemUnderTestName))
{
sb.AppendFormat(" with {0}", this.configuration.SystemUnderTestName);
}
else if (!string.IsNullOrEmpty(this.configuration.SystemUnderTestVersion))
{
sb.AppendFormat(" with version {0}", this.configuration.SystemUnderTestVersion);
}
return sb.ToString();
}
public void ProcessRecordThrowsExpectedExceptionSuccess()
{
// Arrange.
var test = new Test(
"t",
ScriptBlock.Create(string.Empty),
typeof(ArgumentNullException)
);
var testResult = new TestResult("tr", true);
var runtime = MockRepository.GenerateStrictMock<ICommandRuntime>();
runtime.Expect(r => r.WriteObject(testResult));
var cmdlet = MockRepository
.GeneratePartialMock<TestableInvokeTestCmdlet>();
cmdlet.Test = new[] { test };
cmdlet.CommandRuntime = runtime;
cmdlet
.Expect(c => c.InvokeScriptBlock(test.TestScript))
.Throw(new Exception("", new ArgumentNullException()));
cmdlet
.Expect(c => c.CreateTestResult(test.Name, true))
.Return(testResult);
// Act.
cmdlet.DoProcessRecord();
// Assert.
cmdlet.VerifyAllExpectations();
runtime.VerifyAllExpectations();
}
public void AddTestResult(TestResult result)
{
if (result.stateFlag.Equals(TestResultObject.STATE_FLAG_SEVERE)) {
isSevere = true;
}
testResults.Add(result);
}
public override CustomerEventScreeningTestsEntity CreateActualTestResultEntity(TestResult testResult, List <OrderedPair <int, int> > testReadingReadingPairs)
{
var focTestResult = testResult as FocTestResult;
var customerEventScreeningTestsEntity = new CustomerEventScreeningTestsEntity(testResult.Id)
{
TestId = (int)TestType.Foc
};
var resultIntpretations = new List <long>();
var pathwayRecomendations = new List <long>();
var customerEventReadingEntities = new List <CustomerEventReadingEntity>();
if (focTestResult != null)
{
if (testResult.ResultStatus.StateNumber == (int)TestResultStateNumber.Evaluated || testResult.ResultStatus.StateNumber == (int)NewTestResultStateNumber.Evaluated)
{
if (resultIntpretations.Count > 0)
{
testResult.ResultInterpretation = ResultInterpretation.Normal.GetMax(resultIntpretations);
}
if (pathwayRecomendations.Count > 0)
{
testResult.PathwayRecommendation = PathwayRecommendation.None.GetMax(pathwayRecomendations);
}
}
}
customerEventScreeningTestsEntity.CustomerEventReading.AddRange(customerEventReadingEntities);
return(customerEventScreeningTestsEntity);
}
/// <summary>
/// Optional function that is called on test completion and gives an opportunity to create a report
/// </summary>
/// <param name="Result"></param>
/// <param name="Contex"></param>
/// <param name="Build"></param>
public virtual void SubmitToDashboard(TestResult Result, UnrealTestContext Contex, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath)
{
}
protected override void DoOneTimeSetUp(TestResult suiteResult)
{
Console.WriteLine("Extended Fixture SetUp called");
base.DoOneTimeSetUp(suiteResult);
}
/// <summary>
/// Optional function that is called on test completion and gives an opportunity to create a report
/// </summary>
/// <param name="Result"></param>
/// <param name="Contex"></param>
/// <param name="Build"></param>
public virtual void CreateReport(TestResult Result, UnrealTestContext Context, UnrealBuildSource Build, IEnumerable <UnrealRoleArtifacts> Artifacts, string ArtifactPath)
{
}
/// <summary>
/// Called by the test executor to start our test running. After this
/// Test.Status should return InProgress or greater
/// </summary>
/// <returns></returns>
public override bool StartTest(int Pass, int InNumPasses)
{
if (UnrealApp == null)
{
throw new AutomationException("Node already has a null UnrealApp, was PrepareUnrealSession or IsReadyToStart called?");
}
TConfigClass Config = GetCachedConfiguration();
CurrentPass = Pass;
NumPasses = InNumPasses;
// Either use the ArtifactName param or name of this test
string TestFolder = string.IsNullOrEmpty(Context.Options.ArtifactName) ? this.ToString() : Context.Options.ArtifactName;
if (string.IsNullOrEmpty(Context.Options.ArtifactPostfix) == false)
{
TestFolder += "_" + Context.Options.ArtifactPostfix;
}
TestFolder = TestFolder.Replace(" ", "_");
TestFolder = TestFolder.Replace(":", "_");
TestFolder = TestFolder.Replace("|", "_");
TestFolder = TestFolder.Replace(",", "");
ArtifactPath = Path.Combine(Context.Options.LogDir, TestFolder);
// if doing multiple passes, put each in a subdir
if (NumPasses > 1)
{
ArtifactPath = Path.Combine(ArtifactPath, string.Format("Pass_{0}_of_{1}", CurrentPass + 1, NumPasses));
}
// When running with -parallel we could have several identical tests (same test, configurations) in flight so
// we need unique artifact paths. We also don't overwrite dest directories from the build machine for the same
// reason of multiple tests for a build. Really though these should use ArtifactPrefix to save to
// SmokeTest_HighQuality etc
int ArtifactNumericPostfix = 0;
bool ArtifactPathIsTaken = false;
do
{
string PotentialPath = ArtifactPath;
if (ArtifactNumericPostfix > 0)
{
PotentialPath = string.Format("{0}_{1}", ArtifactPath, ArtifactNumericPostfix);
}
ArtifactPathIsTaken = ReservedArtifcactPaths.Contains(PotentialPath) || (CommandUtils.IsBuildMachine && Directory.Exists(PotentialPath));
if (ArtifactPathIsTaken)
{
Log.Info("Directory already exists at {0}", PotentialPath);
ArtifactNumericPostfix++;
}
else
{
ArtifactPath = PotentialPath;
}
} while (ArtifactPathIsTaken);
ReservedArtifcactPaths.Add(ArtifactPath);
// We need to create this directory at the start of the test rather than the end of the test - we are running into instances where multiple A/B tests
// on the same build are seeing the directory as non-existent and thinking it is safe to write to.
Directory.CreateDirectory(ArtifactPath);
// Launch the test
TestInstance = UnrealApp.LaunchSession();
// track the overall session time
if (SessionStartTime == DateTime.MinValue)
{
SessionStartTime = DateTime.Now;
}
if (TestInstance != null)
{
// Update these for the executor
MaxDuration = Config.MaxDuration;
UnrealTestResult = TestResult.Invalid;
MarkTestStarted();
}
return(TestInstance != null);
}
public TestSuiteFinishedEvent(int buildId, [NotNull] string projectName, [NotNull] string versionNumber, TestResult result,
[NotNull] string suiteType, List <TestOutput> testOutputs)
{
if (projectName == null)
{
throw new ArgumentNullException("projectName");
}
if (versionNumber == null)
{
throw new ArgumentNullException("versionNumber");
}
if (suiteType == null)
{
throw new ArgumentNullException("suiteType");
}
_buildId = buildId;
_projectName = projectName;
_versionNumber = versionNumber;
_result = result;
_suiteType = suiteType;
_testOutputs = testOutputs;
}
private void ConnectionReceivedEvent(object sender, EventReceivedEventArgs e)
{
switch (e.Name)
{
case TP.ResultEvent.Name:
var result = (TP.ResultEvent)e.Event;
TestOutcome outcome = TestOutcome.None;
switch (result.outcome)
{
case "passed": outcome = TestOutcome.Passed; break;
case "failed": outcome = TestOutcome.Failed; break;
case "skipped": outcome = TestOutcome.Skipped; break;
}
RecordEnd(
_frameworkHandle,
_curTestResult,
_stdOut.ToString(),
_stdErr.ToString(),
outcome,
result
);
_stdOut.Clear();
_stdErr.Clear();
break;
case TP.StartEvent.Name:
var start = (TP.StartEvent)e.Event;
// Create the TestResult object right away, so that
// StartTime is initialized correctly.
_curTestResult = null;
foreach (var test in GetTestCases())
{
if (test.Key == start.test)
{
_curTestResult = new TestResult(test.Value);
break;
}
}
if (_curTestResult != null)
{
_frameworkHandle.RecordStart(_curTestResult.TestCase);
}
else
{
Warning(Strings.Test_UnexpectedResult.FormatUI(start.classname, start.method));
}
break;
case TP.StdErrEvent.Name:
var err = (TP.StdErrEvent)e.Event;
_stdErr.Append(err.content);
break;
case TP.StdOutEvent.Name:
var outp = (TP.StdOutEvent)e.Event;
_stdOut.Append(outp.content);
break;
case TP.DoneEvent.Name:
_done.Set();
break;
}
}
public override CustomerEventScreeningTestsEntity CreateActualTestResultEntity(TestResult testResult, List <Core.OrderedPair <int, int> > testReadingReadingPairs)
{
var vitaminDTestResult = testResult as VitaminDTestResult;
var customerEventScreeningTestsEntity = new CustomerEventScreeningTestsEntity(testResult.Id)
{
TestId = (int)TestType.VitaminD
};
var customerEventReadingEntities = new List <CustomerEventReadingEntity>();
if (vitaminDTestResult.VitD != null)
{
customerEventReadingEntities.Add(CreateEventReadingEntity(vitaminDTestResult.VitD, (int)ReadingLabels.VitD, testReadingReadingPairs));
}
if (vitaminDTestResult.TechnicallyLimitedbutReadable != null)
{
var customerEventReading = CreateEventReadingEntity(vitaminDTestResult.TechnicallyLimitedbutReadable, (int)ReadingLabels.TechnicallyLimitedbutReadable, testReadingReadingPairs);
customerEventReadingEntities.Add(customerEventReading);
}
if (vitaminDTestResult.RepeatStudy != null)
{
var customerEventReading = CreateEventReadingEntity(vitaminDTestResult.RepeatStudy, (int)ReadingLabels.RepeatStudy, testReadingReadingPairs);
customerEventReadingEntities.Add(customerEventReading);
}
customerEventScreeningTestsEntity.CustomerEventReading.AddRange(customerEventReadingEntities);
return(customerEventScreeningTestsEntity);
}
public void Run(TestConfig config)
{
int count = config.Count;
var res = new TestResult();
config.Result = res;
IDatabase db = config.GetDatabase();
Tests.Assert(db.Root == null);
Root root = new Root();
Tests.AssertDatabaseException(() =>
{ root.idx = db.CreateFieldIndex <RecordFullWithProperty>(new string[] { "NonExistent" }, IndexType.NonUnique); },
DatabaseException.ErrorCode.INDEXED_FIELD_NOT_FOUND);
root.idx = db.CreateFieldIndex <RecordFullWithProperty>(new string[] { "Int64Prop", "StrVal" }, IndexType.Unique);
root.idxNonUnique = db.CreateFieldIndex <RecordFullWithProperty>(new string[] { "Int64Val", "ByteVal" }, IndexType.NonUnique);
db.Root = root;
Tests.Assert(root.idx.IndexedClass == typeof(RecordFullWithProperty));
Tests.Assert(root.idx.KeyField.Name == "Int64Prop");
Tests.Assert(root.idx.KeyFields[1].Name == "StrVal");
Tests.AssertDatabaseException(() =>
{ root.idx.Append(new RecordFullWithProperty(0)); },
DatabaseException.ErrorCode.UNSUPPORTED_INDEX_TYPE);
RecordFullWithProperty rFirst = null;
long firstkey = 0;
foreach (var key in Tests.KeySeq(count))
{
RecordFullWithProperty rec = new RecordFullWithProperty(key);
root.idx.Put(rec);
root.idxNonUnique.Put(rec);
if (rFirst == null)
{
rFirst = rec;
firstkey = key;
}
}
Tests.Assert(root.idx.Count == count);
db.Commit();
Tests.Assert(root.idx.Count == count);
Tests.Assert(rFirst.IsPersistent());
Tests.Assert(root.idx.Contains(rFirst));
Tests.Assert(root.idxNonUnique.Contains(rFirst));
var rTmp = new RecordFullWithProperty(firstkey);
Tests.Assert(!root.idx.Contains(rTmp));
Tests.Assert(!root.idxNonUnique.Contains(rTmp));
RecordFullWithProperty[] recs = root.idx.ToArray();
Tests.Assert(recs.Length == count);
// TODO: figure out why Set() returns null
var removed = root.idx.Set(rTmp);
//Tests.Assert(removed == rFirst);
removed = root.idxNonUnique.Set(rTmp);
//Tests.Assert(removed == rFirst);
long minKey = Int32.MaxValue;
long maxKey = Int32.MinValue;
foreach (var key in Tests.KeySeq(count))
{
String strKey = Convert.ToString(key);
RecordFullWithProperty rec = root.idx.Get(new Key(new Object[] { key, strKey }));
Tests.Assert(rec != null && rec.Int64Val == key && rec.StrVal.Equals(strKey));
if (key < minKey)
{
minKey = key;
}
if (key > maxKey)
{
maxKey = key;
}
}
int n = 0;
string prevStr = "";
long prevInt = minKey;
foreach (RecordFullWithProperty rec in root.idx.Range(new Key(minKey, ""),
new Key(maxKey + 1, "???"),
IterationOrder.AscentOrder))
{
Tests.Assert(rec.Int64Val > prevInt || rec.Int64Val == prevInt && rec.StrVal.CompareTo(prevStr) > 0);
prevStr = rec.StrVal;
prevInt = rec.Int64Val;
n += 1;
}
Tests.Assert(n == count);
n = 0;
prevInt = maxKey + 1;
foreach (RecordFullWithProperty rec in root.idx.Range(new Key(minKey, "", false),
new Key(maxKey + 1, "???", false),
IterationOrder.DescentOrder))
{
Tests.Assert(rec.Int64Val < prevInt || rec.Int64Val == prevInt && rec.StrVal.CompareTo(prevStr) < 0);
prevStr = rec.StrVal;
prevInt = rec.Int64Val;
n += 1;
}
Tests.Assert(n == count);
rFirst = root.idx.ToArray()[0];
Tests.Assert(root.idx.Contains(rFirst));
Tests.Assert(root.idx.Remove(rFirst));
Tests.Assert(!root.idx.Contains(rFirst));
Tests.Assert(!root.idx.Remove(rFirst));
rFirst = root.idxNonUnique.ToArray()[0];
Tests.Assert(root.idxNonUnique.Contains(rFirst));
Tests.Assert(root.idxNonUnique.Remove(rFirst));
Tests.Assert(!root.idxNonUnique.Contains(rFirst));
Tests.Assert(!root.idxNonUnique.Remove(rFirst));
foreach (var o in root.idx.ToArray())
{
long key = o.Int64Val;
String strKey = Convert.ToString(key);
RecordFullWithProperty rec = root.idx.Get(new Key(new Object[] { key, strKey }));
Tests.Assert(rec != null && rec.Int64Val == key && rec.StrVal.Equals(strKey));
Tests.Assert(root.idx.Contains(rec));
root.idx.Remove(rec);
}
Tests.Assert(!root.idx.GetEnumerator().MoveNext());
Tests.Assert(!root.idx.Reverse().GetEnumerator().MoveNext());
db.Close();
}
public void EndTest(TestResult testResult)
{
}
public void CancelledResultIsMarkedAsCancelled()
{
var result = TestResult.CreateCancelledResult(TimeSpan.Zero);
Assert.True(result.IsCancelled, "Cancelled result not marked as cancelled.");
}
internal string GetCorrectAnswer(Stringency stringency, Design design, TestResult testResult, Question question)
{
ParameterSet pset = new ParameterSet(stringency, design, testResult, question);
return(correctAnswers[pset]);
}
public void RecordResult(TestResult result)
{
this.Results.Add(result);
}
public void SetUp()
{
result = new TestCaseResult(null);
}
private static void AddReasonElement(this XmlNode parent, TestResult result)
{
parent.AddElement("reason").AddElementWithCDataSection("message", result.Message);
}
private void RunTests_Combine(IEnumerable <TestCase> tests)
{
if (!tests.Any())
{
return; // Sanity check
}
List <TestCase> groupedtests = new List <TestCase>();
List <TestCase> singledtests = new List <TestCase>();
List <TestCase> remainingtests = new List <TestCase>();
List <TestCase> retrytests = new List <TestCase>();
Catch2Interface.TestCaseGroup testcasegroup = new Catch2Interface.TestCaseGroup();
testcasegroup.Source = tests.First().Source;
LogDebug(TestMessageLevel.Informational, $"Start Grouping tests for {testcasegroup.Source}");
// Select tests with the same source
foreach (var test in tests)
{
if (testcasegroup.Source != test.Source)
{
remainingtests.Add(test);
continue;
}
if (_executor.CanExecuteCombined(test.DisplayName, SharedUtils.GetTags(test)))
{
LogDebug(TestMessageLevel.Informational, $"Add to group: {test.DisplayName}");
testcasegroup.Names.Add(test.DisplayName);
_frameworkHandle.RecordStart(test); // Indicate in the GUI test is running
groupedtests.Add(test);
}
else
{
singledtests.Add(test);
}
}
// Log sort result
LogDebug(TestMessageLevel.Informational, $"Grouped/Singled/Remaining testcase count: {groupedtests.Count}/{singledtests.Count}/{remainingtests.Count}");
// Check if source actually exists
if (!File.Exists(testcasegroup.Source))
{
LogVerbose(TestMessageLevel.Informational, $"Test executable not found: {testcasegroup.Source}");
SkipTests(groupedtests);
}
// Run tests
if (_runContext.IsBeingDebugged)
{
string caselistfilename = _executor.MakeCaselistFilename(testcasegroup.Source);
// Prepare testcase list file
_executor.CreateTestcaseListFile(testcasegroup, caselistfilename);
LogVerbose(TestMessageLevel.Informational, "Start debug run.");
_frameworkHandle
.LaunchProcessWithDebuggerAttached(testcasegroup.Source
, _executor.WorkingDirectory(testcasegroup.Source)
, _executor.GenerateCommandlineArguments_Combined_Dbg(caselistfilename)
, null);
// Do not process output in Debug mode
foreach (var test in groupedtests)
{
TestResult result = new TestResult(test);
result.Outcome = TestOutcome.None;
_frameworkHandle.RecordResult(result);
}
return;
}
LogVerbose(TestMessageLevel.Informational, $"Run {testcasegroup.Names.Count} grouped testcases.");
var testresults = _executor.Run(testcasegroup);
if (!string.IsNullOrEmpty(_executor.Log))
{
LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}");
}
// Process results
LogDebug(TestMessageLevel.Informational, $"Testcase result count: {testresults.TestResults.Count}");
foreach (var test in groupedtests)
{
var testresult = testresults.FindTestResult(test.DisplayName);
LogDebug(TestMessageLevel.Informational, $"Processed testcase: {test.DisplayName}");
TestResult result = new TestResult(test);
if (testresult == null)
{
if (testresults.TimedOut)
{
LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}");
result.Outcome = TestOutcome.Skipped; // When test result not found, probably a timeout occured and the test was skipped as a result.
result.ErrorMessage = "Timeout of combined testcase execution.";
_frameworkHandle.RecordResult(result);
}
else if (testresults.IsPartialOutput)
{
LogDebug(TestMessageLevel.Informational, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Looks like it was caused by a previous test crashing the test executable. Adding it to the retry list for another combined test execution run.");
retrytests.Add(test);
}
else
{
LogNormal(TestMessageLevel.Warning, $"Combined testcase result not found for: {test.DisplayName}{Environment.NewLine}Trying again by running it in isolation, i.e., not combined with other test cases. To prevent this try updating to a later version of Catch2 or changing the test case name.");
singledtests.Add(test);
}
}
else
{
RecordTestResult(result, testresult);
}
}
if (retrytests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process retry tests (count: {retrytests.Count})");
RunTests_Combine(retrytests);
}
if (singledtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process singled tests (count: {singledtests.Count})");
RunTests_Single(singledtests);
}
if (remainingtests.Count > 0)
{
LogDebug(TestMessageLevel.Informational, $"Process remaining tests (count: {remainingtests.Count})");
RunTests_Combine(remainingtests);
}
}
protected StandardTestSuiteForScenarioOutlines(string resultsFileName, bool treatInconclusiveAsFailed = false)
: base(resultsFileName)
{
this.valueForInconclusive = treatInconclusiveAsFailed ? TestResult.Failed : TestResult.Inconclusive;
}
private bool SaveNewTestResult(TestResult currentTestResult, long customerId, long eventId, ref TestResult synchronizedTestResult)
{
if (currentTestResult != null && currentTestResult.ResultStatus != null && currentTestResult.ResultStatus.StateNumber < (int)NewTestResultStateNumber.Evaluated)
{
TestResult existingTestResult = null;
lock (CentralLocker.Locker)
{
existingTestResult = this.GetTestResults(customerId, eventId, currentTestResult.IsNewResultFlow);
}
// TODO: need to throw an exception that records are already being audited. Will not be updated.
if (existingTestResult != null && existingTestResult.ResultStatus != null && existingTestResult.ResultStatus.StateNumber >= (int)NewTestResultStateNumber.PreAudit &&
!(existingTestResult.ResultStatus.StateNumber == (int)NewTestResultStateNumber.PreAudit && existingTestResult.ResultStatus.Status == TestResultStatus.Incomplete))
{
return(true);
}
synchronizedTestResult = _testResultSynchronizationService.SynchronizeTestResult(existingTestResult, currentTestResult, currentTestResult.IsNewResultFlow);
var compareToResultReadings = GetAllReadings((int)TestType.DPN);
synchronizedTestResult = _testResultSynchronizationService.IsTestIncomplete(compareToResultReadings, synchronizedTestResult, currentTestResult.IsNewResultFlow);
synchronizedTestResult = _testResultSynchronizationService.GetManualEntryUploadStatus(compareToResultReadings, synchronizedTestResult, currentTestResult.IsNewResultFlow);
}
else
{
if (currentTestResult.ResultStatus.StateNumber == (int)NewTestResultStateNumber.Evaluated)
{
TestResult existingTestResult = null;
lock (CentralLocker.Locker)
{
existingTestResult = GetTestResults(customerId, eventId, currentTestResult.IsNewResultFlow);
}
synchronizedTestResult = _testResultSynchronizationService.SynchronizePhysicianEvaluation(currentTestResult, existingTestResult);
}
else
{
synchronizedTestResult = currentTestResult;
}
}
return(false);
}
public void TestFinished(TestResult testResult)
{
if (testResult.Test.ClassName != null && !describes.Contains(testResult.Test.ClassName))
{
Display.Write("describe", testResult.Test.ClassName);
describes.Add(testResult.Test.ClassName);
}
if (String.IsNullOrEmpty(testResult.Description))
{
if (testResult.Test.MethodName != null)
{
Display.Write("IT", testResult.Test.MethodName);
}
}
else
{
Display.Write("IT", testResult.Description);
}
foreach (string category in testResult.Test.Categories)
{
Display.Prop("tag", category);
}
foreach (string key in testResult.Test.Properties.Keys)
{
if (!key.StartsWith("_", StringComparison.InvariantCulture))
{
Display.Prop(key, testResult.Test.Properties[key].ToString());
}
}
string stackTrace = testResult.StackTrace != null?StackTraceFilter.Filter(testResult.StackTrace).Trim() : null;
if (String.IsNullOrEmpty(stackTrace))
{
stackTrace = testResult.StackTrace;
}
Console.Write(outputWriter.ToString());
if (testResult.IsSuccess)
{
Display.Passed(testResult.Message == null ? "Test Passed" : testResult.Message);
testRunCount++;
}
else if (testResult.IsFailure)
{
testRunCount++;
failureCount++;
if (testResult.Results != null)
{
Console.WriteLine(testResult.Results.Count);
}
if (testResult.AssertCount > 1)
{
Display.Passed(String.Format("{0} passed", AssertionsName(testResult.AssertCount - 1)));
}
Display.Failed(testResult.Message);
//Display.Serialize(testResult, label: "-Test Result Info");
Display.Log(testResult.StackTrace, label: "-Stack Trace");
}
else if (testResult.IsError)
{
Display.Error(testResult.Message);
//Display.Serialize(testResult, label: "-Test Result Info");
Display.Log(stackTrace, mode: "ESC", label: "Stack Trace");
}
Console.WriteLine(String.Format("<COMPLETEDIN::>{0}", testResult.Time));
currentTestName = string.Empty;
}
// Adds a test-suite or test-case element, with result info
private static void AddResult(this XmlNode parent, TestResult result, bool recursive)
{
var thisNode = parent.AddTest(result.Test, false);
string status = GetTranslatedResultState(result.ResultState);
var colon = status.IndexOf(':');
string label = null;
if (colon > 0)
{
label = status.Substring(colon + 1);
status = status.Substring(0, colon);
}
thisNode.AddAttribute("result", status);
if (label != null)
{
thisNode.AddAttribute("label", label);
}
// TODO: Check values are the same between V2 and V3
if (result.FailureSite != FailureSite.Test)
{
parent.AddAttribute("site", result.FailureSite.ToString());
}
//start-time not available
//end-time not available
thisNode.AddAttribute("duration", result.Time.ToString("0.000000", NumberFormatInfo.InvariantInfo));
if (result.Test.IsSuite)
{
// TODO: These values all should be calculated
thisNode.AddAttribute("total", result.Test.TestCount.ToString());
thisNode.AddAttribute("passed", "0");
thisNode.AddAttribute("failed", "0");
thisNode.AddAttribute("inconclusive", "0");
thisNode.AddAttribute("skipped", "0");
}
thisNode.AddAttribute("asserts", result.AssertCount.ToString());
if (status == "Failed")
{
thisNode.AddFailureElement(result);
}
else if (result.Message != null)
{
thisNode.AddReasonElement(result);
}
//OutputElement not available
if (recursive && result.HasResults)
{
foreach (TestResult child in result.Results)
{
thisNode.AddResult(child, recursive);
}
}
}
public void SuiteFinished(TestResult suiteResult)
{
// Console.WriteLine (unhandledExceptions.Count.ToString());
// Console.WriteLine (suiteResult.IsFailure.ToString ());
// Console.WriteLine (suiteResult.StackTrace);
}
public void AddTestResult(TestResult result)
{
_context.Add(result);
_context.SaveChanges();
}
void EventListener.SuiteFinished(TestResult result)
{
this.FireSuiteFinished(result);
}
public void RunFinished(TestResult result)
{
}
/// <summary>
/// Constructor.
/// </summary>
public TestResultTableEntry(TestResult testResult, Func <TestResult, string> testUrlBuilder)
{
Name = testResult.TestName;
Result = testResult.GetTestStatusHtml(alwaysBold: false);
ResultLink = GetLink(testUrlBuilder(testResult), "Result", preventWrapping: true);
}
void EventListener.TestFinished(TestResult result)
{
this.FireTestFinished(result);
}
protected override void DoOneTimeTearDown(TestResult suiteResult)
{
base.DoOneTimeTearDown(suiteResult);
Console.WriteLine("Extended Fixture TearDown called");
}
void EventListener.RunFinished(TestResult result)
{
this.FireRunFinished(result);
}
public void RunFinished(TestResult result)
{
s.Close();
}
internal GumbelFit(ColumnVector parameters, SymmetricMatrix covariances, TestResult test) : base(parameters, covariances, test)
{
this.distribution = new GumbelDistribution(parameters[0], parameters[1]);
}
