private void OnTestError(ISolutionProjectModel projectViewModel, MsTestCommand command, string errors, bool cancelOnFailures)
{
lock (lockObj)
{
if (projectViewModel.CurrentOperation != null && !string.IsNullOrEmpty(errors))
{
projectViewModel.CurrentOperation.ColorBrush = Brushes.DarkRed;
}
if (!string.IsNullOrEmpty(errors))
{
foreach (MsTestError error in MsTestError.Parse(errors))
{
UnitTestInfo testInfo = command.UnitTests.First(info => info.TestMethodName == error.TestName);
string fileName = error.TestName;
if (testInfo.ProjectFileItem != null && testInfo.ProjectFileItem.GetFile().Exists)
{
fileName = testInfo.ProjectFileItem.GetFile().FullName;
}
Output.WriteError($"{error.TestName}={error.ErrorMessage}", fileName, testInfo.Line, testInfo.Column, null, TaskCategory.User, testInfo.Project.FullPath, File.Exists(fileName));
}
Output.WriteLine(errors);
if (cancelOnFailures)
{
serviceProvider.Get <MainViewModel>().CompleteCancelCommand.Execute(null);
}
}
}
}
/// <summary>
/// Checks that the deck sorts properly
/// after being shuffled
/// </summary>
/// <returns></returns>
public UnitTestInfo DoesDeckSort()
{
int testsToBeRan = 10;
var testInfo = new UnitTestInfo();
Deck testDeck = new Deck();
string[] controlArray = new string[testDeck.DeckOfCards.Length];
testInfo.DidTestPass = true;
for (int i = 0; i < testDeck.DeckOfCards.Length; i++)
{
controlArray[i] = testDeck.DeckOfCards[i].ToString();
}
for (int i = 0; i < testsToBeRan; i++)
{
testDeck.Shuffle();
testDeck.Sort();
for (int x = 0; x < testDeck.DeckOfCards.Length; x++)
{
if (testDeck.DeckOfCards[x].ToString() != controlArray[x])
{
testInfo.DidTestPass = false;
testInfo.TestFailureMessage = "If test 3 worked as it should, deck does not sort properly after shuffling.";
}
}
}
testInfo.MethodName = "DoesDeckSort()";
return(testInfo);
}
public static IList <UnitTestInfo> GetTestMethods(this Project project)
{
var list = new List <UnitTestInfo>();
FileInfo testOutputDll = project.GetOutputFile();
if (!testOutputDll.Exists)
{
return(list);
}
Assembly testAssembly = Assembly.LoadFile(testOutputDll.FullName);
//Type[] types = testAssembly.GetExportedTypes();
Type[] types;
try
{
types = testAssembly.GetTypes();
}
catch (ReflectionTypeLoadException e)
{
types = e.Types;
}
foreach (Type type in types)
{
var type1 = type;
var testClassAttr = Check.TryCatch <Attribute, Exception>(() => type1.GetCustomAttribute(typeof(TestClassAttribute)));
if (testClassAttr != null)
{
var members = Check.TryCatch <MethodInfo[], Exception>(() => type1.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.InvokeMethod));
foreach (MethodInfo member in members)
{
var member1 = member;
Attribute testMethodAttr = Check.TryCatch <Attribute, Exception>(() => member1.GetCustomAttribute(typeof(TestMethodAttribute)));
Attribute ignoreAttr = Check.TryCatch <Attribute, Exception>(() => member1.GetCustomAttribute(typeof(IgnoreAttribute)));
if (testMethodAttr != null && ignoreAttr == null)
{
var unitTestInfo = new UnitTestInfo(testOutputDll.FullName, member, project);
foreach (var t in member.CustomAttributes)
{
if (t.AttributeType.Name == "OwnerAttribute")
{
unitTestInfo.Owner = t.ConstructorArguments[0].Value.ToString();
}
else if (t.AttributeType.Name == "PriorityAttribute")
{
unitTestInfo.Priority = Int32.Parse(t.ConstructorArguments[0].Value.ToString());
}
}
list.Add(unitTestInfo);
}
}
}
}
return(list);
}
/// <summary>
/// Method used in CorrectOrder.
/// Checks that the input deck is sorted
/// in the correct order.
/// </summary>
/// <param name="testDeck"> This is the deck to be checked </param>
/// <param name="testInfo"> This is being updated if the test fails </param>
/// <param name="errorMessage"> The error message to be input into testInfo </param>
/// <returns></returns>
private static UnitTestInfo CheckOrderOfCards(Deck testDeck, UnitTestInfo testInfo, string errorMessage)
{
for (int i = 0; i < testDeck.DeckOfCards.Length; i++)
{
Card curCard = testDeck.DeckOfCards[i];
if (curCard.face != testDeck.faces[i % 13] && curCard.suit != testDeck.suits[i % 13] && curCard.value != (i % 13) + 1)
{
testInfo.TestFailureMessage = errorMessage;
testInfo.DidTestPass = false;
}
}
return(testInfo);
}
/// <summary>
/// Checks that the deck shuffles.
/// </summary>
/// <returns></returns>
public UnitTestInfo DoesDeckShuffle()
{
int testsToBeRan = 1000;
var TestInfo = new UnitTestInfo();
Deck testDeck = new Deck();
string[] oldDeck = new string[testDeck.DeckOfCards.Length];
int similarities = 0;
int closeOnes = 0;
for (int i = 0; i < testsToBeRan; i++)
{
for (int y = 0; y < testDeck.DeckOfCards.Length; y++)
{
oldDeck[y] = testDeck.DeckOfCards[y].ToString();
}
testDeck.Shuffle();
for (int y = 0; y < testDeck.DeckOfCards.Length; y++)
{
if (testDeck.DeckOfCards[y].ToString() == oldDeck[y])
{
similarities++;
}
if (y > 0)
{
if (testDeck.DeckOfCards[y].ToString() == oldDeck[y - 1])
{
closeOnes++;
}
}
if (y < testDeck.DeckOfCards.Length - 1)
{
if (testDeck.DeckOfCards[y].ToString() == oldDeck[y + 1])
{
closeOnes++;
}
}
}
}
if (similarities < ((testsToBeRan * 52) / 13) && closeOnes < (testsToBeRan * 5))
{
TestInfo.DidTestPass = true;
}
else
{
TestInfo.DidTestPass = false;
}
TestInfo.TestFailureMessage = $"{testsToBeRan * 52} cards tested, {similarities} similarities, {closeOnes} close cards.";
TestInfo.MethodName = "DoesDeckShuffle()";
return(TestInfo);
}
/// <summary>
/// Checks that the amount of cards in the deck is correct.
/// </summary>
/// <returns></returns>
public UnitTestInfo DeckContains52Cards()
{
var TestInfo = new UnitTestInfo();
var d = new Deck();
int numCardsInDeck = d.DeckOfCards.Length;
if (numCardsInDeck % 52 == 0)
{
TestInfo.DidTestPass = true;
}
else
{
TestInfo.DidTestPass = false;
TestInfo.TestFailureMessage = "Deck contains " + numCardsInDeck + " cards, while it should be dividable by 52.";
}
TestInfo.MethodName = "DeckContains52Cards()";
return(TestInfo);
}
/// <summary>
/// Checks that all the cards are in their
/// intended position in the stack, first
/// for a new deck and then for a sorted stack.
/// </summary>
/// <returns></returns>
public UnitTestInfo CorrectOrder()
{
Deck testDeck = new Deck();
var testInfo = new UnitTestInfo();
testInfo.DidTestPass = true;
string error1 = "The cards are not in the correct order when the deck is formed.",
error2 = "The cards are not in the correct order after shuffle and sort.";
testInfo = CheckOrderOfCards(testDeck, testInfo, error1);
testDeck.Shuffle();
testDeck.Sort();
if (testInfo.TestFailureMessage != error1)
{
testInfo = CheckOrderOfCards(testDeck, testInfo, error2);
}
testInfo.MethodName = "CorrectOrder()";
return(testInfo);
}
/// <summary>
/// Checks that all cards can be pulled from the stack
/// and that they are assigned properties.
/// </summary>
/// <returns></returns>
public UnitTestInfo PullAllCards()
{
var TestInfo = new UnitTestInfo();
var d = new Deck();
for (int i = 0; i < d.DeckOfCards.Length; i++)
{
Card curCard = d.DealCard();
if (curCard.ToString() != "" && curCard != null)
{
TestInfo.DidTestPass = true;
}
else
{
TestInfo.DidTestPass = false;
TestInfo.TestFailureMessage = $"Card number {i} in the Deck did not pass validation. This card was {curCard.face} of {curCard.suit}";
return(TestInfo);
}
}
TestInfo.MethodName = "PullAllCards()";
return(TestInfo);
}
/// <summary>
/// Starts google test console main function, which in a turn either lists available tests or executes tests
/// </summary>
/// <param name="runTests">true if run tests by default, false if not</param>
/// <param name="args">command line arguments</param>
/// <param name="suits">test suites to use for test discovery and execution</param>
/// <returns>return true if command arguments were handled, false if not (application can continue)</returns>
public static bool TestMain(bool runTests, string[] args, params UnitSuiteInfo[] suits)
{
var asm = Assembly.GetExecutingAssembly();
String exeName = Path.GetFileName(asm.Location);
bool bListTests = false;
Dictionary <String, List <String> > filterClassMethodsToRun = null;
XDocument testsuites = new XDocument(new XDeclaration("1.0", "utf-8", ""), new XElement("testsuites"));
String slash = "[-/]+"; // "-tests", "--tests" or "/tests"
// Google unit testing uses --gtest_output=xml:<file>, can be shorten to "-out:<file>"
Regex reOutput = new Regex(slash + "(gtest_)?out(put=)?(xml)?:(.*)$");
// Google unit testing uses --gtest_list_tests, can be shorten to "-tests"
Regex reTests = new Regex(slash + "(gtest_list_)?tests");
// Google unit testing uses --gtest_filter=, can be shorten to "-filter=<class name>.<method name>" or "-filter:..."
Regex reFilter = new Regex(slash + "(gtest_)?filter.(.*)");
// Additional command line argument to run testing - "-test" or "-t"
Regex reDoTest = new Regex(slash + "t(est)?$");
bool printGoogleUnitTestFormat = false;
String xmlFilePath = null;
StringBuilder errorMessageHeadline = new StringBuilder();
StringBuilder errorMessage = new StringBuilder();
Regex reStackFrame = new Regex("^ *at +(.*?) in +(.*):line ([0-9]+)$");
foreach (var arg in args)
{
if (reDoTest.Match(arg).Success)
{
runTests = true;
continue;
}
if (reTests.Match(arg).Success)
{
bListTests = true;
runTests = false;
printGoogleUnitTestFormat = true;
continue;
}
var filtMatch = reFilter.Match(arg);
if (filtMatch.Success)
{
runTests = true;
printGoogleUnitTestFormat = true;
filterClassMethodsToRun = new Dictionary <string, List <string> >();
foreach (String classMethod in filtMatch.Groups[2].ToString().Split(':'))
{
var items = classMethod.Split('.').ToArray();
if (items.Length < 2)
{
continue;
}
String className = items[0];
if (!filterClassMethodsToRun.ContainsKey(className))
{
filterClassMethodsToRun.Add(className, new List <string>());
}
filterClassMethodsToRun[className].Add(items[1]);
}
continue;
}
var match = reOutput.Match(arg);
if (match.Success)
{
runTests = true;
printGoogleUnitTestFormat = true;
xmlFilePath = Path.GetFullPath(match.Groups[4].ToString());
}
}
if (!runTests && !bListTests)
{
return(false);
}
if (runTests && !printGoogleUnitTestFormat)
{
Console.Write("Testing ");
}
Stopwatch timer = new Stopwatch();
TestResults totalTestResults = new TestResults();
bool allowContinueTesting = true;
Stopwatch totalTestTimer = new Stopwatch();
totalTestTimer.Start();
List <UnitSuiteInfo> testSuites = new List <UnitSuiteInfo>();
foreach (var suiteLister in suits)
{
testSuites.AddRange(suiteLister.GetSuites());
}
// Sort alphabetically so would be executed in same order as in Test Explorer
testSuites.Sort((a, b) => a.SuiteName.CompareTo(b.SuiteName));
foreach (UnitSuiteInfo testSuite in testSuites)
{
String suiteName = testSuite.SuiteName;
List <String> filterMethods = null;
XElement testsuite = null;
TestResults localTestResults = null;
// Filter classes to execute
if (filterClassMethodsToRun != null)
{
if (!filterClassMethodsToRun.ContainsKey(suiteName))
{
continue;
}
filterMethods = filterClassMethodsToRun[suiteName];
}
if (printGoogleUnitTestFormat)
{
Console.WriteLine(suiteName);
}
List <UnitTestInfo> unitTests = testSuite.GetUnitTests().ToList();
for (int i = 0; i < unitTests.Count; i++)
{
UnitTestInfo testinfo = unitTests[i];
bool isLastMethod = i == unitTests.Count - 1;
if (!isLastMethod)
{
isLastMethod = unitTests.Skip(i + 1).Take(unitTests.Count - i - 1).Select(x => x.ignored).Contains(false);
}
if (bListTests)
{
Console.WriteLine(" <loc>" + testinfo.sourceCodePath + "(" + testinfo.line + ")");
}
String unitTestName = testinfo.UnitTestName;
String displayUnitTestName = unitTestName;
if (unitTestName.Contains('.'))
{
displayUnitTestName = Path.GetFileNameWithoutExtension(unitTestName) + " # GetParam() = " + Path.GetExtension(unitTestName);
unitTestName = Path.GetFileNameWithoutExtension(unitTestName);
}
if (filterMethods != null)
{
if (!filterMethods.Contains(unitTestName) && !filterMethods.Contains("*"))
{
continue;
}
}
// Info about current class in xml report
if (localTestResults == null)
{
localTestResults = new TestResults();
testsuite = new XElement("testsuite");
testsuite.Add(new XAttribute("name", suiteName));
testsuites.Root.Add(testsuite);
}
if (bListTests)
{
Console.WriteLine(" " + displayUnitTestName);
localTestResults.tests++;
continue;
}
if (testinfo.ignored)
{
localTestResults.disabled++;
if (printGoogleUnitTestFormat)
{
Console.WriteLine("[ SKIPPED ] " + suiteName + "." + unitTestName + " (0 ms)");
}
XElement skippedMethod = new XElement("testcase");
skippedMethod.Add(new XAttribute("name", unitTestName));
skippedMethod.Add(new XAttribute("status", "notrun"));
skippedMethod.Add(new XAttribute("time", 0));
skippedMethod.Add(new XAttribute("classname", suiteName));
testsuite.Add(skippedMethod);
continue;
}
if (printGoogleUnitTestFormat)
{
Console.WriteLine("[ RUN ] " + suiteName + "." + unitTestName);
}
try
{
timer.Restart();
try
{
localTestResults.tests++;
errorMessageHeadline.Clear();
errorMessage.Clear();
testinfo.InvokeTest(isLastMethod, localTestResults);
}
finally
{
timer.Stop();
}
}
catch (Exception _ex)
{
Exception ex = _ex;
if (_ex.InnerException != null)
{
ex = _ex.InnerException;
}
bool isFailure = true;
if (testinfo.ExceptionType == ex.GetType())
{
isFailure = false;
}
if (ex.GetType() == typeof(OperationCanceledException))
{
allowContinueTesting = false;
}
if (isFailure)
{
localTestResults.failures++;
String errorMsgShort = ex.Message;
errorMessage.AppendLine("Test method " + suiteName + "." + unitTestName + " threw exception: ");
errorMessage.AppendLine(ex.GetType().FullName + ": " + ex.Message);
errorMessage.AppendLine("Call stack:");
foreach (String frameEntry in ex.StackTrace.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
{
var fmatch = reStackFrame.Match(frameEntry);
if (fmatch.Success)
{
errorMessage.AppendLine(fmatch.Groups[2] + "(" + fmatch.Groups[3] + "): " + fmatch.Groups[1]);
}
}
errorMessageHeadline.Append(errorMsgShort);
}
}
long elapsedTime = timer.ElapsedMilliseconds;
XElement methodTestInfo = new XElement("testcase");
methodTestInfo.Add(new XAttribute("name", unitTestName));
methodTestInfo.Add(new XAttribute("status", "run"));
methodTestInfo.Add(new XAttribute("time", elapsedTime / 1000.0));
methodTestInfo.Add(new XAttribute("classname", suiteName));
String testState = "[ OK ]";
if (errorMessageHeadline.Length != 0)
{
String msg = errorMessage.ToString();
String errorMessageOneLiner = errorMessageHeadline.ToString().Replace("\n", "").Replace("\r", "");
XElement failure = new XElement("failure", new XCData(msg));
failure.Add(new XAttribute("message", errorMessageOneLiner));
methodTestInfo.Add(failure);
// Needs to be printed to console window as well, otherwise not seen by visual studio
Console.WriteLine(msg);
//MessageBox.Show(
// "errorMessageOneLiner: " + errorMessageOneLiner + "\r\n\r\n" +
// "msg: " + msg
//);
testState = "[ FAILED ]";
}
if (printGoogleUnitTestFormat)
{
Console.WriteLine(testState + " " + suiteName + "." + unitTestName + " (" + elapsedTime.ToString() + " ms)");
Console.Out.Flush();
}
else
{
Console.Write(".");
}
testsuite.Add(methodTestInfo);
if (!allowContinueTesting)
{
break;
}
}
// class scan complete, fetch results if necessary
if (testsuite != null)
{
FetchTestResults(testsuite, localTestResults);
totalTestResults.Add(localTestResults);
}
if (!allowContinueTesting)
{
break;
}
}
totalTestTimer.Stop();
// Save test results if requivested.
if (xmlFilePath != null)
{
try
{
String dir = Path.GetDirectoryName(xmlFilePath);
if (dir != "" && !Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
FetchTestResults(testsuites.Root, totalTestResults);
testsuites.Save(xmlFilePath);
}
catch (Exception ex)
{
Console.WriteLine("Error: Could not save '" + xmlFilePath + ": " + ex.Message);
}
}
if (runTests)
{
if (!printGoogleUnitTestFormat)
{
Console.WriteLine(" ok.");
String summaryLine = (totalTestResults.tests - totalTestResults.failures) + " tests passed";
if (totalTestResults.files != 0)
{
summaryLine += " (" + totalTestResults.files + " files verified)";
}
if (totalTestResults.failures != 0)
{
summaryLine += ", " + totalTestResults.failures + " FAILED";
}
if (totalTestResults.disabled != 0)
{
summaryLine += ", " + totalTestResults.disabled + " skipped";
}
Console.WriteLine(summaryLine);
Console.WriteLine();
}
TimeSpan elapsedtime = totalTestTimer.Elapsed;
String elapsed = "";
if (elapsedtime.Minutes != 0)
{
elapsed += elapsedtime.Minutes + " min ";
}
elapsed += elapsedtime.ToString(@"ss\.ff") + " sec";
Console.WriteLine("Test time: " + elapsed);
}
return(true);
}
