private void LogCommandExecution(TpmCc cc, TpmRc rc, TimeSpan executionTime)
{
NumCommands++;
if (!KnownCommandCodes.Contains(cc.ToString()))
{
cc = TpmCc.None;
}
if (!CmdStats.ContainsKey(cc))
{
CmdStats.Add(cc, new CommandStats());
}
CommandStats stat = CmdStats[cc];
if (rc == TpmRc.Success)
{
stat.NumSuccess++;
stat.SuccessExecutionTime += executionTime;
}
else
{
stat.NumFailures++;
stat.FailureExecutionTime += executionTime;
if (!stat.FailureResponses.Contains(rc))
{
stat.FailureResponses.Add(rc);
}
}
// No need to update stat.CallerTests here. Current test will be logged
// in the command execution statistics in the TestCompleted() notification.
}
internal void GenerateWLKReport(ref int Run, ref int Fail, ref int Success, bool PassFailedTests)
{
foreach (TestCaseInfo c in CumulativeFailures)
{
string parmsText = "";
// >>> write parms
if (c.Parms != null)
{
foreach (Object o in c.Parms)
{
if (o is TestCaseInfo)
{
// Formerly this parameter contained an exception object,
// information from which is now encapsulated in 'c'.
continue;
}
else if (o is Enum)
{
parmsText += Enum.GetName(o.GetType(), o) + " : ";
bool first = true;
Enum e = o as Enum;
Array values = Enum.GetValues(o.GetType());
foreach (Enum v in values)
{
if (e.HasFlag(v))
{
if (!first)
{
parmsText += " | ";
}
else
{
first = false;
}
parmsText += Enum.GetName(o.GetType(), v);
}
}
}
else if (o is byte[])
{
parmsText += Globs.HexFromByteArray((byte[])o);
}
else
{
parmsText += o.ToString();
}
parmsText += "<p>";
}
}
else
{
parmsText = "<br>";
}
string details = c.Message + "\n"
+ c.Location + "\n"
+ "Parameters:\n" + parmsText
+ (string.IsNullOrEmpty(c.RngSeed) ? ""
: "To reproduce use option: -seed " + c.RngSeed + "\n");
details.Replace("\r", "");
string errorText = c.TestCase.Replace(":", " : ") + ";\n" + details;
// c.StackTrace.Replace("\r", "");
WriteToWLKLog(errorText, TestResult.Failed, PassFailedTests);
}
// ================ ABORTED CASES SECTION ==========================
foreach (var item in AbortedTests)
{
string errorText = "TestCaseInfo:" + item.Key + " aborted because: "
+ GetAbortReasonMessage(item.Value);
if (item.Value.reason == AbortReason.BlockedCommand)
{
WriteToWLKLog(errorText, TestResult.NotRun, PassFailedTests);
}
else
{
WriteToWLKLog(errorText, TestResult.Failed, PassFailedTests);
}
}
// ================ COMMAND STATS SECTION ==========================
Log.Comment(string.Format("Total TPM command count: {0}", TotalNumCommands));
string successfulCommands = "Commands executed: ";
var sortedStats = CumulativeCmdStats.OrderBy(item => item.Key.ToString());
//var sortedStats = CumulativeCmdStats;
foreach (var item in sortedStats)
{
CommandStats stat = item.Value;
successfulCommands += item.Key + " (" + stat.NumSuccess + ") ";
}
Log.Comment(successfulCommands);
WriteToLog(successfulCommands);
// commands not executed -
List <TpmCc> notExecuted = new List <TpmCc>();
foreach (var c in CommandInformation.Info)
{
int num = sortedStats.Sum(y => (y.Key == c.CommandCode) ? 1 : 0);
if (num == 0 && TestCategorizer.CommandDefinedForMinTpm(c.CommandCode))
{
Log.Comment("Command " + c.CommandCode + " not executed.");
}
}
// ================ TEST ROUTINE STATS SECTION ==========================
int totalTestRoutineCount = this.TestRoutinesStats.Sum(item =>
item.Value.NumAborted + item.Value.NumFailed + item.Value.NumSuccess);
var sortedTestStats = TestRoutinesStats.OrderByDescending(item => item.Value.NumFailed);
foreach (var item in sortedTestStats)
{
if (item.Value.NumFailed == 0)
{
break; // no more failures
}
WriteToWLKLog(string.Format("{0} failed.", item.Key),
TestResult.Failed, PassFailedTests);
Fail++;
}
sortedTestStats = TestRoutinesStats.OrderByDescending(item => item.Value.NumSuccess);
foreach (var item in sortedTestStats)
{
if (item.Value.NumSuccess == 0)
{
break; // no more failures
}
Verify.IsTrue(true, item.Key);
WriteToLog("{0} passed.", item.Key);
Success++;
}
foreach (var item in sortedTestStats)
{
TestStats stat = item.Value;
if (stat.NumSuccess == 0 && stat.NumFailed == 0)
{
Log.Result(TestResult.NotRun, string.Format("{0} not run.", item.Key));
Run++;
}
}
}
} // GenerateTestCasesTable
internal void GenerateReport(string fileName)
{
TimeSpan testTimeSpan = DateTime.Now - TestSessStartTime;
StringBuilder sb = new StringBuilder();
StringWriter sw = new System.IO.StringWriter(sb);
HtmlTextWriter w = new HtmlTextWriter(sw);
w.RenderBeginTag(HtmlTextWriterTag.Body);
WriteLine(w, "TPM Test Report", HtmlTextWriterTag.H1);
WriteLine(w, "Start Time: " + TestSessStartTime.ToString("F"));
WriteLine(w, "Duration: " + testTimeSpan.TotalMinutes.ToString("F2") + " min");
bool failed = CumulativeFailures.Count != 0;
bool aborted = AbortedTests.Count != 0;
bool warnings = CumulativeWarnings.Count != 0;
string title;
if (failed)
{
title = "Some tests FAILED"
+ (aborted ? " or were ABORTED" : "")
+ (warnings ? " and there were WARNINGS" : "");
}
else
{
title = "All Tests PASSED"
+ (warnings ? " with WARNINGS" : "")
+ (aborted ? (warnings ? " and" : " but") + " some were ABORTED" : "");
}
WriteLine(w, title, HtmlTextWriterTag.H1);
WriteLine(w, "Total Substrate.Assertions Checked = " + TotalNumAsserts);
// ================ FAILING CASES SECTION ==========================
GenerateTestCasesTable(CumulativeFailures, "Failing Cases", w);
// ================ ABORTED CASES SECTION ==========================
if (AbortedTests.Count != 0)
{
WriteLine(w, "Aborted Cases", HtmlTextWriterTag.H2);
WriteLine(w, "Tests aborted because of commands not implemented by " +
"the target TPM, or blocked by OS.");
BeginTable(w, "Test Case", "Abort Reason");
foreach (var item in AbortedTests)
{
WriteTableRow(w, item.Key, GetAbortReasonMessage(item.Value));
}
EndTable(w); // table
}
// ================ WARNINGS SECTION ==========================
GenerateTestCasesTable(CumulativeWarnings, "Warnings", w);
// ================ COMMAND STATS SECTION ==========================
WriteLine(w, "Command Statistics", HtmlTextWriterTag.H2);
WriteLine(w, "Total TPM Commands Executed = " + TotalNumCommands);
BeginTable(w, "Command Code", "Successes", "Failures",
"Average<br>Success<br>Time, ms", "Average<br>Failure<br>Time, ms",
"Error Codes", "Calling Tests");
var sortedStats = CumulativeCmdStats.OrderBy(item => item.Key.ToString()); // item.Value.NumSuccess
//var sortedStats = CumulativeCmdStats;
foreach (var item in sortedStats)
{
CommandStats stat = item.Value;
string avgSuccessTime = stat.NumSuccess == 0 ? "-" : String.Format("{0:F1}",
stat.SuccessExecutionTime.TotalMilliseconds / stat.NumSuccess);
string avgFailureTime = stat.NumFailures == 0 ? "-" : String.Format("{0:F2}",
stat.FailureExecutionTime.TotalMilliseconds / stat.NumFailures);
WriteTableRow(w, item.Key, stat.NumSuccess, stat.NumFailures,
avgSuccessTime, avgFailureTime,
Globs.ToString(stat.FailureResponses, ", ", "-"),
Globs.ToString(stat.CallerTests, ", ", "-"));
}
EndTable(w);
// commands not executed -
List <TpmCc> notExecuted = new List <TpmCc>();
foreach (var c in CommandInformation.Info)
{
int num = sortedStats.Sum(y => (y.Key == c.CommandCode) ? 1 : 0);
if (num == 0)
{
Debug.WriteLine("Not executed:" + c.CommandCode.ToString());
}
}
// ================ TEST ROUTINE STATS SECTION ==========================
WriteLine(w, "Test Routine Statistics", HtmlTextWriterTag.H2);
int totalTestRoutineCount = this.TestRoutinesStats.Sum(item =>
item.Value.NumAborted + item.Value.NumFailed + item.Value.NumSuccess);
WriteLine(w, "Total Test Routines Executed = " + totalTestRoutineCount);
BeginTable(w, "Test Name", "Succeeded", "Failed", "Aborted", "Average<BR>Time, s");
var sortedTestStats = TestRoutinesStats.OrderBy(item => item.Key.ToString());
foreach (var item in sortedTestStats)
{
TestStats stat = item.Value;
int n = stat.NumSuccess + stat.NumFailed + stat.NumAborted;
if (n == 0)
{
continue;
}
string avgTime = String.Format("{0:F2}",
(double)stat.TotalExecutionTime / n / 1000);
WriteTableRow(w, item.Key, stat.NumSuccess, stat.NumFailed,
stat.NumAborted, avgTime);
}
EndTable(w);
w.RenderEndTag(); // Body
File.WriteAllText(fileName, sb.ToString());
}
internal void TestCompleted(TestContext ctx)
{
double testTime = (DateTime.Now - ctx.CurTestStartTime).TotalMilliseconds;
string TestName = ctx.CurTestMethod;
TestStatus status = ctx.CurTestStatus;
Debug.Assert((status & ~(TestStatus.OK | TestStatus.Aborted
| TestStatus.Failed)) == 0, "No test status set");
lock (TestRoutinesStats)
{
if (!TestRoutinesStats.ContainsKey(TestName))
{
TestRoutinesStats.Add(TestName, new TestStats());
}
TestStats s = TestRoutinesStats[TestName];
s.TotalExecutionTime += (int)testTime;
if (status == TestStatus.OK)
{
s.NumSuccess++;
}
else
{
// Multiple failure statuses are possible in case the test used
// restart feature.
if ((status & TestStatus.Failed) != 0)
{
s.NumFailed++;
}
if ((status & TestStatus.Aborted) != 0)
{
s.NumAborted++;
}
}
}
lock (CumulativeCmdStats)
{
// Merge in statistics accumulated in the test context during this
// test run.
TotalNumCommands += ctx.NumCommands;
TotalNumAsserts += ctx.NumAsserts;
foreach (var kv in ctx.CmdStats)
{
TpmCc cc = kv.Key;
CommandStats stat = kv.Value;
if (!CumulativeCmdStats.ContainsKey(cc))
{
Debug.Assert(stat.CallerTests.Count == 0);
stat.CallerTests.Add(ctx.CurTestMethod);
CumulativeCmdStats.Add(cc, stat);
}
else
{
CommandStats cumulStat = CumulativeCmdStats[cc];
cumulStat.NumSuccess += stat.NumSuccess;
cumulStat.NumFailures += stat.NumFailures;
cumulStat.FailureResponses = new SortedSet <TpmRc>(
cumulStat.FailureResponses.Union(stat.FailureResponses));
cumulStat.SuccessExecutionTime += stat.SuccessExecutionTime;
cumulStat.FailureExecutionTime += stat.FailureExecutionTime;
if (!cumulStat.CallerTests.Contains(ctx.CurTestMethod))
{
cumulStat.CallerTests.Add(ctx.CurTestMethod);
}
}
}
}
}
