// 20140720
// string getStatisticsStringScenario(TestScenario scenario, bool skipAutomatic)
string getStatisticsStringScenario(ITestScenario scenario, bool skipAutomatic)
{
string result = string.Empty;
// 20130322
//TestData.RefreshScenarioStatistics(scenario);
// 20141107
// TestData.RefreshScenarioStatistics(scenario, skipAutomatic);
var testStatistics = new TestStatistics();
testStatistics.RefreshScenarioStatistics(scenario, skipAutomatic);
result += @"<div id=""scenariostat"">";
// test results
result += @"Test cases:";
result += scenario.Statistics.All.ToString();
result += " Passed:";
result += scenario.Statistics.Passed.ToString();
result += " Failed:";
result += scenario.Statistics.Failed.ToString();
result += " Not tested:";
result += scenario.Statistics.NotTested.ToString();
result += " Time spent:";
result += Convert.ToInt32(scenario.Statistics.TimeSpent).ToString();
result += @" seconds</div>";
return(result);
}
protected override void BeginProcessing()
{
// var stat = new TestStat();
//
// if (null != TestData.TestSuites && 0 < TestData.TestSuites.Count) {
//
// // 20141107
// var testStatistics = new TestStatistics();
//
// foreach (var testSuite in TestData.TestSuites) {
//
// // 20141107
// // TestData.RefreshSuiteStatistics(testSuite, false);
// testStatistics.RefreshSuiteStatistics(testSuite, false);
// stat.Passed += testSuite.Statistics.Passed;
// stat.Failed += testSuite.Statistics.Failed;
// stat.PassedButWithBadSmell += testSuite.Statistics.PassedButWithBadSmell;
// stat.NotTested += testSuite.Statistics.NotTested;
// stat.All += testSuite.Statistics.All;
// }
//
// }
var testStatistics = new TestStatistics();
var stat = testStatistics.RefreshAllStatistics(TestData.TestSuites, false);
WriteObject(this, stat);
}
protected override void BeginProcessing()
{
// var stat = new TestStat();
//
// if (null != TestData.TestSuites && 0 < TestData.TestSuites.Count) {
//
// // 20141107
// var testStatistics = new TestStatistics();
//
// foreach (var testSuite in TestData.TestSuites) {
//
// // 20141107
// // TestData.RefreshSuiteStatistics(testSuite, false);
// testStatistics.RefreshSuiteStatistics(testSuite, false);
// stat.Passed += testSuite.Statistics.Passed;
// stat.Failed += testSuite.Statistics.Failed;
// stat.PassedButWithBadSmell += testSuite.Statistics.PassedButWithBadSmell;
// stat.NotTested += testSuite.Statistics.NotTested;
// stat.All += testSuite.Statistics.All;
// }
//
// }
var testStatistics = new TestStatistics();
var stat = testStatistics.RefreshAllStatistics(TestData.TestSuites, false);
WriteObject(this, stat);
}
public double GetAverageResultsTest(int id)
{
TestStatistics statistic = new TestStatistics(id);
double actual = statistic.GetAverageResults(id);
return(actual);
}
public void Inconclusive_should_be_incremented_if_completed_test_step_was_inconclusive()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Inconclusive));
Assert.AreEqual(1, testStatistics.Inconclusive.Value);
}
public void Inconclusive_should_be_incremented_if_completed_test_step_was_inconclusive()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Inconclusive));
Assert.AreEqual(1, testStatistics.Inconclusive.Value);
}
public void Skipped_should_be_incremented_if_completed_test_step_was_skipped()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Skipped));
Assert.AreEqual(1, testStatistics.Skipped.Value);
}
public void Skipped_should_be_incremented_if_completed_test_step_was_skipped()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Skipped));
Assert.AreEqual(1, testStatistics.Skipped.Value);
}
public void GetPassedFailedStatsTest(int id)
{
TestStatistics statistic = new TestStatistics(id);
PassedFailedModel actual = statistic.GetPassedFailedStats(id);
TestExpectedMock test = new TestExpectedMock();
PassedFailedModel expected = test.GetPassedFailed(id);
Assert.AreEqual(expected, actual);
}
public void Reset_should_set_passed_to_zero()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Passed));
testStatistics.Handle(new TestsReset());
Assert.AreEqual(0, testStatistics.Passed.Value);
}
public void Reset_should_set_inconclusive_to_zero()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Skipped));
testStatistics.Handle(new TestsReset());
Assert.AreEqual(0, testStatistics.Inconclusive.Value);
}
public void Reset_should_set_failed_to_zero()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Failed));
testStatistics.Handle(new TestsReset());
Assert.AreEqual(0, testStatistics.Failed.Value);
}
public void Count_should_not_be_updated_if_completed_test_step_is_not_test_case()
{
var testStatistics = new TestStatistics();
var testStepRun = new TestStepRun(new TestStepData("id", "name", "fullName", "testId"))
{
Result = new TestResult(new TestOutcome(TestStatus.Passed))
};
testStatistics.Handle(new TestStepFinished(null, testStepRun));
Assert.AreEqual(0, testStatistics.Passed.Value);
}
protected override void BeginProcessing()
{
if (null != TestData.TestSuites && 0 < TestData.TestSuites.Count)
{
// 20141107
var testStatistics = new TestStatistics();
foreach (var testSuite in TestData.TestSuites)
{
// 20141107
// TestData.RefreshSuiteStatistics(testSuite, false);
testStatistics.RefreshSuiteStatistics(testSuite, false);
}
}
}
public void SetUp()
{
windowManager = TestWindowManager.Create();
menuManager = MockRepository.GenerateStub <IMenuManager>();
windowManager.Stub(wm => wm.MenuManager).Return(menuManager);
menuManager.Stub(mm => mm.Add(Arg <string> .Is.Anything, Arg <Common.GallioFunc <MenuCommand> > .Is.Anything))
.Do((Common.GallioAction <string, Common.GallioFunc <MenuCommand> >)((m, f) => menuCommand = f()));
var testResultsController = MockRepository.GenerateStub <ITestResultsController>();
var optionsController = MockRepository.GenerateStub <IOptionsController>();
var testTreeModel = MockRepository.GenerateStub <ITestTreeModel>();
var testStatistics = new TestStatistics();
package = new TestResultsPackage(windowManager, testResultsController, optionsController, testTreeModel, testStatistics);
}
public void SetUp()
{
windowManager = TestWindowManager.Create();
menuManager = MockRepository.GenerateStub<IMenuManager>();
windowManager.Stub(wm => wm.MenuManager).Return(menuManager);
menuManager.Stub(mm => mm.Add(Arg<string>.Is.Anything, Arg<Common.Func<MenuCommand>>.Is.Anything))
.Do((Common.Action<string, Common.Func<MenuCommand>>)((m, f) => menuCommand = f()));
var testResultsController = MockRepository.GenerateStub<ITestResultsController>();
var optionsController = MockRepository.GenerateStub<IOptionsController>();
var testTreeModel = MockRepository.GenerateStub<ITestTreeModel>();
var testStatistics = new TestStatistics();
package = new TestResultsPackage(windowManager, testResultsController, optionsController, testTreeModel, testStatistics);
}
public ActionResult <List <TestOutputModel> > GetAllTests()
{
Mapper mapper = new Mapper();
AuthorDataAccess tests = new AuthorDataAccess();
List <TestOutputModel> listOfModels = mapper.ConvertTestDTOToTestModelList(tests.GetAllTests());
foreach (var i in listOfModels)
{
TestStatistics statistics = new TestStatistics(i.ID);
PassedFailedModel pfs = statistics.GetPassedFailedStats(i.ID);
i.AverageResult = statistics.GetAverageResults(i.ID);
i.Passed = pfs.Passed;
i.Failed = pfs.Failed;
i.SuccessRate = pfs.SuccessRate;
}
return(Ok(listOfModels));
}
protected override void BeginProcessing()
{
if (null != TestData.TestSuites && 0 < TestData.TestSuites.Count) {
// 20141107
var testStatistics = new TestStatistics();
foreach (var testSuite in TestData.TestSuites) {
// 20141107
// TestData.RefreshSuiteStatistics(testSuite, false);
testStatistics.RefreshSuiteStatistics(testSuite, false);
}
}
}
public ActionResult <TestOutputModel> GetTestAllInfoById(int testId)
{
Mapper mapper = new Mapper();
AuthorDataAccess tests = new AuthorDataAccess();
var test = tests.GetTestById(testId);
TestStatistics testStatistics = new TestStatistics(testId);
if (test == null)
{
return(BadRequest("Теста не существует"));
}
TestOutputModel model = mapper.ConvertTestDTOToTestOutputModel(tests.GetTestById(testId));
PassedFailedModel pfs = testStatistics.GetPassedFailedStats(testId);
model.Questions = mapper.ConvertQuestionDTOToQuestionModelList(tests.GetQuestionsByTestID(testId));
model.Tags = mapper.ConvertTagDTOToTagModelList(tests.GetTagsInTest(testId));
model.AverageResult = testStatistics.GetAverageResults(testId);
model.Passed = pfs.Passed;
model.Failed = pfs.Failed;
model.SuccessRate = pfs.SuccessRate;
foreach (QuestionOutputModel qModel in model.Questions)
{
qModel.Answers = mapper.ConvertAnswerDTOToAnswerModelList(tests.GetAnswerByQuestionId(qModel.ID));
QuestionStatistics statistics = new QuestionStatistics(qModel.ID);
qModel.PercentageOfCorrectlyAnswered = statistics.GetPercentageOfCorrectlyAnswered(qModel.ID);
Dictionary <int, double> answersPercent = new Dictionary <int, double>();
answersPercent = statistics.GetPercentageOfPeopleChoosingAnswer(qModel.ID);
foreach (var answer in qModel.Answers)
{
foreach (var i in answersPercent)
{
if (answer.ID == i.Key)
{
answer.PercentageOfPeopleChoosingAnswer = i.Value;
}
else
{
answer.PercentageOfPeopleChoosingAnswer = 0;
}
}
}
}
return(Ok(model));
}
public static TestStatistics TestBSP(TestJob job, LoadedBSP bsp, string temporaryDirectory)
{
TestStatistics stats = new TestStatistics();
Directory.CreateDirectory(temporaryDirectory);
using (var r = new TestResults(Path.Combine(temporaryDirectory, "bsptest.log")))
{
LoadedBSP.LoadedMCU[] MCUs;
if (job.DeviceRegex == null)
MCUs = bsp.MCUs.ToArray();
else
{
var rgFilter = new Regex(job.DeviceRegex);
MCUs = bsp.MCUs.Where(mcu => rgFilter.IsMatch(mcu.ExpandedMCU.ID)).ToArray();
}
if (job.SkippedDeviceRegex != null)
{
var rg = new Regex(job.SkippedDeviceRegex);
MCUs = MCUs.Where(mcu => !rg.IsMatch(mcu.ExpandedMCU.ID)).ToArray();
}
var loadedRules = job.RegisterRenamingRules?.Select(rule => new LoadedRenamingRule(rule))?.ToArray();
var noValidateReg = job.NonValidatedRegisters;
foreach (var sample in job.Samples)
{
r.BeginSample(sample.Name);
int cnt = 0, failed = 0, succeeded = 0;
foreach (var mcu in MCUs)
{
if (string.IsNullOrEmpty(mcu.ExpandedMCU.ID))
throw new Exception("Invalid MCU ID!");
var extraParams = job.DeviceParameterSets?.FirstOrDefault(s => s.DeviceRegexObject?.IsMatch(mcu.ExpandedMCU.ID) == true);
string mcuDir = Path.Combine(temporaryDirectory, mcu.ExpandedMCU.ID);
DateTime start = DateTime.Now;
var result = TestMCU(mcu, mcuDir + sample.TestDirSuffix, sample, extraParams, loadedRules, noValidateReg);
Console.WriteLine($"[{(DateTime.Now - start).TotalMilliseconds:f0} msec]");
if (result == TestResult.Failed)
failed++;
else if (result == TestResult.Succeeded)
succeeded++;
r.LogTestResult(mcu.ExpandedMCU.ID, result);
cnt++;
Console.WriteLine("{0}: {1}% done ({2}/{3} devices, {4} failed)", sample.Name, (cnt * 100) / MCUs.Length, cnt, MCUs.Length, failed);
}
if (succeeded == 0)
throw new Exception("Not a single MCU supports " + sample.Name);
r.EndSample();
stats.Passed += succeeded;
stats.Failed += failed;
}
}
return stats;
}
public static TestStatistics TestVendorSamples(VendorSampleDirectory samples, string bspDir, string temporaryDirectory, double testProbability = 1)
{
const string defaultToolchainID = "SysGCC-arm-eabi-5.3.0";
var toolchainPath = (string)Registry.CurrentUser.OpenSubKey(@"Software\Sysprogs\GNUToolchains").GetValue(defaultToolchainID);
if (toolchainPath == null)
throw new Exception("Cannot locate toolchain path from registry");
var toolchain = LoadedToolchain.Load(Environment.ExpandEnvironmentVariables(toolchainPath), new ToolchainRelocationManager());
var bsp = LoadedBSP.Load(new BSPManager.BSPSummary(Environment.ExpandEnvironmentVariables(Path.GetFullPath(bspDir))), toolchain);
TestStatistics stats = new TestStatistics();
int cnt = 0, failed = 0, succeeded = 0;
LoadedBSP.LoadedMCU[] MCUs = bsp.MCUs.ToArray();
string outputDir = Path.Combine(temporaryDirectory, "VendorSamples");
if (!Directory.Exists(outputDir))
Directory.CreateDirectory(outputDir);
int sampleCount = samples.Samples.Length;
Random rng = new Random();
using (var r = new TestResults(Path.Combine(temporaryDirectory, "bsptest.log")))
{
foreach (var vs in samples.Samples)
{
LoadedBSP.LoadedMCU mcu;
try
{
var rgFilterID = new Regex(vs.DeviceID.Replace('x', '.'), RegexOptions.IgnoreCase);
mcu = bsp.MCUs.Where(f => rgFilterID.IsMatch(f.ExpandedMCU.ID)).ToArray()?.First();
vs.DeviceID = mcu.ExpandedMCU.ID;
}
catch (Exception ex)
{
r.ExceptionSample(ex.Message, "mcu " + vs.DeviceID + " not found in bsp");
Console.WriteLine("bsp have not mcu:" + vs.DeviceID);
continue;
}
if (testProbability < 1 && rng.NextDouble() > testProbability)
{
cnt++;
continue;
}
bool aflSoftFPU = true;
string mcuDir = Path.Combine(temporaryDirectory, "VendorSamples", vs.UserFriendlyName);
if (!Directory.Exists(mcuDir))
Directory.CreateDirectory(mcuDir);
if (vs.UserFriendlyName.ToUpper().Contains("RTOS"))
aflSoftFPU = false;
DateTime start = DateTime.Now;
var result = TestVendorSample(mcu, vs, mcuDir, aflSoftFPU, samples);
Console.WriteLine($"[{(DateTime.Now - start).TotalMilliseconds:f0} msec]");
if (result == TestResult.Failed)
failed++;
else if (result == TestResult.Succeeded)
succeeded++;
r.LogTestResult(vs.UserFriendlyName, result);
cnt++;
Console.WriteLine("{0}: {1}% done ({2}/{3} projects, {4} failed)", vs.UserFriendlyName, (cnt * 100) / sampleCount, cnt, sampleCount, failed);
}
}
stats.Passed += succeeded;
stats.Failed += failed;
if (samples is ConstructedVendorSampleDirectory)
{
(samples as ConstructedVendorSampleDirectory).ToolchainDirectory = toolchainPath;
(samples as ConstructedVendorSampleDirectory).BSPDirectory = Path.GetFullPath(bspDir);
}
return stats;
}
// 20140720
// string getStatisticsStringScenario(TestScenario scenario, bool skipAutomatic)
string getStatisticsStringScenario(ITestScenario scenario, bool skipAutomatic)
{
string result = string.Empty;
// 20130322
//TestData.RefreshScenarioStatistics(scenario);
// 20141107
// TestData.RefreshScenarioStatistics(scenario, skipAutomatic);
var testStatistics = new TestStatistics();
testStatistics.RefreshScenarioStatistics(scenario, skipAutomatic);
result += @"<div id=""scenariostat"">";
// test results
result += @"Test cases:";
result += scenario.Statistics.All.ToString();
result += " Passed:";
result += scenario.Statistics.Passed.ToString();
result += " Failed:";
result += scenario.Statistics.Failed.ToString();
result += " Not tested:";
result += scenario.Statistics.NotTested.ToString();
result += " Time spent:";
result += Convert.ToInt32(scenario.Statistics.TimeSpent).ToString();
result += @" seconds</div>";
return result;
}
// 20140720
// string getStatisticsStringSuite(TestSuite suite, bool skipAutomatic)
string getStatisticsStringSuite(ITestSuite suite, bool skipAutomatic)
{
string result = string.Empty;
// 20141107
// TestData.RefreshSuiteStatistics(suite, skipAutomatic);
var testStatistics = new TestStatistics();
testStatistics.RefreshSuiteStatistics(suite, skipAutomatic);
result += @"<div id=""suitestat"">";
int scPassed = 0;
int scFailed = 0;
int scNotTested = 0;
foreach (var tsc in suite.TestScenarios) {
switch (tsc.enStatus) {
case TestStatuses.Passed:
scPassed++;
break;
case TestStatuses.Failed:
scFailed++;
break;
case TestStatuses.NotRun:
scNotTested++;
break;
default:
// 20130428
//throw new Exception("Invalid value for TestScenarioStatuses");
scNotTested++;
break;
}
}
// scenarios
result += @"Scenarios:";
result += suite.TestScenarios.Count.ToString();
result += " Passed:";
result += scPassed.ToString();
result += " Failed:";
result += scFailed.ToString();
result += " Not tested:";
result += scNotTested.ToString();
result += newLineHTML;
// test results
result += @"Test cases:";
result += suite.Statistics.All.ToString();
result += " Passed:";
result += suite.Statistics.Passed.ToString();
result += " Failed:";
result += suite.Statistics.Failed.ToString();
result += " Not tested:";
result += suite.Statistics.NotTested.ToString();
result += " Time spent:";
result += Convert.ToInt32(suite.Statistics.TimeSpent).ToString();
result += @" seconds</div>";
return result;
}
// 20140720
// string getStatisticsStringSuite(TestSuite suite, bool skipAutomatic)
string getStatisticsStringSuite(ITestSuite suite, bool skipAutomatic)
{
string result = string.Empty;
// 20141107
// TestData.RefreshSuiteStatistics(suite, skipAutomatic);
var testStatistics = new TestStatistics();
testStatistics.RefreshSuiteStatistics(suite, skipAutomatic);
result += @"<div id=""suitestat"">";
int scPassed = 0;
int scFailed = 0;
int scNotTested = 0;
foreach (var tsc in suite.TestScenarios)
{
switch (tsc.enStatus)
{
case TestStatuses.Passed:
scPassed++;
break;
case TestStatuses.Failed:
scFailed++;
break;
case TestStatuses.NotRun:
scNotTested++;
break;
default:
// 20130428
//throw new Exception("Invalid value for TestScenarioStatuses");
scNotTested++;
break;
}
}
// scenarios
result += @"Scenarios:";
result += suite.TestScenarios.Count.ToString();
result += " Passed:";
result += scPassed.ToString();
result += " Failed:";
result += scFailed.ToString();
result += " Not tested:";
result += scNotTested.ToString();
result += newLineHTML;
// test results
result += @"Test cases:";
result += suite.Statistics.All.ToString();
result += " Passed:";
result += suite.Statistics.Passed.ToString();
result += " Failed:";
result += suite.Statistics.Failed.ToString();
result += " Not tested:";
result += suite.Statistics.NotTested.ToString();
result += " Time spent:";
result += Convert.ToInt32(suite.Statistics.TimeSpent).ToString();
result += @" seconds</div>";
return(result);
}
public static TestStatistics TestVendorSamples(VendorSampleDirectory samples, string bspDir, string temporaryDirectory, double testProbability = 1)
{
const string defaultToolchainID = "SysGCC-arm-eabi-5.3.0";
var toolchainPath = (string)Registry.CurrentUser.OpenSubKey(@"Software\Sysprogs\GNUToolchains").GetValue(defaultToolchainID);
if (toolchainPath == null)
{
throw new Exception("Cannot locate toolchain path from registry");
}
var toolchain = LoadedToolchain.Load(Environment.ExpandEnvironmentVariables(toolchainPath), new ToolchainRelocationManager());
var bsp = LoadedBSP.Load(new BSPManager.BSPSummary(Environment.ExpandEnvironmentVariables(Path.GetFullPath(bspDir))), toolchain);
TestStatistics stats = new TestStatistics();
int cnt = 0, failed = 0, succeeded = 0;
LoadedBSP.LoadedMCU[] MCUs = bsp.MCUs.ToArray();
string outputDir = Path.Combine(temporaryDirectory, "VendorSamples");
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
int sampleCount = samples.Samples.Length;
Random rng = new Random();
using (var r = new TestResults(Path.Combine(temporaryDirectory, "bsptest.log")))
{
foreach (var vs in samples.Samples)
{
LoadedBSP.LoadedMCU mcu;
try
{
var rgFilterID = new Regex(vs.DeviceID.Replace('x', '.'), RegexOptions.IgnoreCase);
mcu = bsp.MCUs.Where(f => rgFilterID.IsMatch(f.ExpandedMCU.ID)).ToArray()?.First();
vs.DeviceID = mcu.ExpandedMCU.ID;
}
catch (Exception ex)
{
r.ExceptionSample(ex.Message, "mcu " + vs.DeviceID + " not found in bsp");
Console.WriteLine("bsp have not mcu:" + vs.DeviceID);
continue;
}
if (testProbability < 1 && rng.NextDouble() > testProbability)
{
cnt++;
continue;
}
bool aflSoftFPU = true;
string mcuDir = Path.Combine(temporaryDirectory, "VendorSamples", vs.UserFriendlyName);
if (!Directory.Exists(mcuDir))
{
Directory.CreateDirectory(mcuDir);
}
if (vs.UserFriendlyName.ToUpper().Contains("RTOS"))
{
aflSoftFPU = false;
}
DateTime start = DateTime.Now;
var result = TestVendorSample(mcu, vs, mcuDir, aflSoftFPU, samples);
Console.WriteLine($"[{(DateTime.Now - start).TotalMilliseconds:f0} msec]");
if (result == TestResult.Failed)
{
failed++;
}
else if (result == TestResult.Succeeded)
{
succeeded++;
}
r.LogTestResult(vs.UserFriendlyName, result);
cnt++;
Console.WriteLine("{0}: {1}% done ({2}/{3} projects, {4} failed)", vs.UserFriendlyName, (cnt * 100) / sampleCount, cnt, sampleCount, failed);
}
}
stats.Passed += succeeded;
stats.Failed += failed;
if (samples is ConstructedVendorSampleDirectory)
{
(samples as ConstructedVendorSampleDirectory).ToolchainDirectory = toolchainPath;
(samples as ConstructedVendorSampleDirectory).BSPDirectory = Path.GetFullPath(bspDir);
}
return(stats);
}
public void Reset_should_set_inconclusive_to_zero()
{
var testStatistics = new TestStatistics();
testStatistics.Handle(GetTestStepFinished(TestStatus.Skipped));
testStatistics.Handle(new TestsReset());
Assert.AreEqual(0, testStatistics.Inconclusive.Value);
}
public void Count_should_not_be_updated_if_completed_test_step_is_not_test_case()
{
var testStatistics = new TestStatistics();
var testStepRun = new TestStepRun(new TestStepData("id", "name", "fullName", "testId"))
{
Result = new TestResult(new TestOutcome(TestStatus.Passed))
};
testStatistics.Handle(new TestStepFinished(null, testStepRun));
Assert.AreEqual(0, testStatistics.Passed.Value);
}
public PassedFailedModel qq(int testId)
{
TestStatistics st = new TestStatistics(testId);
return(st.GetPassedFailedStats(testId));
}
public static TestStatistics TestVendorSamples(VendorSampleDirectory samples, string bspDir, string temporaryDirectory, double testProbability = 1, bool esp32 = false)
{
string defaultToolchainID = "SysGCC-arm-eabi-6.2.0";
flEsp32 = false;
if (esp32)
{
defaultToolchainID = "SysGCC-xtensa-esp32-elf-5.2.0";
flEsp32 = true;
}
var toolchainPath = (string)Registry.CurrentUser.OpenSubKey(@"Software\Sysprogs\GNUToolchains").GetValue(defaultToolchainID);
if (toolchainPath == null)
{
throw new Exception("Cannot locate toolchain path from registry");
}
var toolchain = LoadedToolchain.Load(new ToolchainSource.Other(Environment.ExpandEnvironmentVariables(toolchainPath)));
var bsp = LoadedBSP.Load(new BSPEngine.BSPSummary(Environment.ExpandEnvironmentVariables(Path.GetFullPath(bspDir))), toolchain);
if (flEsp32)
{
toolchainPath = toolchainPath.Replace('\\', '/'); //esp32
}
TestStatistics stats = new TestStatistics();
int cnt = 0, failed = 0, succeeded = 0;
LoadedBSP.LoadedMCU[] MCUs = bsp.MCUs.ToArray();
string outputDir = Path.Combine(temporaryDirectory, "VendorSamples");
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
int sampleCount = samples.Samples.Length;
Random rng = new Random();
using (var r = new TestResults(Path.Combine(temporaryDirectory, "bsptest.log")))
{
r.BeginSample("Vendor Samples");
foreach (var vs in samples.Samples)
{
LoadedBSP.LoadedMCU mcu;
try
{
var rgFilterID = new Regex(vs.DeviceID.Replace('x', '.'), RegexOptions.IgnoreCase);
mcu = bsp.MCUs.Where(f => rgFilterID.IsMatch(f.ExpandedMCU.ID)).ToArray()?.First();
vs.DeviceID = mcu.ExpandedMCU.ID;
}
catch (Exception ex)
{
r.ExceptionSample(ex.Message, "mcu " + vs.DeviceID + " not found in bsp");
Console.WriteLine("bsp have not mcu:" + vs.DeviceID);
continue;
}
if (testProbability < 1 && rng.NextDouble() > testProbability)
{
cnt++;
continue;
}
string mcuDir = Path.Combine(temporaryDirectory, "VendorSamples", vs.UserFriendlyName);
mcuDir += $"-{mcu.ExpandedMCU.ID}";
if (!Directory.Exists(mcuDir))
{
Directory.CreateDirectory(mcuDir);
}
DateTime start = DateTime.Now;
string[] hwSubstrings = new[]
{
@"\ARM_CM4F\port.c",
@"ARM_CM7\r0p1\port.c",
@"CM4_GCC.a",
};
if (vs.SourceFiles.FirstOrDefault(f => ContainsAnySubstrings(f, hwSubstrings)) != null)
{
if (vs.Configuration.MCUConfiguration != null)
{
var dict = PropertyDictionary2.ReadPropertyDictionary(vs.Configuration.MCUConfiguration);
dict["com.sysprogs.bspoptions.arm.floatmode"] = "-mfloat-abi=hard";
vs.Configuration.MCUConfiguration = new PropertyDictionary2 {
Entries = dict.Select(kv => new PropertyDictionary2.KeyValue {
Key = kv.Key, Value = kv.Value
}).ToArray()
};
}
else
{
vs.Configuration.MCUConfiguration = new PropertyDictionary2
{
Entries = new PropertyDictionary2.KeyValue[]
{ new PropertyDictionary2.KeyValue {
Key = "com.sysprogs.bspoptions.arm.floatmode", Value = "-mfloat-abi=hard"
} }
};
}
}
vs.SourceFiles = vs.SourceFiles.Where(s => !IsNonGCCFile(vs, s)).ToArray();
var result = TestVendorSample(mcu, vs, mcuDir, samples);
Console.WriteLine($"[{(DateTime.Now - start).TotalMilliseconds:f0} msec]");
if (result == TestResult.Failed)
{
failed++;
}
else if (result == TestResult.Succeeded)
{
succeeded++;
}
r.LogTestResult(vs.UserFriendlyName, result);
cnt++;
Console.WriteLine("{0}: {1}% done ({2}/{3} projects, {4} failed)", vs.UserFriendlyName, (cnt * 100) / sampleCount, cnt, sampleCount, failed);
}
r.EndSample();
}
stats.Passed += succeeded;
stats.Failed += failed;
if (samples is ConstructedVendorSampleDirectory)
{
(samples as ConstructedVendorSampleDirectory).ToolchainDirectory = toolchainPath;
(samples as ConstructedVendorSampleDirectory).BSPDirectory = Path.GetFullPath(bspDir);
}
return(stats);
}
public static TestStatistics TestBSP(TestJob job, LoadedBSP bsp, string temporaryDirectory)
{
TestStatistics stats = new TestStatistics();
Directory.CreateDirectory(temporaryDirectory);
using (var r = new TestResults(Path.Combine(temporaryDirectory, "bsptest.log")))
{
LoadedBSP.LoadedMCU[] MCUs;
if (job.DeviceRegex == null)
{
MCUs = bsp.MCUs.ToArray();
}
else
{
var rgFilter = new Regex(job.DeviceRegex);
MCUs = bsp.MCUs.Where(mcu => rgFilter.IsMatch(mcu.ExpandedMCU.ID)).ToArray();
}
if (job.SkippedDeviceRegex != null)
{
var rg = new Regex(job.SkippedDeviceRegex);
MCUs = MCUs.Where(mcu => !rg.IsMatch(mcu.ExpandedMCU.ID)).ToArray();
}
var loadedRules = job.RegisterRenamingRules?.Select(rule => new LoadedRenamingRule(rule))?.ToArray();
var noValidateReg = job.NonValidatedRegisters;
foreach (var sample in job.Samples)
{
r.BeginSample(sample.Name);
int cnt = 0, failed = 0, succeeded = 0;
var effectiveMCUs = MCUs;
if (!string.IsNullOrEmpty(sample.DeviceRegex))
{
Regex rgDevice = new Regex(sample.DeviceRegex);
effectiveMCUs = MCUs.Where(mcu => rgDevice.IsMatch(mcu.ExpandedMCU.ID)).ToArray();
}
foreach (var mcu in effectiveMCUs)
{
if (string.IsNullOrEmpty(mcu.ExpandedMCU.ID))
{
throw new Exception("Invalid MCU ID!");
}
var extraParams = job.DeviceParameterSets?.FirstOrDefault(s => s.DeviceRegexObject?.IsMatch(mcu.ExpandedMCU.ID) == true);
string mcuDir = Path.Combine(temporaryDirectory, mcu.ExpandedMCU.ID);
DateTime start = DateTime.Now;
var result = TestMCU(mcu, mcuDir + sample.TestDirSuffix, sample, extraParams, loadedRules, noValidateReg, job.UndefinedMacros);
Console.WriteLine($"[{(DateTime.Now - start).TotalMilliseconds:f0} msec]");
if (result == TestResult.Failed)
{
failed++;
}
else if (result == TestResult.Succeeded)
{
succeeded++;
}
r.LogTestResult(mcu.ExpandedMCU.ID, result);
cnt++;
Console.WriteLine("{0}: {1}% done ({2}/{3} devices, {4} failed)", sample.Name, (cnt * 100) / effectiveMCUs.Length, cnt, effectiveMCUs.Length, failed);
}
if ((succeeded + failed) == 0)
{
throw new Exception("Not a single MCU supports " + sample.Name);
}
r.EndSample();
stats.Passed += succeeded;
stats.Failed += failed;
}
}
return(stats);
}
