public TestSuiteBuilder(TestSuiteInfo testSuiteInfo)
{
if (testSuiteInfo == null)
{
throw new ArgumentNullException("testSuiteInfo");
}
this.testSuiteInfo = testSuiteInfo;
// get run descriptor
this.runDescriptor =
testSuiteInfo.TesterType.MethodsWith(Flags.AllMembers, typeof(PerfRunDescriptorAttribute))
.FirstOrDefault(
m => m.ReturnType == typeof(double) && m.HasParameterSignature(new[] { typeof(int) }));
// get set up
this.setUp =
testSuiteInfo.TesterType.MethodsWith(Flags.AllMembers, typeof(PerfSetUpAttribute))
.FirstOrDefault(
m =>
m.ReturnType == typeof(void) &&
m.HasParameterSignature(new[] { typeof(int), testSuiteInfo.TestedAbstraction }));
// get tear down
this.tearDown =
testSuiteInfo.TesterType.MethodsWith(Flags.AllMembers, typeof(PerfTearDownAttribute))
.FirstOrDefault(
m =>
m.ReturnType == typeof(void) &&
m.HasParameterSignature(new[] { testSuiteInfo.TestedAbstraction }));
}
public TestSuiteCodeBuilder(
TestSuiteInfo testSuiteInfo,
string runDescriptorMethodName,
string setUpMethodName,
string tearDownMethodName)
{
this.testSuiteInfo = testSuiteInfo;
this.runDescriptorMethodName = runDescriptorMethodName;
this.setUpMethodName = setUpMethodName;
this.tearDownMethodName = tearDownMethodName;
this.testerType = new CodeTypeReference(testSuiteInfo.TesterType);
this.testedAbstraction = new CodeTypeReference(testSuiteInfo.TestedAbstraction);
}
public TestNodeViewModel(TesterNodeViewModel parent, PerfLab perfLab, TestSuiteInfo testSuite, string testMethodName)
: base(parent)
{
this.PerfLab = perfLab;
this.TestSuite = testSuite;
this.Tests = testSuite.Tests.Where(x => x.TestMethodName == testMethodName).ToArray();
this.Name = testMethodName;
this.Children.AddRange(from testInfo in this.Tests
select new TestedTypeNodeViewModel(this, perfLab, testInfo)
{
IsChecked = false
});
this.IsEnabled = this.Children.Count > 0;
}
public static void GenerateNUnitCode(TestSuiteInfo testSuite, string targetFilePath)
{
var builder = new StringBuilder();
builder.AppendLine("using NUnit.Framework;");
builder.AppendLine("namespace TestCodeGenerator");
builder.AppendLine("{");
builder.AppendLine("\t[TestFixture]");
var pattern = @"\.(\w+)";
var match = Regex.Match(testSuite.Target, pattern);
var methodName = match.Groups[1].Value;
var className = "FunctionLib";
builder.AppendLine($"\tpublic class {className}Tests");
builder.AppendLine("\t{");
builder.AppendLine("\t\t[Test]");
builder.AppendLine($"\t\tpublic void {methodName}Test()");
builder.AppendLine("\t\t{");
foreach (var assertion in testSuite.TestCases.SelectMany(c => c.Assertions).ToList())
{
var expectedOutput = assertion.ExpectedOutput;
var argumentList = string.Join(", ", assertion.InputValues);
builder.AppendLine(
$"\t\t\tAssert.AreEqual(\"{expectedOutput}\", {className}.{methodName}({argumentList}));");
}
builder.AppendLine("\t\t}");
builder.AppendLine("\t}");
builder.AppendLine("}");
var fileContent = builder.ToString();
using (var writer = new StreamWriter(targetFilePath))
{
try
{
writer.Write(fileContent);
writer.Close();
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
}
public TesterNodeViewModel(PerfLab lab, TestSuiteInfo testSuiteInfo)
: base(null)
{
var descr = testSuiteInfo.TestSuiteDescription;
this.Name = string.IsNullOrEmpty(descr) ? testSuiteInfo.TesterType.FullName : descr;
this.TestSuiteInfo = testSuiteInfo;
this.Lab = lab;
this.Children.AddRange(
testSuiteInfo.Tests.GroupBy(x => x.TestMethodName)
.Select(x => new TestNodeViewModel(this, lab, testSuiteInfo, x.Key)));
MessageBus.Current.Listen <ChartRemoved>().Subscribe(OnChartRemoved);
this.IsEnabled = this.Children.Count > 0;
}
private TestSuiteInfo GetTestSuiteInfoFromTestSuiteFile(string filePath)
{
var testSuite = new TestSuiteInfo();
XDocument xdoc;
try
{
xdoc = XDocument.Load(filePath);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
testSuite.Name = xdoc.XPathSelectElement("/TestSuite/Name")?.Value;
testSuite.Target = xdoc.XPathSelectElement("/TestSuite/Target")?.Value;
foreach (var testcaseElement in xdoc.XPathSelectElements("/TestSuite/TestCases/TestCase"))
{
var testCase = new TestCaseInfo {
Id = testcaseElement.Attribute("ID")?.Value
};
foreach (var assertionElement in testcaseElement.XPathSelectElements("Assertions/Assertion"))
{
var assertion = new AssertionInfo
{
Id = assertionElement.Attribute("ID")?.Value,
InputValues = assertionElement.XPathSelectElements("InputValues/InputValue")
.Select(v => Convert.ToDouble(v.Value)).ToList(),
ExpectedOutput = assertionElement.XPathSelectElement("ExpectedOutput")?.Value,
ActualOutput = assertionElement.XPathSelectElement("ActualOutput")?.Value,
Result = assertionElement.XPathSelectElement("Result")?.Value
};
testCase.Assertions.Add(assertion);
}
testSuite.TestCases.Add(testCase);
}
return(testSuite);
}
//生成单元测试代码
public void GenerateUnitTestFile(TestSuiteInfo testSuite)
{
var fileName = $"{testSuite.FunctionName}Test.cs";
var filePath = Path.Combine(_unitTestFilesDir, fileName);
try
{
//若单元测试代码输出文件夹不存在，则创建该文件夹
if (!Directory.Exists(_unitTestFilesDir))
{
Directory.CreateDirectory(_unitTestFilesDir);
}
UnitTestCodeGenerator.GenerateNUnitCode(testSuite, filePath);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
/// <summary>
/// Load test suite.
/// </summary>
/// <param name="filename">Filename of the profile</param>
public void LoadTestSuite(string filename)
{
ProfileUtil profile = ProfileUtil.LoadProfile(filename);
TestSuiteInfo tsinfo = null;
foreach (var g in testSuites)
{
foreach (var info in g)
{
if (profile.VerifyVersion(info.TestSuiteName, info.TestSuiteVersion))
{
tsinfo = info;
goto FindTestSuite;
}
}
}
FindTestSuite:
profile.Dispose();
util.LoadTestSuiteConfig(tsinfo);
util.LoadTestSuiteAssembly();
util.LoadProfileSettings(filename);
}
private void btnGA_Click(object sender, EventArgs e)
{
//路径覆盖测试数据集
var gaAssertions = new List <AssertionInfo>();
//边界值测试数据集
var bounaryTestAssertions = new List <AssertionInfo>();
//加载参数
LoadParameters();
txtResult.Clear();
//通过遗传算法得到路径覆盖测试数据
gaAssertions.AddRange(Task.Run(() => GaTestDataGenerator.GetAssertions(_gaParameters, _function, _targetPaths)).Result);
//得到边界值测试数据
bounaryTestAssertions.AddRange(Task.Run(() => BoundaryTestDataGenerator.GetAssertions(_function)).Result);
var testSuite = new TestSuiteInfo
{
Name = $"针对{cmbFunction.Text}函数的测试套件",
Target = $"{AbstractFunction.CreateInstance(cmbFunction.SelectedValue.ToString())}"
};
testSuite.TestCases.Add(new TestCaseInfo {
Name = "路径覆盖测试", Assertions = gaAssertions
});
testSuite.TestCases.Add(new TestCaseInfo {
Name = "边界值测试", Assertions = bounaryTestAssertions
});
GenerateTestSuiteFile(testSuite);
// ShowTestData(gaAssertions.Union(bounaryTestAssertions).ToList());
// ShowAssertions(gaAssertions);
//将 galog 显示到文本框中
const string logPath = @"c:\#GA_DEMO\galog.txt";
txtResult.AppendText(File.ReadAllText(logPath));
//滚动到光标处
txtResult.ScrollToCaret();
}
//生成测试数据文档
private void GenerateTestSuiteFile(TestSuiteInfo testSuite)
{
var fileName = $"TS_{testSuite.FunctionName}_{DateTime.Now:yyyyMMddHHmmss}.xml";
var filePath = Path.Combine(_testSutieFilesDir, fileName);
try
{
//若输出文件夹不存在，则创建该文件夹
if (!Directory.Exists(_testSutieFilesDir))
{
Directory.CreateDirectory(_testSutieFilesDir);
}
var xdoc = new XElement("TestSuite", new XElement("Name", testSuite.Name),
new XElement("Target", $"{testSuite.Target}"),
new XElement("TestCases", testSuite.TestCases.Select(c => new XElement("TestCase",
new XAttribute("ID", $"TC_{testSuite.TestCases.IndexOf(c) + 1:000}"),
new XElement("Name", c.Name),
new XElement("Assertions",
c.Assertions.Select(a => new XElement("Assertion",
new XAttribute("ID", $"TA_{c.Assertions.IndexOf(a) + 1:0000}"),
new XElement("InputValues",
a.InputValues.Select(v => new XElement("InputValue", v))),
new XElement("ExpectedOutput", a.ExpectedOutput),
new XElement("ActualOutput", a.ActualOutput),
new XElement("Result", a.Result))))))),
new XElement("TestSummery",
new XElement("Executed", testSuite.TestSummery.Executed),
new XElement("Passed", testSuite.TestSummery.Passed),
new XElement("Failed", testSuite.TestSummery.Failed)));
xdoc.Save(filePath);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
private static TestInfo GetTestInfo(Guid id, MethodInfo method, Type testedAbstraction, TestSuiteInfo suiteInfo, Type testedType)
{
CheckTestability(suiteInfo.TesterType, testedType);
var testAttribute = method.Attribute <PerfTestAttribute>();
if (testAttribute == null)
{
throw new ArgumentNullException("method");
}
if (method.ReturnType != typeof(void) || !method.HasParameterSignature(new[] { testedAbstraction }))
{
throw new ArgumentException("Incorrect parameter signature");
}
if (testedType.IsGenericType)
{
// Fill the generic type arguments of the loaded generic type
// with the tested abstraction interface actual generic type arguments.
// Example: tested abstraction = IList<int>, tested type = List<T>
// This line converts List<T> in List<int>
testedType = testedType.MakeGenericType(testedAbstraction.GetGenericArguments());
}
TestInfo result;
var ignoreAttribute = method.Attribute <PerfIgnoreAttribute>();
if (ignoreAttribute == null)
{
result = new TestInfo
{
TestId = id,
TestDescription = testAttribute.Description,
TestMethodName = method.Name,
TestedType = testedType,
Suite = suiteInfo
};
}
else
{
result = new TestInfoIgnored
{
TestId = id,
TestDescription = testAttribute.Description,
TestMethodName = method.Name,
IgnoreMessage = ignoreAttribute.Message,
TestedType = testedType,
Suite = suiteInfo
};
}
return(result);
}
private static string BuildTestSuiteAssembly(TestSuiteInfo testSuiteInfo)
{
var builder = new TestSuiteBuilder(testSuiteInfo);
return(builder.Build());
}
public static IObservable <PerfTestResult> Run(TestSuiteInfo testSuiteInfo, int start, int step, int end, PerfTestConfiguration configuration, bool parallel = false)
{
return(CreateRunObservable(testSuiteInfo, x => true, processes => processes.Start(start, step, end, !parallel), configuration, parallel));
}
private static IObservable <PerfTestResult> CreateRunObservable(TestSuiteInfo testSuiteInfo,
Predicate <TestInfo> testFilter,
Action <ExperimentProcess> startProcess,
PerfTestConfiguration configuration,
bool parallel = false)
{
return(Observable.Create <PerfTestResult>(
observer =>
{
var assemblyLocation = BuildTestSuiteAssembly(testSuiteInfo);
var processes = new MultiExperimentProcess(
(from testMethod in testSuiteInfo.Tests
where testFilter(testMethod)
select
new ExperimentProcess(
string.Format(
"{0}.{1}({2})",
testSuiteInfo.TesterType.Name,
testMethod.TestMethodName,
testMethod.TestedType.Name),
assemblyLocation,
TestSuiteCodeBuilder.TestSuiteClassName,
testSuiteInfo.TesterType,
testMethod.TestedType,
testMethod.TestMethodName,
configuration)).ToArray());
var listeners = Observable.Empty <PerfTestResult>();
if (!parallel)
{
listeners = processes.Experiments.Aggregate(listeners,
(current, experiment) =>
current.Concat(
new SingleExperimentListener(experiment,
startProcess)));
}
else
{
listeners = from experiment in processes.Experiments.ToObservable()
from result in new SingleExperimentListener(experiment, startProcess)
select result;
}
IDisposable subscription = null;
subscription = listeners.SubscribeSafe(observer);
return Disposable.Create(
() =>
{
if (subscription != null)
{
subscription.Dispose();
subscription = null;
}
processes.Dispose();
if (!string.IsNullOrEmpty(assemblyLocation))
{
File.Delete(assemblyLocation);
}
});
}));
}
public ChartViewModel(PerfLab lab, TestSuiteInfo suiteInfo)
{
this.perfTestConfiguration = new PerfTestConfiguration(Settings.Default.IgnoreFirstRunDueToJITting, Settings.Default.TriggerGCBeforeEachTest);
this.Title = suiteInfo.TestSuiteDescription;
Lab = lab;
TestSuiteInfo = suiteInfo;
this.StartValue = 1;
this.EndValue = suiteInfo.DefaultTestCount;
this.StepValue = 1;
SpeedPlotModel =
new PlotModel(string.Format("\"{0}\": Time characteristics", suiteInfo.TestSuiteDescription));
SpeedPlotModel.Axes.Clear();
SpeedPlotModel.Axes.Add(new LinearAxis(AxisPosition.Bottom, suiteInfo.FeatureDescription));
MemoryPlotModel =
new PlotModel(string.Format("\"{0}\": Memory usage", suiteInfo.TestSuiteDescription));
MemoryPlotModel.Axes.Clear();
MemoryPlotModel.Axes.Add(new LinearAxis(AxisPosition.Bottom, suiteInfo.FeatureDescription));
memorySeries = new Dictionary <TestInfo, LineSeries>();
speedSeries = new Dictionary <TestInfo, LineSeries>();
IsLinear = true;
IsStarted = false;
var errorHandler = IoC.Instance.Resolve <ErrorHandler>();
var whenStarted = this.WhenAny(x => x.IsStarted, x => x.Value);
this.StartSequential = new ReactiveAsyncCommand(whenStarted.Select(x => !x));
StartSequential.RegisterAsyncAction(OnStartSequential, RxApp.DeferredScheduler);
errorHandler.HandleErrors(this.StartSequential);
this.StartParallel = new ReactiveAsyncCommand(whenStarted.Select(x => !x));
StartParallel.RegisterAsyncAction(OnStartParallel, RxApp.DeferredScheduler);
errorHandler.HandleErrors(this.StartParallel);
this.Stop = new ReactiveAsyncCommand(whenStarted);
Stop.RegisterAsyncAction(OnStop, RxApp.DeferredScheduler);
errorHandler.HandleErrors(this.Stop);
this.WhenAny(x => x.IsLinear, x => x.Value ? EAxisType.Linear : EAxisType.Logarithmic)
.Subscribe(SetAxisType);
MessageBus.Current.Listen <PerfTestResult>()
.Where(x => tests.FirstOrDefault(t => t.TestId.Equals(x.TestId)) != null)
.Subscribe(
res =>
{
var nextRes = res as NextResult;
var errorRes = res as ExperimentError;
if (nextRes != null)
{
AddPoint(memorySeries, MemoryPlotModel,
tests.First(x => x.TestId.Equals(res.TestId)), nextRes.Descriptor,
nextRes.MemoryUsage);
AddPoint(speedSeries, SpeedPlotModel,
tests.First(x => x.TestId.Equals(res.TestId)), nextRes.Descriptor,
nextRes.Duration);
}
if (errorRes != null)
{
var test = tests.FirstOrDefault(x => x.TestId.Equals(errorRes.TestId));
Task.Factory.StartNew(() => errorHandler.ReportExperimentError(errorRes, test));
}
}, errorHandler.ReportException);
ConnectIterationAndDescriptors();
}
public static async Task <IActionResult> PublishResult([HttpTrigger(AuthorizationLevel.Function, "post", Route = "publish")] TestRunInputData input, TraceWriter log)
{
try
{
// remove special characters
input.ProjectName = Helpers.RemoveUnsafeChars(input.ProjectName);
input.TestSuiteName = Helpers.RemoveUnsafeChars(input.TestSuiteName);
input.BuildNumber = Helpers.RemoveUnsafeChars(input.BuildNumber);
input.TestFullName = Helpers.RemoveUnsafeChars(input.TestFullName, allowUrlChars: true);
// save test run results
var entity = new TestRun()
{
PartitionKey = TestRun.CreatePartitionKey(input.TestSuiteName, input.BuildNumber),
RowKey = TestRun.CreateRowKey(input.TestFullName),
Attachments = new List <TestRunAttachmentLink>(),
Timestamp = DateTimeOffset.UtcNow,
TestResult = input.TestResult
};
// store test output - if it is short, put it directly in the table; if it is too long, put it in blob storage
if (input.TestOutput.Length <= MaxFieldLength)
{
entity.TestOutput = input.TestOutput;
}
else
{
// prepare URL
var testOutputUrl = GetTestOutputBlobUrl(input.TestSuiteName, input.BuildNumber, input.TestFullName);
// upload test output
var container = await GetBlobContainerForProject(input.ProjectName);
var testOutputBlob = container.GetBlockBlobReference(testOutputUrl);
await testOutputBlob.UploadTextAsync(input.TestOutput);
entity.TestOutput = input.TestOutput.Substring(0, MaxFieldLength);
entity.TestOutputUrl = testOutputUrl;
}
// save attachments to blob storage
if (input.Attachments != null)
{
foreach (var attachment in input.Attachments)
{
// prepare URL
var attachmentUrl = GetTestAttachmentBlobUrl(input.TestSuiteName, input.BuildNumber, input.TestFullName, attachment.FileName);
// upload blob
var container = await GetBlobContainerForProject(input.ProjectName);
var attachmentBlob = container.GetBlockBlobReference(attachmentUrl);
var attachmentData = Convert.FromBase64String(attachment.ContentBase64);
await attachmentBlob.UploadFromByteArrayAsync(attachmentData, 0, attachmentData.Length);
entity.Attachments.Add(new TestRunAttachmentLink()
{
FileName = attachment.FileName,
BlobUrl = attachmentUrl
});
}
}
// store entity
var table = await GetTableForProject(input.ProjectName, createIfNotExists : true);
await table.ExecuteAsync(TableOperation.Insert(entity));
// store index entity
var indexTable = await GetIndexTable(createIfNotExists : true);
await indexTable.ExecuteAsync(TableOperation.InsertOrReplace(new TestSuiteInfo()
{
PartitionKey = TestSuiteInfo.CreatePartitionKey(input.ProjectName),
RowKey = TestSuiteInfo.CreateRowKey(input.TestSuiteName, input.BuildNumber),
Timestamp = DateTimeOffset.UtcNow
}));
return(new OkObjectResult(new TestRunInputResult()
{
TestSuiteUrl = $"{AppBaseUrl}/results/{input.ProjectName}/{input.TestSuiteName}/{input.BuildNumber}",
TestResultUrl = $"{AppBaseUrl}/results/{input.ProjectName}/{input.TestSuiteName}/{input.BuildNumber}#{input.TestFullName}"
}));
}
catch (Exception ex)
{
log.Error(ex.ToString());
return(new BadRequestErrorMessageResult(ex.Message));
}
}
