Exemple #1
0
        public TestSuiteBuilder(TestSuiteInfo testSuiteInfo)
        {
            if (testSuiteInfo == null)
            {
                throw new ArgumentNullException("testSuiteInfo");
            }

            this.testSuiteInfo = testSuiteInfo;

            // get run descriptor
            this.runDescriptor =
                testSuiteInfo.TesterType.MethodsWith(Flags.AllMembers, typeof(PerfRunDescriptorAttribute))
                .FirstOrDefault(
                    m => m.ReturnType == typeof(double) && m.HasParameterSignature(new[] { typeof(int) }));

            // get set up
            this.setUp =
                testSuiteInfo.TesterType.MethodsWith(Flags.AllMembers, typeof(PerfSetUpAttribute))
                .FirstOrDefault(
                    m =>
                    m.ReturnType == typeof(void) &&
                    m.HasParameterSignature(new[] { typeof(int), testSuiteInfo.TestedAbstraction }));

            // get tear down
            this.tearDown =
                testSuiteInfo.TesterType.MethodsWith(Flags.AllMembers, typeof(PerfTearDownAttribute))
                .FirstOrDefault(
                    m =>
                    m.ReturnType == typeof(void) &&
                    m.HasParameterSignature(new[] { testSuiteInfo.TestedAbstraction }));
        }
Exemple #2
0
 public TestSuiteCodeBuilder(
     TestSuiteInfo testSuiteInfo,
     string runDescriptorMethodName,
     string setUpMethodName,
     string tearDownMethodName)
 {
     this.testSuiteInfo           = testSuiteInfo;
     this.runDescriptorMethodName = runDescriptorMethodName;
     this.setUpMethodName         = setUpMethodName;
     this.tearDownMethodName      = tearDownMethodName;
     this.testerType        = new CodeTypeReference(testSuiteInfo.TesterType);
     this.testedAbstraction = new CodeTypeReference(testSuiteInfo.TestedAbstraction);
 }
Exemple #3
0
 public TestNodeViewModel(TesterNodeViewModel parent, PerfLab perfLab, TestSuiteInfo testSuite, string testMethodName)
     : base(parent)
 {
     this.PerfLab   = perfLab;
     this.TestSuite = testSuite;
     this.Tests     = testSuite.Tests.Where(x => x.TestMethodName == testMethodName).ToArray();
     this.Name      = testMethodName;
     this.Children.AddRange(from testInfo in this.Tests
                            select new TestedTypeNodeViewModel(this, perfLab, testInfo)
     {
         IsChecked = false
     });
     this.IsEnabled = this.Children.Count > 0;
 }
        public static void GenerateNUnitCode(TestSuiteInfo testSuite, string targetFilePath)
        {
            var builder = new StringBuilder();

            builder.AppendLine("using NUnit.Framework;");
            builder.AppendLine("namespace TestCodeGenerator");
            builder.AppendLine("{");
            builder.AppendLine("\t[TestFixture]");

            var pattern    = @"\.(\w+)";
            var match      = Regex.Match(testSuite.Target, pattern);
            var methodName = match.Groups[1].Value;
            var className  = "FunctionLib";

            builder.AppendLine($"\tpublic class {className}Tests");
            builder.AppendLine("\t{");
            builder.AppendLine("\t\t[Test]");
            builder.AppendLine($"\t\tpublic void {methodName}Test()");
            builder.AppendLine("\t\t{");

            foreach (var assertion in testSuite.TestCases.SelectMany(c => c.Assertions).ToList())
            {
                var expectedOutput = assertion.ExpectedOutput;
                var argumentList   = string.Join(", ", assertion.InputValues);
                builder.AppendLine(
                    $"\t\t\tAssert.AreEqual(\"{expectedOutput}\", {className}.{methodName}({argumentList}));");
            }

            builder.AppendLine("\t\t}");
            builder.AppendLine("\t}");
            builder.AppendLine("}");

            var fileContent = builder.ToString();

            using (var writer = new StreamWriter(targetFilePath))
            {
                try
                {
                    writer.Write(fileContent);
                    writer.Close();
                }
                catch (Exception ex)
                {
                    throw new Exception(ex.Message);
                }
            }
        }
        public TesterNodeViewModel(PerfLab lab, TestSuiteInfo testSuiteInfo)
            : base(null)
        {
            var descr = testSuiteInfo.TestSuiteDescription;

            this.Name = string.IsNullOrEmpty(descr) ? testSuiteInfo.TesterType.FullName : descr;

            this.TestSuiteInfo = testSuiteInfo;

            this.Lab = lab;

            this.Children.AddRange(
                testSuiteInfo.Tests.GroupBy(x => x.TestMethodName)
                .Select(x => new TestNodeViewModel(this, lab, testSuiteInfo, x.Key)));

            MessageBus.Current.Listen <ChartRemoved>().Subscribe(OnChartRemoved);

            this.IsEnabled = this.Children.Count > 0;
        }
        private TestSuiteInfo GetTestSuiteInfoFromTestSuiteFile(string filePath)
        {
            var       testSuite = new TestSuiteInfo();
            XDocument xdoc;

            try
            {
                xdoc = XDocument.Load(filePath);
            }
            catch (Exception ex)
            {
                throw new Exception(ex.Message);
            }

            testSuite.Name   = xdoc.XPathSelectElement("/TestSuite/Name")?.Value;
            testSuite.Target = xdoc.XPathSelectElement("/TestSuite/Target")?.Value;

            foreach (var testcaseElement in xdoc.XPathSelectElements("/TestSuite/TestCases/TestCase"))
            {
                var testCase = new TestCaseInfo {
                    Id = testcaseElement.Attribute("ID")?.Value
                };

                foreach (var assertionElement in testcaseElement.XPathSelectElements("Assertions/Assertion"))
                {
                    var assertion = new AssertionInfo
                    {
                        Id          = assertionElement.Attribute("ID")?.Value,
                        InputValues = assertionElement.XPathSelectElements("InputValues/InputValue")
                                      .Select(v => Convert.ToDouble(v.Value)).ToList(),
                        ExpectedOutput = assertionElement.XPathSelectElement("ExpectedOutput")?.Value,
                        ActualOutput   = assertionElement.XPathSelectElement("ActualOutput")?.Value,
                        Result         = assertionElement.XPathSelectElement("Result")?.Value
                    };
                    testCase.Assertions.Add(assertion);
                }

                testSuite.TestCases.Add(testCase);
            }

            return(testSuite);
        }
        //生成单元测试代码
        public void GenerateUnitTestFile(TestSuiteInfo testSuite)
        {
            var fileName = $"{testSuite.FunctionName}Test.cs";
            var filePath = Path.Combine(_unitTestFilesDir, fileName);

            try
            {
                //若单元测试代码输出文件夹不存在,则创建该文件夹
                if (!Directory.Exists(_unitTestFilesDir))
                {
                    Directory.CreateDirectory(_unitTestFilesDir);
                }

                UnitTestCodeGenerator.GenerateNUnitCode(testSuite, filePath);
            }
            catch (Exception ex)
            {
                throw new Exception(ex.Message);
            }
        }
        /// <summary>
        /// Load test suite.
        /// </summary>
        /// <param name="filename">Filename of the profile</param>
        public void LoadTestSuite(string filename)
        {
            ProfileUtil   profile = ProfileUtil.LoadProfile(filename);
            TestSuiteInfo tsinfo  = null;

            foreach (var g in testSuites)
            {
                foreach (var info in g)
                {
                    if (profile.VerifyVersion(info.TestSuiteName, info.TestSuiteVersion))
                    {
                        tsinfo = info;
                        goto FindTestSuite;
                    }
                }
            }
FindTestSuite:
            profile.Dispose();
            util.LoadTestSuiteConfig(tsinfo);
            util.LoadTestSuiteAssembly();
            util.LoadProfileSettings(filename);
        }
        private void btnGA_Click(object sender, EventArgs e)
        {
            //路径覆盖测试数据集
            var gaAssertions = new List <AssertionInfo>();
            //边界值测试数据集
            var bounaryTestAssertions = new List <AssertionInfo>();

            //加载参数
            LoadParameters();
            txtResult.Clear();
            //通过遗传算法得到路径覆盖测试数据
            gaAssertions.AddRange(Task.Run(() => GaTestDataGenerator.GetAssertions(_gaParameters, _function, _targetPaths)).Result);
            //得到边界值测试数据
            bounaryTestAssertions.AddRange(Task.Run(() => BoundaryTestDataGenerator.GetAssertions(_function)).Result);

            var testSuite = new TestSuiteInfo
            {
                Name   = $"针对{cmbFunction.Text}函数的测试套件",
                Target = $"{AbstractFunction.CreateInstance(cmbFunction.SelectedValue.ToString())}"
            };

            testSuite.TestCases.Add(new TestCaseInfo {
                Name = "路径覆盖测试", Assertions = gaAssertions
            });
            testSuite.TestCases.Add(new TestCaseInfo {
                Name = "边界值测试", Assertions = bounaryTestAssertions
            });

            GenerateTestSuiteFile(testSuite);
//                        ShowTestData(gaAssertions.Union(bounaryTestAssertions).ToList());
//            ShowAssertions(gaAssertions);

            //将 galog 显示到文本框中
            const string logPath = @"c:\#GA_DEMO\galog.txt";

            txtResult.AppendText(File.ReadAllText(logPath));
            //滚动到光标处
            txtResult.ScrollToCaret();
        }
        //生成测试数据文档
        private void GenerateTestSuiteFile(TestSuiteInfo testSuite)
        {
            var fileName = $"TS_{testSuite.FunctionName}_{DateTime.Now:yyyyMMddHHmmss}.xml";
            var filePath = Path.Combine(_testSutieFilesDir, fileName);

            try
            {
                //若输出文件夹不存在,则创建该文件夹
                if (!Directory.Exists(_testSutieFilesDir))
                {
                    Directory.CreateDirectory(_testSutieFilesDir);
                }

                var xdoc = new XElement("TestSuite", new XElement("Name", testSuite.Name),
                                        new XElement("Target", $"{testSuite.Target}"),
                                        new XElement("TestCases", testSuite.TestCases.Select(c => new XElement("TestCase",
                                                                                                               new XAttribute("ID", $"TC_{testSuite.TestCases.IndexOf(c) + 1:000}"),
                                                                                                               new XElement("Name", c.Name),
                                                                                                               new XElement("Assertions",
                                                                                                                            c.Assertions.Select(a => new XElement("Assertion",
                                                                                                                                                                  new XAttribute("ID", $"TA_{c.Assertions.IndexOf(a) + 1:0000}"),
                                                                                                                                                                  new XElement("InputValues",
                                                                                                                                                                               a.InputValues.Select(v => new XElement("InputValue", v))),
                                                                                                                                                                  new XElement("ExpectedOutput", a.ExpectedOutput),
                                                                                                                                                                  new XElement("ActualOutput", a.ActualOutput),
                                                                                                                                                                  new XElement("Result", a.Result))))))),
                                        new XElement("TestSummery",
                                                     new XElement("Executed", testSuite.TestSummery.Executed),
                                                     new XElement("Passed", testSuite.TestSummery.Passed),
                                                     new XElement("Failed", testSuite.TestSummery.Failed)));
                xdoc.Save(filePath);
            }
            catch (Exception ex)
            {
                throw new Exception(ex.Message);
            }
        }
Exemple #11
0
        private static TestInfo GetTestInfo(Guid id, MethodInfo method, Type testedAbstraction, TestSuiteInfo suiteInfo, Type testedType)
        {
            CheckTestability(suiteInfo.TesterType, testedType);

            var testAttribute = method.Attribute <PerfTestAttribute>();

            if (testAttribute == null)
            {
                throw new ArgumentNullException("method");
            }

            if (method.ReturnType != typeof(void) || !method.HasParameterSignature(new[] { testedAbstraction }))
            {
                throw new ArgumentException("Incorrect parameter signature");
            }

            if (testedType.IsGenericType)
            {
                // Fill the generic type arguments of the loaded generic type
                // with the tested abstraction interface actual generic type arguments.
                // Example: tested abstraction = IList<int>, tested type = List<T>
                // This line converts List<T> in List<int>
                testedType = testedType.MakeGenericType(testedAbstraction.GetGenericArguments());
            }

            TestInfo result;
            var      ignoreAttribute = method.Attribute <PerfIgnoreAttribute>();

            if (ignoreAttribute == null)
            {
                result = new TestInfo
                {
                    TestId          = id,
                    TestDescription = testAttribute.Description,
                    TestMethodName  = method.Name,
                    TestedType      = testedType,
                    Suite           = suiteInfo
                };
            }
            else
            {
                result = new TestInfoIgnored
                {
                    TestId          = id,
                    TestDescription = testAttribute.Description,
                    TestMethodName  = method.Name,
                    IgnoreMessage   = ignoreAttribute.Message,
                    TestedType      = testedType,
                    Suite           = suiteInfo
                };
            }

            return(result);
        }
Exemple #12
0
        private static string BuildTestSuiteAssembly(TestSuiteInfo testSuiteInfo)
        {
            var builder = new TestSuiteBuilder(testSuiteInfo);

            return(builder.Build());
        }
Exemple #13
0
 public static IObservable <PerfTestResult> Run(TestSuiteInfo testSuiteInfo, int start, int step, int end, PerfTestConfiguration configuration, bool parallel = false)
 {
     return(CreateRunObservable(testSuiteInfo, x => true, processes => processes.Start(start, step, end, !parallel), configuration, parallel));
 }
Exemple #14
0
        private static IObservable <PerfTestResult> CreateRunObservable(TestSuiteInfo testSuiteInfo,
                                                                        Predicate <TestInfo> testFilter,
                                                                        Action <ExperimentProcess> startProcess,
                                                                        PerfTestConfiguration configuration,
                                                                        bool parallel = false)
        {
            return(Observable.Create <PerfTestResult>(
                       observer =>
            {
                var assemblyLocation = BuildTestSuiteAssembly(testSuiteInfo);

                var processes = new MultiExperimentProcess(
                    (from testMethod in testSuiteInfo.Tests
                     where testFilter(testMethod)
                     select
                     new ExperimentProcess(
                         string.Format(
                             "{0}.{1}({2})",
                             testSuiteInfo.TesterType.Name,
                             testMethod.TestMethodName,
                             testMethod.TestedType.Name),
                         assemblyLocation,
                         TestSuiteCodeBuilder.TestSuiteClassName,
                         testSuiteInfo.TesterType,
                         testMethod.TestedType,
                         testMethod.TestMethodName,
                         configuration)).ToArray());

                var listeners = Observable.Empty <PerfTestResult>();

                if (!parallel)
                {
                    listeners = processes.Experiments.Aggregate(listeners,
                                                                (current, experiment) =>
                                                                current.Concat(
                                                                    new SingleExperimentListener(experiment,
                                                                                                 startProcess)));
                }
                else
                {
                    listeners = from experiment in processes.Experiments.ToObservable()
                                from result in new SingleExperimentListener(experiment, startProcess)
                                select result;
                }

                IDisposable subscription = null;

                subscription = listeners.SubscribeSafe(observer);

                return Disposable.Create(
                    () =>
                {
                    if (subscription != null)
                    {
                        subscription.Dispose();
                        subscription = null;
                    }

                    processes.Dispose();

                    if (!string.IsNullOrEmpty(assemblyLocation))
                    {
                        File.Delete(assemblyLocation);
                    }
                });
            }));
        }
        public ChartViewModel(PerfLab lab, TestSuiteInfo suiteInfo)
        {
            this.perfTestConfiguration = new PerfTestConfiguration(Settings.Default.IgnoreFirstRunDueToJITting, Settings.Default.TriggerGCBeforeEachTest);

            this.Title = suiteInfo.TestSuiteDescription;

            Lab = lab;

            TestSuiteInfo = suiteInfo;

            this.StartValue = 1;

            this.EndValue = suiteInfo.DefaultTestCount;

            this.StepValue = 1;

            SpeedPlotModel =
                new PlotModel(string.Format("\"{0}\": Time characteristics", suiteInfo.TestSuiteDescription));

            SpeedPlotModel.Axes.Clear();
            SpeedPlotModel.Axes.Add(new LinearAxis(AxisPosition.Bottom, suiteInfo.FeatureDescription));

            MemoryPlotModel =
                new PlotModel(string.Format("\"{0}\": Memory usage", suiteInfo.TestSuiteDescription));

            MemoryPlotModel.Axes.Clear();
            MemoryPlotModel.Axes.Add(new LinearAxis(AxisPosition.Bottom, suiteInfo.FeatureDescription));

            memorySeries = new Dictionary <TestInfo, LineSeries>();
            speedSeries  = new Dictionary <TestInfo, LineSeries>();

            IsLinear  = true;
            IsStarted = false;

            var errorHandler = IoC.Instance.Resolve <ErrorHandler>();

            var whenStarted = this.WhenAny(x => x.IsStarted, x => x.Value);

            this.StartSequential = new ReactiveAsyncCommand(whenStarted.Select(x => !x));
            StartSequential.RegisterAsyncAction(OnStartSequential, RxApp.DeferredScheduler);
            errorHandler.HandleErrors(this.StartSequential);

            this.StartParallel = new ReactiveAsyncCommand(whenStarted.Select(x => !x));
            StartParallel.RegisterAsyncAction(OnStartParallel, RxApp.DeferredScheduler);
            errorHandler.HandleErrors(this.StartParallel);

            this.Stop = new ReactiveAsyncCommand(whenStarted);
            Stop.RegisterAsyncAction(OnStop, RxApp.DeferredScheduler);
            errorHandler.HandleErrors(this.Stop);

            this.WhenAny(x => x.IsLinear, x => x.Value ? EAxisType.Linear : EAxisType.Logarithmic)
            .Subscribe(SetAxisType);

            MessageBus.Current.Listen <PerfTestResult>()
            .Where(x => tests.FirstOrDefault(t => t.TestId.Equals(x.TestId)) != null)
            .Subscribe(
                res =>
            {
                var nextRes  = res as NextResult;
                var errorRes = res as ExperimentError;
                if (nextRes != null)
                {
                    AddPoint(memorySeries, MemoryPlotModel,
                             tests.First(x => x.TestId.Equals(res.TestId)), nextRes.Descriptor,
                             nextRes.MemoryUsage);

                    AddPoint(speedSeries, SpeedPlotModel,
                             tests.First(x => x.TestId.Equals(res.TestId)), nextRes.Descriptor,
                             nextRes.Duration);
                }

                if (errorRes != null)
                {
                    var test = tests.FirstOrDefault(x => x.TestId.Equals(errorRes.TestId));
                    Task.Factory.StartNew(() => errorHandler.ReportExperimentError(errorRes, test));
                }
            }, errorHandler.ReportException);

            ConnectIterationAndDescriptors();
        }
Exemple #16
0
        public static async Task <IActionResult> PublishResult([HttpTrigger(AuthorizationLevel.Function, "post", Route = "publish")] TestRunInputData input, TraceWriter log)
        {
            try
            {
                // remove special characters
                input.ProjectName   = Helpers.RemoveUnsafeChars(input.ProjectName);
                input.TestSuiteName = Helpers.RemoveUnsafeChars(input.TestSuiteName);
                input.BuildNumber   = Helpers.RemoveUnsafeChars(input.BuildNumber);
                input.TestFullName  = Helpers.RemoveUnsafeChars(input.TestFullName, allowUrlChars: true);

                // save test run results
                var entity = new TestRun()
                {
                    PartitionKey = TestRun.CreatePartitionKey(input.TestSuiteName, input.BuildNumber),
                    RowKey       = TestRun.CreateRowKey(input.TestFullName),
                    Attachments  = new List <TestRunAttachmentLink>(),
                    Timestamp    = DateTimeOffset.UtcNow,
                    TestResult   = input.TestResult
                };

                // store test output - if it is short, put it directly in the table; if it is too long, put it in blob storage
                if (input.TestOutput.Length <= MaxFieldLength)
                {
                    entity.TestOutput = input.TestOutput;
                }
                else
                {
                    // prepare URL
                    var testOutputUrl = GetTestOutputBlobUrl(input.TestSuiteName, input.BuildNumber, input.TestFullName);

                    // upload test output
                    var container = await GetBlobContainerForProject(input.ProjectName);

                    var testOutputBlob = container.GetBlockBlobReference(testOutputUrl);
                    await testOutputBlob.UploadTextAsync(input.TestOutput);

                    entity.TestOutput    = input.TestOutput.Substring(0, MaxFieldLength);
                    entity.TestOutputUrl = testOutputUrl;
                }

                // save attachments to blob storage
                if (input.Attachments != null)
                {
                    foreach (var attachment in input.Attachments)
                    {
                        // prepare URL
                        var attachmentUrl = GetTestAttachmentBlobUrl(input.TestSuiteName, input.BuildNumber, input.TestFullName, attachment.FileName);

                        // upload blob
                        var container = await GetBlobContainerForProject(input.ProjectName);

                        var attachmentBlob = container.GetBlockBlobReference(attachmentUrl);
                        var attachmentData = Convert.FromBase64String(attachment.ContentBase64);
                        await attachmentBlob.UploadFromByteArrayAsync(attachmentData, 0, attachmentData.Length);

                        entity.Attachments.Add(new TestRunAttachmentLink()
                        {
                            FileName = attachment.FileName,
                            BlobUrl  = attachmentUrl
                        });
                    }
                }

                // store entity
                var table = await GetTableForProject(input.ProjectName, createIfNotExists : true);

                await table.ExecuteAsync(TableOperation.Insert(entity));

                // store index entity
                var indexTable = await GetIndexTable(createIfNotExists : true);

                await indexTable.ExecuteAsync(TableOperation.InsertOrReplace(new TestSuiteInfo()
                {
                    PartitionKey = TestSuiteInfo.CreatePartitionKey(input.ProjectName),
                    RowKey       = TestSuiteInfo.CreateRowKey(input.TestSuiteName, input.BuildNumber),
                    Timestamp    = DateTimeOffset.UtcNow
                }));

                return(new OkObjectResult(new TestRunInputResult()
                {
                    TestSuiteUrl = $"{AppBaseUrl}/results/{input.ProjectName}/{input.TestSuiteName}/{input.BuildNumber}",
                    TestResultUrl = $"{AppBaseUrl}/results/{input.ProjectName}/{input.TestSuiteName}/{input.BuildNumber}#{input.TestFullName}"
                }));
            }
            catch (Exception ex)
            {
                log.Error(ex.ToString());
                return(new BadRequestErrorMessageResult(ex.Message));
            }
        }