public void ExecuteGamblingModelTest() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Casino", "Digimedia", "Racing", "Seabrook", "Sky City", "Slots", "TAB", "ThePalac", "B365", "Sportsbet" }; var inputs = testData.Select(records => new AnalyzerInput(GamblingModel.Instance, new GamblingInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = records })).ToList(); //Test Singel One var outputs = Analyzer.Instance.Execute(inputs.FirstOrDefault(i => i.ModelInput.BankRecords.Code == "RE85MC")); Assert.IsNotNull(outputs); Assert.IsNotNull(outputs.ModelOutput); Assert.IsTrue(outputs.ModelOutput.Count == 6); Assert.IsNotNull((outputs.ModelOutput as GamblingOverallSummary)); Assert.IsTrue((outputs.ModelOutput as GamblingOverallSummary).GamblingGroupSummaries.Count == 4); //Test Parallel var outputs2 = Analyzer.Instance.ExecuteMultipleModels(inputs); Assert.IsTrue(outputs2.AnySave()); }
public void ExecuteFinanceWithdrawalsModelTest() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Finance", "Loan", "Admiral", "Avanti", "Cashburst", "Cash Converters", "Cash in a Flash", "Cash Relief", "Cash Train", "CC Finance", "Chester", "Ferratum", "Handy Cash", "Harmoney", "Instant Finance", "Loan Plus", "Loans 2 Go", "Payday Advance", "Pretty Penny Loans", "Rapid Loans", "Save My Bacon", "Seed", "Smart Cash", "Smart Shop", "SMB", "Superloans", "Teleloans" }; var inputs = testData.Select(records => new AnalyzerInput(FinanceWithdrawalsModel.Instance, new FinanceWithdrawalsInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = records })).ToList(); //Test Singel One var outputs = Analyzer.Instance.Execute(inputs.FirstOrDefault(i => i.ModelInput.BankRecords.Code == "RE85MC")); Assert.IsNotNull(outputs); Assert.IsNotNull(outputs.ModelOutput); Assert.IsTrue(outputs.ModelOutput.Count == 4); Assert.IsNotNull((outputs.ModelOutput as FinanceWithdrawalsOverallSummary)); Assert.IsTrue((outputs.ModelOutput as FinanceWithdrawalsOverallSummary).GamblingGroupSummaries.Count == 2); //Test Parallel var outputs2 = Analyzer.Instance.ExecuteMultipleModels(inputs); Assert.IsTrue(outputs2.AnySave()); }
public void ExecuteAccountConduceModelTest() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Dishonour Fee", "Dishonour", "Failed", "Reversal", "Unpaid", "Overdraft" }; var inputs = testData.Select(records => new AnalyzerInput(AccountConductModel.Instance, new AccountConductInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = records })).ToList(); //Test Singel One var outputs = Analyzer.Instance.Execute(inputs.FirstOrDefault(i => i.ModelInput.BankRecords.Code == "WVBBKJ")); Assert.IsNotNull(outputs); Assert.IsNotNull(outputs.ModelOutput); Assert.IsTrue(outputs.ModelOutput.Count == 3); Assert.IsNotNull((outputs.ModelOutput as AccountConductOverallSummary)); Assert.IsTrue((outputs.ModelOutput as AccountConductOverallSummary).AccountConductGroupSummaries.Count == 2); //Test Parallel var outputs2 = Analyzer.Instance.ExecuteMultipleModels(inputs); Assert.IsTrue(outputs2.AnySave()); }
public void ExecuteForeignExchangeModelTest() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Currency", "Conversion" }; var inputs = testData.Select(records => new AnalyzerInput(ForeignExchangeModel.Instance, new ForeignExchangeInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = records })).ToList(); //Test Singel One var outputs = Analyzer.Instance.Execute(inputs.FirstOrDefault(i => i.ModelInput.BankRecords.Code == "SNCT7W")); Assert.IsNotNull(outputs); Assert.IsNotNull(outputs.ModelOutput); Assert.IsTrue(outputs.ModelOutput.Count == 7); Assert.IsNotNull((outputs.ModelOutput as ForeignExchangeOverallSummary)); Assert.IsTrue((outputs.ModelOutput as ForeignExchangeOverallSummary).ForeignExchangeGroupSummaries.Count == 1); //Test Parallel var outputs2 = Analyzer.Instance.ExecuteMultipleModels(inputs); Assert.IsTrue(outputs2.AnySave()); }
/// <summary> /// Distributes tests to a set of distribution keys that were previously /// registered via RegisterKey. The distribution directory specified for /// RegisterKey and Distribute must match. /// </summary> /// <param name="records">Tests to distribute.</param> /// <param name="distributionDirectory">Directory under which tests will be distributed.</param> /// <param name="strategyPrefix">Prefix of strategy to distribute the tests.</param> /// <param name="testBinariesDirectory">Location of test binaries.</param> public static void Distribute(TestRecords records, DirectoryInfo distributionDirectory, string strategyPrefix, DirectoryInfo testBinariesDirectory) { // Directory name == distribution key DirectoryInfo[] subdirectories = distributionDirectory.GetDirectories(); MachineRecord[] machines = subdirectories.Select(directory => (MachineRecord)ObjectSerializer.Deserialize(new XmlTextReader(Path.Combine(directory.FullName, "MachineRecord.xml")), typeof(MachineRecord), null)).ToArray(); int subdirectoriesCount = subdirectories.Length; if (subdirectoriesCount == 0) { throw new ApplicationException("You should execute RegisterForDistribution before DiscoverAndDistribute."); } if (records.ExecutionGroupRecords.Count > 0) { throw new ApplicationException("Execution Group Records are already populated - tests appear to already have run."); } List <TestRecords> partitionedTests = null; DistributionStrategy strategy = DistributionStrategy.CreateDistributionStrategy(strategyPrefix); partitionedTests = strategy.PartitionTests(records, machines, testBinariesDirectory); for (int i = 0; i < subdirectoriesCount; i++) { // Save each subset of tests to a directory for each registered key. partitionedTests[i].Save(subdirectories[i]); } }
public void GamblingModelTestSuccess() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Casino", "Digimedia", "Racing", "Seabrook", "Sky City", "Slots", "TAB", "ThePalac", "B365", "Sportsbet" }; var recorddata = testData.FirstOrDefault(data => data.Code == "RE85MC"); var input = new GamblingInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = recorddata }; var outputs = GamblingModel.Instance.Analyze(input); Assert.IsNotNull(outputs as GamblingOverallSummary); Assert.IsTrue((outputs as GamblingOverallSummary).GamblingGroupSummaries.Count == 4); Assert.IsTrue((outputs as GamblingOverallSummary).Count == 6); }
public void AccountConductModelTestSuccess() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Dishonour Fee", "Dishonour", "Failed", "Reversal", "Unpaid", "Overdraft" }; var recorddata = testData.FirstOrDefault(data => data.Code == "WVBBKJ"); var input = new AccountConductInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = recorddata }; var outputs = AccountConductModel.Instance.Analyze(input); Assert.IsNotNull(outputs as AccountConductOverallSummary); Assert.IsTrue((outputs as AccountConductOverallSummary).AccountConductGroupSummaries.Count == 2); Assert.IsTrue((outputs as AccountConductOverallSummary).Count == 3); }
/// <summary> /// Generates a complete set of static/non-interactive run reports for offline analysis. /// </summary> internal static void GenerateXmlReport(TestRecords tests, RunInfo runInfo, DirectoryInfo reportRoot, DirectoryInfo testBinRoot) { Profiler.StartMethod(); ReportingUtilities.CreateScorchedDirectory(reportRoot); CopyStyleSheets(reportRoot, testBinRoot); TestRecords executedTests; TestRecords filteredTests; SeparateExecutedTests(tests, out executedTests, out filteredTests); //Note: Summary Report may well be superceded by the more elaborate Run Report... SummaryReportGenerator.Generate(executedTests, reportRoot); MachineSummaryReportGenerator.Generate(executedTests, reportRoot); VariationReportGenerator.Generate(executedTests, reportRoot); DrtReportGenerator.Generate(executedTests, reportRoot); InfraTrackingReportGenerator.Generate(executedTests, reportRoot); RunReportGenerator.Generate(executedTests, runInfo, reportRoot); XUnitReportGenerator.Generate(executedTests, reportRoot); FilteringReportGenerator.Generate(filteredTests, reportRoot); Profiler.EndMethod(); }
/// <summary> /// Encapsulates logic for reporting results for non-interactive analysis. /// </summary> public override void Execute() { TestRecords tests = TestRecords.Load(RunDirectory); RunInfo runInfo = RunInfo.Load(RunDirectory); tests.GenerateXmlReport(tests, runInfo, ReportDirectory, TestBinariesDirectory); }
internal static void WriteSummaryReport(XmlTableWriter tableWriter, TestRecords tests) { // Dictionary <string, MachineSummaryEntry> SummaryTable = ProduceMachineSummaries(tests); tableWriter.WriteStartElement("Summary"); foreach (MachineSummaryEntry entry in SummaryTable.Values) { tableWriter.WriteStartElement("MachineSummary"); tableWriter.WriteAttributeString("MachineName", entry.Name); tableWriter.WriteAttributeString("FailingVariations", entry.FailedVariations.ToString(CultureInfo.InvariantCulture)); tableWriter.WriteAttributeString("TotalVariations", (entry.TotalVariations).ToString(CultureInfo.InvariantCulture)); tableWriter.WriteAttributeString("TestsWithoutVariation", (entry.TestsWithoutVariation).ToString(CultureInfo.InvariantCulture)); float passRate = 0; float adjustedPassRate = 0; if (entry.TotalVariations > 0) { passRate = (((entry.TotalVariations - entry.FailedVariations) / (float)entry.TotalVariations) * 100); //Failures on tests with known bugs can be treated as passing, but we make clear this is not the actual pass rate. adjustedPassRate = (((entry.TotalVariations - entry.FailedVariations + entry.FailedVariationsWithBugs) / (float)entry.TotalVariations) * 100); } tableWriter.WriteAttributeString("PassRate", passRate.ToString("0.00", CultureInfo.InvariantCulture)); tableWriter.WriteAttributeString("AdjustedPassRate", adjustedPassRate.ToString("0.00", CultureInfo.InvariantCulture)); tableWriter.WriteAttributeString("TestExecutionTime", ReportingUtilities.FormatTimeSpanAsHms(entry.TestExecutionTime)); tableWriter.WriteEndElement(); } tableWriter.WriteEndElement(); }
/// <summary> /// Encapsulates logic for executing. /// </summary> public override void Execute() { DirectoryInfo distributedExecutionDirectory = TestRecords.GetDistributedDirectory(DistributionKey, RunDirectory); TestRecords tests = TestRecords.Load(distributedExecutionDirectory); ExecutionSettings settings = new ExecutionSettings(); settings.Tests = tests; settings.TestBinariesDirectory = TestBinariesDirectory; settings.DebugTests = DebugTests; settings.DebugSti = DebugSti; settings.WaitForDebugger = WaitForDebugger; settings.LogFilesPath = distributedExecutionDirectory; settings.JitDebuggerCommand = JitDebuggerCommand; settings.TimeoutMultiplier = TimeoutMultiplier; settings.ContinueExecution = ContinueExecution; settings.CodeCoverageEnabled = CodeCoverage; settings.CodeCoverageImport = CodeCoverageImport; settings.RerunFailures = RerunFailures; settings.SkipDxDiag = SkipDxDiag; CodeCoverageUtilities.ValidateForCodeCoverage(CodeCoverage, CodeCoverageImport); tests.Execute(settings); tests.Save(distributedExecutionDirectory); ExecutionBackupStore.ClearAllIntermediateTestResults(settings.LogFilesPath); tests.DisplayConsoleSummary(); }
public void ForeignExchangeModelTestSuccess() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Currency", "Conversion" }; var recorddata = testData.FirstOrDefault(data => data.Code == "SNCT7W"); var input = new ForeignExchangeInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = recorddata }; var outputs = ForeignExchangeModel.Instance.Analyze(input); Assert.IsNotNull(outputs); Assert.IsNotNull(outputs as ForeignExchangeOverallSummary); Assert.IsTrue((outputs as ForeignExchangeOverallSummary).ForeignExchangeGroupSummaries.Count == 1); Assert.IsTrue((outputs as ForeignExchangeOverallSummary).Count == 7); }
public void FinanceWithdrawalsModellTestSuccess() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var searchTerm = new List <string> { "Finance", "Loan", "Admiral", "Avanti", "Cashburst", "Cash Converters", "Cash in a Flash", "Cash Relief", "Cash Train", "CC Finance", "Chester", "Ferratum", "Handy Cash", "Harmoney", "Instant Finance", "Loan Plus", "Loans 2 Go", "Payday Advance", "Pretty Penny Loans", "Rapid Loans", "Save My Bacon", "Seed", "Smart Cash", "Smart Shop", "SMB", "Superloans", "Teleloans" }; var recorddata = testData.FirstOrDefault(data => data.Code == "RE85MC"); var input = new FinanceWithdrawalsInput() { DateRangeInDays = 90, FilterTerms = searchTerm, BankRecords = recorddata }; var outputs = FinanceWithdrawalsModel.Instance.Analyze(input); Assert.IsNotNull(outputs as FinanceWithdrawalsOverallSummary); Assert.IsTrue((outputs as FinanceWithdrawalsOverallSummary).GamblingGroupSummaries.Count == 2); Assert.IsTrue((outputs as FinanceWithdrawalsOverallSummary).Count == 4); }
private void _Recipe_TestRecordAdded(object sender, TestRecordAddedEventArgs e) { if (e.IsFirst) { TestRecords.Add(new TestRecordViewModel(e.NewTestRecord)); } }
/// <summary> /// Encapsulates logic of merging results. /// </summary> public override void Execute() { CodeCoverageUtilities.ValidateForCodeCoverage(CodeCoverage, CodeCoverageImport); TestRecords.Merge(RunDirectory, CodeCoverage, CodeCoverageImport); //Record the lab Run information at this stage. We assume homogeneous configuration. RunInfo.FromEnvironment().Save(RunDirectory); }
internal static void Generate(TestRecords tests, DirectoryInfo ReportRoot) { using (XmlTableWriter tableWriter = new XmlTableWriter(Path.Combine(ReportRoot.FullName, "MachineSummary.xml"))) { tableWriter.AddXsl("MachineSummary.xsl"); WriteSummaryReport(tableWriter, tests); } }
public void Init(object instance, MethodBase method, object[] args) { if (null == method) { throw new ArgumentNullException("method"); } TestRecords.RecordInit(instance, method.DeclaringType.FullName + "." + method.Name, args.Length); }
/// <summary> /// Filter a collection of test cases. Goes through each TestRecord /// and verifies its TestInfo passes the filter expression. If it /// does not, marks the TestRecord as not enabled for execution, /// and provides an explanation. The explanation is inferred by /// parsing the expression tree, so the quality of the explanation /// can be variable. If a collection of test cases has been filtered /// before, the results of that filter are cleared. /// </summary> /// <param name="filteringSettings">Filter to evaluate TestRecord against.</param> /// <param name="testRecords">Set of TestRecords to filter.</param> /// <param name="testBinariesDirectory"/> public static void Filter(FilteringSettings filteringSettings, TestRecords testRecords, DirectoryInfo testBinariesDirectory) { filteringSettings.EnsureVersions(); foreach (TestRecord testRecord in testRecords.TestCollection) { FilterTestInfo(testRecord, filteringSettings); } }
/// <summary> /// Encapsulates logic for discovering and distributing tests. /// </summary> public override void Execute() { // Area is inherited from FilterableCommand - it is used for both // target filtering and filtering of discovered tests. FilteringSettings filteringSettings = FilteringSettings; TestRecords allTests = TestRecords.Discover(DiscoveryInfoPath, filteringSettings); allTests.Filter(filteringSettings, DiscoveryInfoPath.Directory); allTests.Distribute(RunDirectory, DistributionStrategy, DiscoveryInfoPath.Directory); }
internal static void Generate(TestRecords records, DirectoryInfo ReportRoot) { TestRecords drtVariations = FilterDrts(records); string path = Path.Combine(ReportRoot.FullName, "DrtReport.xml"); //Only produce this view if DRT's were executed. if (drtVariations.TestCollection.Count > 0) { Generate(drtVariations, path); } }
public void GetTestDataShallReturnTestData() { var testData = TestRecords.GetTestData(); Assert.IsNotNull(testData); var count = testData.SelectMany(i => i.Records).Count(); //5700 should be enough eh? Assert.IsTrue(count > 0); }
/// <summary> /// Returns a list of test for each methodVariant associated with the Report /// generated from the entity collections loaded in the instance /// </summary> /// <returns>An IEnumerable of Tuples where Value1 is a methodVAriant and /// Value2 is an IEnumerable of tests</returns> public IEnumerable <Tuple <MethodVariant, IEnumerable <Test> > > GetResultCollection() { List <Tuple <MethodVariant, IEnumerable <Test> > > output = new List <Tuple <MethodVariant, IEnumerable <Test> > >(); foreach (MethodVariant mtdvar in MethodVariants) { IEnumerable <Test> testList = TestRecords.SelectMany(tstr => tstr.Tests) .Where(tst => tst.MethodVariantID == mtdvar.ID); output.Add(new Tuple <MethodVariant, IEnumerable <Test> >(mtdvar, testList)); } return(output); }
/// <summary> /// Provides the Pass rate as a percentage string of 0-100%. /// </summary> /// <param name="tests"></param> /// <returns></returns> internal static string CalculatePassRate(TestRecords tests) { int failures = 0; int total = 0; foreach (TestRecord test in tests.TestCollection) { foreach (VariationRecord variation in test.Variations) { failures += OneForFail(variation.Result); total += OneForCountable(variation.Result); } } return(String.Format("{0:0.00}%", (1 - (failures / (float)total)) * 100)); }
/// <summary> /// Merges the results of distributed execution back together. /// </summary> /// <param name="distributionDirectory">Root distribution directory for the full set of tests.</param> /// /// <param name="useCodeCoverage">Specifies if distribution should account for code coverage data</param> /// <param name="codeCoverageConnection">Connection string for Code Coverage Database</param> public static void Merge(DirectoryInfo distributionDirectory, bool useCodeCoverage, string codeCoverageConnection) { DirectoryInfo[] subdirectories = distributionDirectory.GetDirectories(); TestRecords mergedTestResults = new TestRecords(); for (int i = 0; i < subdirectories.Length; i++) { TestRecords deserializedTestResults = TestRecords.Load(subdirectories[i]); // If for a testcollection it looks like nothing was run, this // suggests badness and we want to halt the merge. If we proceed // along with a merge, we end up deleted the TestCollection // subsets, making the run unsalvagable. By failing hard here, // the culprit set of tests can be rerun, and then merge // attempted again. if (!ValidateTestsWereExecuted(deserializedTestResults)) { throw new ApplicationException("TestCollection located in the " + subdirectories[i].FullName + " directory did not have any results. It is highly likely that execution on the associated machine failed. Aborting merge."); } foreach (TestRecord test in deserializedTestResults.TestCollection) { mergedTestResults.TestCollection.Add(test); } foreach (ExecutionGroupRecord group in deserializedTestResults.ExecutionGroupRecords) { mergedTestResults.ExecutionGroupRecords.Add(group); } } mergedTestResults.Save(distributionDirectory); if (useCodeCoverage) { CodeCoverageUtilities.MergeCodeCoverage(subdirectories, distributionDirectory); Console.WriteLine("Attempting to Upload Results."); CodeCoverageUtilities.UploadCodeCoverage(distributionDirectory, codeCoverageConnection); CodeCoverageUtilities.DeleteCodeCoverageMergeInputs(subdirectories); } // Once we have safely merged the test collections and saved the merged // TestCollection to disk, delete those distributed subsets. for (int i = 0; i < subdirectories.Length; i++) { TestRecords.Delete(subdirectories[i]); } }
internal static void Generate(TestRecords tests, DirectoryInfo ReportRoot) { DirectoryInfo areasDirectory = new DirectoryInfo(Path.Combine(ReportRoot.FullName, "AreaReports")); ReportingUtilities.CreateAreaReportsDirectories(areasDirectory); Dictionary <string, Dictionary <string, TestCollection> > areas = ReportingUtilities.GroupByArea(tests); foreach (KeyValuePair <string, Dictionary <string, TestCollection> > bucket in areas) { string area = bucket.Key; Dictionary <string, TestCollection> areaDictionary = bucket.Value; Generate(areaDictionary, areasDirectory.FullName, area); } }
protected virtual StageResult Load(int?recordBatchSize = null, int?recordLimit = null, Dictionary <string, string> options = null) { Contract.Requires(InputRecords.Count > 0); float s = TrainingTestSplit; float c = InputRecords.Count; int splitCount = (int)((1f / s) * c); for (int i = 0; i < splitCount; i++) { TestRecords.Add(InputRecords[i]); } for (int i = splitCount; i < InputRecords.Count; i++) { TrainingRecords.Add(InputRecords[i]); } return(StageResult.SUCCESS); }
/// <summary> /// Partitions tests into two buckets: executed cases and filtered cases. Using out args to avoid doing same work twice. /// </summary> private static void SeparateExecutedTests(TestRecords tests, out TestRecords executedTests, out TestRecords filteredTests) { executedTests = new TestRecords(); filteredTests = new TestRecords(); foreach (TestRecord test in tests.TestCollection) { if (test.ExecutionEnabled) { executedTests.TestCollection.Add(test); } else { filteredTests.TestCollection.Add(test); } } executedTests.ExecutionGroupRecords = tests.ExecutionGroupRecords; }
private static bool ValidateTestsWereExecuted(TestRecords results) { // If for every TestRecord Execution was not Enabled, then nothing // was expected to be run, so we're fine. if (results.TestCollection.All(testCollection => !testCollection.ExecutionEnabled)) { return(true); } // If for every TestRecord there were no variations, it looks like // nothing was run. if (results.TestCollection.All(testCollection => testCollection.Variations.Count == 0)) { return(false); } return(true); }
/// <summary> /// Pushes the current Instance to the database updating all the values /// </summary> /// <param name="updateTests">If true all the related SubTest entities are updated too</param> public void Update(bool updateTests = false) { using (LabDbEntities entities = new LabDbEntities()) { entities.ExternalReports.AddOrUpdate(this); if (updateTests) { foreach (SubTest sts in TestRecords.SelectMany(tsr => tsr.Tests) .SelectMany(tst => tst.SubTests) .ToList()) { entities.SubTests.AddOrUpdate(sts); } } entities.SaveChanges(); } }
private static void Generate(TestRecords Records, string path) { using (XmlTableWriter tableWriter = new XmlTableWriter(path)) { tableWriter.AddXsl(@"FilteringReport.xsl"); tableWriter.WriteStartElement("Tests"); foreach (TestRecord test in Records.TestCollection) { TestInfo testInfo = test.TestInfo; { tableWriter.WriteStartElement("Test"); tableWriter.WriteAttributeString("Area", testInfo.Area); tableWriter.WriteAttributeString("SubArea", testInfo.SubArea); tableWriter.WriteAttributeString("TestName", testInfo.Name); tableWriter.WriteAttributeString("Explanation", test.FilteringExplanation); tableWriter.WriteEndElement(); } } tableWriter.WriteEndElement(); } }