/// <summary>
/// Starts a mutation test run
/// </summary>
/// <exception cref="StrykerInputException">For managed exceptions</exception>
/// <param name="options">The user options</param>
public StrykerRunResult RunMutationTest(StrykerOptions options)
{
// start stopwatch
var stopwatch = new Stopwatch();
stopwatch.Start();
// Create output dir with gitignore
FileSystem.Directory.CreateDirectory(options.OutputPath);
FileSystem.File.Create(Path.Combine(options.OutputPath, ".gitignore")).Close();
using (var file = FileSystem.File.CreateText(Path.Combine(options.OutputPath, ".gitignore")))
{
file.WriteLine("*");
}
// setup logging
ApplicationLogging.ConfigureLogger(options.LogOptions);
var logger = ApplicationLogging.LoggerFactory.CreateLogger <StrykerRunner>();
logger.LogDebug("Stryker started with options: {0}",
JsonConvert.SerializeObject(options, new StringEnumConverter()));
try
{
// initialize
Reporter = ReporterFactory.Create(options);
InitialisationProcess = InitialisationProcess ?? new InitialisationProcess();
Input = InitialisationProcess.Initialize(options);
MutationTestProcess = MutationTestProcess ?? new MutationTestProcess(
mutationTestInput: Input,
reporter: Reporter,
mutationTestExecutor: new MutationTestExecutor(Input.TestRunner));
// initial test
Input.TimeoutMs = InitialisationProcess.InitialTest(options);
// mutate
MutationTestProcess.Mutate(options);
// coverage
var coverage = InitialisationProcess.GetCoverage(options);
MutationTestProcess.Optimize(coverage);
// test mutations and return results
return(MutationTestProcess.Test(options));
}
catch (Exception ex) when(!(ex is StrykerInputException))
{
logger.LogError(ex, "An error occurred during the mutation test run ");
throw;
}
finally
{
// log duration
stopwatch.Stop();
logger.LogInformation("Time Elapsed {0}", stopwatch.Elapsed);
}
}
public void InitialisationProcess_ShouldCallNeededResolvers()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var inputFileResolverMock = new Mock <IInputFileResolver>(MockBehavior.Strict);
var initialBuildProcessMock = new Mock <IInitialBuildProcess>(MockBehavior.Strict);
var initialTestProcessMock = new Mock <IInitialTestProcess>(MockBehavior.Strict);
var assemblyReferenceResolverMock = new Mock <IAssemblyReferenceResolver>(MockBehavior.Strict);
testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>(), null, null))
.Returns(new TestRunResult(true)); // testrun is successful
testRunnerMock.Setup(x => x.DiscoverNumberOfTests()).Returns(999);
testRunnerMock.Setup(x => x.Dispose());
var projectContents = new CsharpFolderComposite();
projectContents.Add(new CsharpFileLeaf
{
Name = "SomeFile.cs"
});
var folder = new CsharpFolderComposite
{
Name = "ProjectRoot"
};
folder.AddRange(new Collection <IProjectComponent>
{
new CsharpFileLeaf
{
Name = "SomeFile.cs"
}
});
inputFileResolverMock.Setup(x => x.ResolveInput(It.IsAny <IStrykerOptions>()))
.Returns(new ProjectInfo
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(
references: new string[0]).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(
projectFilePath: "C://Example/Dir/ProjectFolder").Object
},
ProjectContents = folder
});
initialTestProcessMock.Setup(x => x.InitialTest(It.IsAny <ITestRunner>())).Returns(999);
initialBuildProcessMock.Setup(x => x.InitialBuild(It.IsAny <bool>(), It.IsAny <string>(), It.IsAny <string>()));
assemblyReferenceResolverMock.Setup(x => x.LoadProjectReferences(It.IsAny <string[]>()))
.Returns(Enumerable.Empty <PortableExecutableReference>());
var target = new InitialisationProcess(
inputFileResolverMock.Object,
initialBuildProcessMock.Object,
initialTestProcessMock.Object,
testRunnerMock.Object,
assemblyReferenceResolverMock.Object);
var options = new StrykerOptions();
var result = target.Initialize(options);
inputFileResolverMock.Verify(x => x.ResolveInput(It.IsAny <IStrykerOptions>()), Times.Once);
}
public void InitialisationProcess_ShouldThrowOnFailedInitialTestRun()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var inputFileResolverMock = new Mock <IInputFileResolver>(MockBehavior.Strict);
var initialBuildProcessMock = new Mock <IInitialBuildProcess>(MockBehavior.Strict);
var initialTestProcessMock = new Mock <IInitialTestProcess>(MockBehavior.Strict);
var assemblyReferenceResolverMock = new Mock <IAssemblyReferenceResolver>(MockBehavior.Strict);
testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>(), null, null));
var folder = new FolderComposite
{
Name = "ProjectRoot"
};
folder.Add(new FileLeaf
{
Name = "SomeFile.cs"
});
inputFileResolverMock.Setup(x => x.ResolveInput(It.IsAny <StrykerOptions>())).Returns(
new ProjectInfo
{
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
References = new string[0]
},
TestProjectAnalyzerResults = new List <ProjectAnalyzerResult> {
new ProjectAnalyzerResult(null, null)
{
ProjectFilePath = "C://Example/Dir/ProjectFolder"
}
},
ProjectContents = folder
});
initialBuildProcessMock.Setup(x => x.InitialBuild(It.IsAny <bool>(), It.IsAny <string>(), It.IsAny <string>()));
testRunnerMock.Setup(x => x.DiscoverNumberOfTests()).Returns(999);
testRunnerMock.Setup(x => x.Dispose());
initialTestProcessMock.Setup(x => x.InitialTest(It.IsAny <ITestRunner>())).Throws(new StrykerInputException("")); // failing test
assemblyReferenceResolverMock.Setup(x => x.LoadProjectReferences(It.IsAny <string[]>()))
.Returns(Enumerable.Empty <PortableExecutableReference>())
.Verifiable();
var target = new InitialisationProcess(
inputFileResolverMock.Object,
initialBuildProcessMock.Object,
initialTestProcessMock.Object,
testRunnerMock.Object,
assemblyReferenceResolverMock.Object);
var options = new StrykerOptions();
target.Initialize(options);
Assert.Throws <StrykerInputException>(() => target.InitialTest(options, out var nbTests));
inputFileResolverMock.Verify(x => x.ResolveInput(It.IsAny <StrykerOptions>()), Times.Once);
assemblyReferenceResolverMock.Verify();
initialTestProcessMock.Verify(x => x.InitialTest(testRunnerMock.Object), Times.Once);
}
public void InitialisationProcess_ShouldCallNeededResolvers()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var inputFileResolverMock = new Mock <IInputFileResolver>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var initialBuildProcessMock = new Mock <IInitialBuildProcess>(MockBehavior.Strict);
var initialTestProcessMock = new Mock <IInitialTestProcess>(MockBehavior.Strict);
var assemblyReferenceResolverMock = new Mock <IAssemblyReferenceResolver>(MockBehavior.Strict);
testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>(), It.IsAny <int>()))
.Returns(new TestRunResult {
Success = true
}); // testrun is successful
reporterMock.Setup(x => x.OnInitialisationStarted()).Verifiable();
reporterMock.Setup(x => x.OnInitialTestRunStarted()).Verifiable();
reporterMock.Setup(x => x.OnInitialBuildStarted()).Verifiable();
reporterMock.Setup(x => x.OnInitialisationDone()).Verifiable();
inputFileResolverMock.Setup(x => x.ResolveInput(It.IsAny <string>(), It.IsAny <string>()))
.Returns(new Core.Initialisation.ProjectInfo
{
TestProjectPath = "c:/test",
ProjectContents = new FolderComposite
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>
{
new FileLeaf
{
Name = "SomeFile.cs"
}
}
},
ProjectUnderTestAssemblyName = "ExampleProject.dll",
ProjectUnderTestPath = @"c:\ExampleProject\",
TargetFramework = "netcoreapp2.0"
});
initialTestProcessMock.Setup(x => x.InitialTest(It.IsAny <ITestRunner>())).Returns(999);
initialBuildProcessMock.Setup(x => x.InitialBuild(It.IsAny <string>(), It.IsAny <string>()));
assemblyReferenceResolverMock.Setup(x => x.ResolveReferences(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Enumerable.Empty <PortableExecutableReference>());
var target = new InitialisationProcess(reporterMock.Object,
inputFileResolverMock.Object,
initialBuildProcessMock.Object,
initialTestProcessMock.Object,
testRunnerMock.Object,
assemblyReferenceResolverMock.Object);
var options = new StrykerOptions("c:/test", "Console", "", 2000, null, false, 1, 80, 60, 0);
var result = target.Initialize(options);
inputFileResolverMock.Verify(x => x.ResolveInput(It.IsAny <string>(), It.IsAny <string>()), Times.Once);
reporterMock.VerifyAll();
}
public void InitialisationProcess_ShouldCallNeededResolvers()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var inputFileResolverMock = new Mock <IInputFileResolver>(MockBehavior.Strict);
var initialBuildProcessMock = new Mock <IInitialBuildProcess>(MockBehavior.Strict);
var initialTestProcessMock = new Mock <IInitialTestProcess>(MockBehavior.Strict);
var assemblyReferenceResolverMock = new Mock <IAssemblyReferenceResolver>(MockBehavior.Strict);
testRunnerMock.Setup(x => x.RunAll(It.IsAny <ITimeoutValueCalculator>(), null, null))
.Returns(new TestRunResult(true)); // testrun is successful
testRunnerMock.Setup(x => x.DiscoverTests()).Returns(new TestSet());
testRunnerMock.Setup(x => x.Dispose());
var projectContents = new CsharpFolderComposite();
projectContents.Add(new CsharpFileLeaf());
var folder = new CsharpFolderComposite();
folder.AddRange(new Collection <IProjectComponent>
{
new CsharpFileLeaf()
});
inputFileResolverMock.Setup(x => x.ResolveInput(It.IsAny <StrykerOptions>()))
.Returns(new ProjectInfo(new MockFileSystem())
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(
references: new string[0]).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(
projectFilePath: "C://Example/Dir/ProjectFolder",
targetFramework: "netcoreapp2.1").Object
},
ProjectContents = folder
});
initialTestProcessMock.Setup(x => x.InitialTest(It.IsAny <StrykerOptions>(), It.IsAny <ITestRunner>())).Returns(new InitialTestRun(new TestRunResult(true), new TimeoutValueCalculator(1)));
initialBuildProcessMock.Setup(x => x.InitialBuild(It.IsAny <bool>(), It.IsAny <string>(), It.IsAny <string>(), null));
assemblyReferenceResolverMock.Setup(x => x.LoadProjectReferences(It.IsAny <string[]>()))
.Returns(Enumerable.Empty <PortableExecutableReference>());
var target = new InitialisationProcess(
inputFileResolverMock.Object,
initialBuildProcessMock.Object,
initialTestProcessMock.Object,
testRunnerMock.Object,
assemblyReferenceResolverMock.Object);
var options = new StrykerOptions
{
ProjectName = "TheProjectName",
ProjectVersion = "TheProjectVersion"
};
var result = target.Initialize(options);
inputFileResolverMock.Verify(x => x.ResolveInput(It.IsAny <StrykerOptions>()), Times.Once);
}
public void InitialisationProcess_ShouldCallNeededResolvers()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var inputFileResolverMock = new Mock <IInputFileResolver>(MockBehavior.Strict);
var initialBuildProcessMock = new Mock <IInitialBuildProcess>(MockBehavior.Strict);
var initialTestProcessMock = new Mock <IInitialTestProcess>(MockBehavior.Strict);
var assemblyReferenceResolverMock = new Mock <IAssemblyReferenceResolver>(MockBehavior.Strict);
testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>(), It.IsAny <int>()))
.Returns(new TestRunResult {
Success = true
}); // testrun is successful
inputFileResolverMock.Setup(x => x.ResolveInput(It.IsAny <StrykerOptions>()))
.Returns(new ProjectInfo
{
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
References = new string[0]
},
TestProjectAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
ProjectFilePath = "C://Example/Dir/ProjectFolder"
},
ProjectContents = new FolderComposite
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>
{
new FileLeaf
{
Name = "SomeFile.cs"
}
}
}
});
initialTestProcessMock.Setup(x => x.InitialTest(It.IsAny <ITestRunner>())).Returns(999);
initialBuildProcessMock.Setup(x => x.InitialBuild(It.IsAny <bool>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()));
assemblyReferenceResolverMock.Setup(x => x.LoadProjectReferences(It.IsAny <string[]>()))
.Returns(Enumerable.Empty <PortableExecutableReference>());
var target = new InitialisationProcess(
inputFileResolverMock.Object,
initialBuildProcessMock.Object,
initialTestProcessMock.Object,
testRunnerMock.Object,
assemblyReferenceResolverMock.Object);
var options = new StrykerOptions();
var result = target.Initialize(options);
inputFileResolverMock.Verify(x => x.ResolveInput(It.IsAny <StrykerOptions>()), Times.Once);
}
public void InitialisationProcess_ShouldThrowOnFailedInitialTestRun()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var inputFileResolverMock = new Mock <IInputFileResolver>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var initialBuildProcessMock = new Mock <IInitialBuildProcess>(MockBehavior.Strict);
var initialTestProcessMock = new Mock <IInitialTestProcess>(MockBehavior.Strict);
var assemblyReferenceResolverMock = new Mock <IAssemblyReferenceResolver>(MockBehavior.Strict);
testRunnerMock.Setup(x => x.RunAll(It.IsAny <int>()));
reporterMock.Setup(x => x.OnInitialisationStarted()).Verifiable();
reporterMock.Setup(x => x.OnInitialTestRunStarted()).Verifiable();
reporterMock.Setup(x => x.OnInitialBuildStarted()).Verifiable();
inputFileResolverMock.Setup(x => x.ResolveInput(It.IsAny <string>(), It.IsAny <string>())).Returns(
new Core.Initialisation.ProjectInfo()
{
TestProjectPath = "c:/test",
ProjectContents = new FolderComposite()
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "SomeFile.cs"
}
}
},
ProjectUnderTestAssemblyName = "ExampleProject.dll",
ProjectUnderTestPath = @"c:\ExampleProject\",
TargetFramework = "netcoreapp2.0"
});
initialBuildProcessMock.Setup(x => x.InitialBuild(It.IsAny <string>(), It.IsAny <string>()));
initialTestProcessMock.Setup(x => x.InitialTest(It.IsAny <ITestRunner>())).Throws(new InitialTestRunFailedException("")); // failing test
assemblyReferenceResolverMock.Setup(x => x.ResolveReferences(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(Enumerable.Empty <PortableExecutableReference>()).Verifiable();
var target = new InitialisationProcess(reporterMock.Object,
inputFileResolverMock.Object,
initialBuildProcessMock.Object,
initialTestProcessMock.Object,
testRunnerMock.Object,
assemblyReferenceResolverMock.Object);
var options = new StrykerOptions("c:/test", "Console", "", 2000, null, false);
var exception = Assert.Throws <InitialTestRunFailedException>(() => target.Initialize(options));
inputFileResolverMock.Verify(x => x.ResolveInput(It.IsAny <string>(), It.IsAny <string>()), Times.Once);
reporterMock.Verify();
assemblyReferenceResolverMock.Verify();
initialTestProcessMock.Verify(x => x.InitialTest(testRunnerMock.Object), Times.Once);
}
public void InitialisationProcess_ShouldThrowOnFailedInitialTestRun()
{
var testRunnerMock = new Mock <ITestRunner>(MockBehavior.Strict);
var inputFileResolverMock = new Mock <IInputFileResolver>(MockBehavior.Strict);
var initialBuildProcessMock = new Mock <IInitialBuildProcess>(MockBehavior.Strict);
var initialTestProcessMock = new Mock <IInitialTestProcess>(MockBehavior.Strict);
var assemblyReferenceResolverMock = new Mock <IAssemblyReferenceResolver>(MockBehavior.Strict);
testRunnerMock.Setup(x => x.RunAll(It.IsAny <ITimeoutValueCalculator>(), null, null));
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf());
inputFileResolverMock.Setup(x => x.ResolveInput(It.IsAny <StrykerOptions>())).Returns(
new ProjectInfo(new MockFileSystem())
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(
references: new string[0]).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(
projectFilePath: "C://Example/Dir/ProjectFolder").Object
},
ProjectContents = folder
});
initialBuildProcessMock.Setup(x => x.InitialBuild(It.IsAny <bool>(), It.IsAny <string>(), It.IsAny <string>(), null));
testRunnerMock.Setup(x => x.DiscoverTests()).Returns(new TestSet());
testRunnerMock.Setup(x => x.Dispose());
initialTestProcessMock.Setup(x => x.InitialTest(It.IsAny <StrykerOptions>(), It.IsAny <ITestRunner>())).Throws(new InputException("")); // failing test
assemblyReferenceResolverMock.Setup(x => x.LoadProjectReferences(It.IsAny <string[]>()))
.Returns(Enumerable.Empty <PortableExecutableReference>())
.Verifiable();
var target = new InitialisationProcess(
inputFileResolverMock.Object,
initialBuildProcessMock.Object,
initialTestProcessMock.Object,
testRunnerMock.Object,
assemblyReferenceResolverMock.Object);
var options = new StrykerOptions
{
ProjectName = "TheProjectName",
ProjectVersion = "TheProjectVersion"
};
target.Initialize(options);
Assert.Throws <InputException>(() => target.InitialTest(options));
inputFileResolverMock.Verify(x => x.ResolveInput(It.IsAny <StrykerOptions>()), Times.Once);
assemblyReferenceResolverMock.Verify();
initialTestProcessMock.Verify(x => x.InitialTest(It.IsAny <StrykerOptions>(), testRunnerMock.Object), Times.Once);
}
