public void ShouldHandleTestFailingAtInit()
{
var scenario = new FullRunScenario();
var basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
scenario.CreateMutants(1, 2);
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf()
{
SourceCode = SourceFile,
Mutants = scenario.GetMutants()
});
scenario.CreateTests(1, 2, 3);
// mutant 1 is covered by both tests
scenario.DeclareCoverageForMutant(1);
// mutant 2 is covered only by test 1
scenario.DeclareCoverageForMutant(2, 1, 3);
scenario.DeclareTestsFailingAtInit(1);
// test 1 succeeds, test 2 fails
scenario.DeclareTestsFailingWhenTestingMutant(1, 1, 2);
scenario.DeclareTestsFailingWhenTestingMutant(2, 1);
var runnerMock = scenario.GetTestRunnerMock();
// setup coverage
var executor = new MutationTestExecutor(runnerMock.Object);
var input = new MutationTestInput
{
ProjectInfo = new ProjectInfo(new MockFileSystem())
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "Language", "C#" }
}).Object,
ProjectContents = folder
},
AssemblyReferences = _assemblies,
InitialTestRun = new InitialTestRun(scenario.GetInitialRunResult(), new TimeoutValueCalculator(500))
};
var mutantFilterMock = new Mock <IMutantFilter>(MockBehavior.Loose);
var options = new StrykerOptions
{
BasePath = basePath,
Concurrency = 1,
OptimizationMode = OptimizationModes.CoverageBasedTest
};
var target = new MutationTestProcess(input,
null,
executor,
mutantFilter: mutantFilterMock.Object,
options: options);
// test mutants
target.GetCoverage();
target.Test(input.ProjectInfo.ProjectContents.Mutants);
// first mutant should be killed by test 2
scenario.GetMutantStatus(1).ShouldBe(MutantStatus.Killed);
// other mutant survives
scenario.GetMutantStatus(2).ShouldBe(MutantStatus.Survived);
}
private static IEnumerable <Mutant> MutateAndCompileSource(string sourceFile)
{
var filesystemRoot = Path.GetPathRoot(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
var inputFile = new CsharpFileLeaf()
{
SourceCode = sourceFile,
SyntaxTree = CSharpSyntaxTree.ParseText(sourceFile)
};
var folder = new CsharpFolderComposite();
folder.Add(inputFile);
foreach (var(name, code) in CodeInjection.MutantHelpers)
{
folder.AddCompilationSyntaxTree(CSharpSyntaxTree.ParseText(code, path: name, encoding: Encoding.UTF32));
}
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(filesystemRoot, "ExampleProject", "Calculator.cs"), new MockFileData(sourceFile) },
{
Path.Combine(filesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll"),
new MockFileData("Bytecode")
},
{
Path.Combine(filesystemRoot, "ExampleProject.Test", "obj", "Release", "netcoreapp2.0",
"ExampleProject.dll"),
new MockFileData("Bytecode")
}
});
var input = new MutationTestInput
{
ProjectInfo = new ProjectInfo(fileSystem)
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(
properties: new Dictionary <string, string>()
{
{ "TargetDir", "Project" },
{ "AssemblyName", "AssemblyName" },
{ "TargetFileName", "TargetFileName.dll" },
}).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult>
{
TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "AssemblyName", "TargetFileName" },
{ "TargetDir", "Test" },
{ "TargetFileName", "TestTargetFileName.dll" },
}).Object
},
ProjectContents = folder
},
AssemblyReferences = new List <PortableExecutableReference>
{
MetadataReference.CreateFromFile(typeof(object).Assembly.Location)
},
TestRunner = new Mock <ITestRunner>(MockBehavior.Default).Object
};
var options = new StrykerOptions
{
MutationLevel = MutationLevel.Complete,
OptimizationMode = OptimizationModes.CoverageBasedTest,
};
var process = new CsharpMutationProcess(input, fileSystem, options);
process.Mutate();
var projectContentsMutants = input.ProjectInfo.ProjectContents.Mutants;
return(projectContentsMutants);
}
public void ShouldCallExecutorForEveryCoveredMutant()
{
var scenario = new FullRunScenario();
scenario.CreateMutants(1, 2);
// we need at least one test
scenario.CreateTest(1);
// and we need to declare that the mutant is covered
scenario.DeclareCoverageForMutant(1);
var basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf()
{
SourceCode = SourceFile,
Mutants = scenario.GetMutants()
});
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo(new MockFileSystem())
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "Language", "C#" }
}).Object
,
ProjectContents = folder
},
AssemblyReferences = _assemblies,
InitialTestRun = new InitialTestRun(scenario.GetInitialRunResult(), new TimeoutValueCalculator(500))
};
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
reporterMock.Setup(x => x.OnMutantTested(It.IsAny <Mutant>()));
var mutationExecutor = new MutationTestExecutor(scenario.GetTestRunnerMock().Object);
var mutantFilterMock = new Mock <IMutantFilter>(MockBehavior.Loose);
var options = new StrykerOptions()
{
BasePath = basePath,
Concurrency = 1,
OptimizationMode = OptimizationModes.CoverageBasedTest
};
var target = new MutationTestProcess(input,
reporterMock.Object,
mutationExecutor,
mutantFilter: mutantFilterMock.Object,
options: options);
target.GetCoverage();
target.Test(scenario.GetCoveredMutants());
scenario.GetMutantStatus(1).ShouldBe(MutantStatus.Survived);
scenario.GetMutantStatus(2).ShouldBe(MutantStatus.NoCoverage);
}
public void ShouldNotCallExecutorForNotCoveredMutants()
{
var mutant = new Mutant {
Id = 1, ResultStatus = MutantStatus.Survived
};
var otherMutant = new Mutant {
Id = 2, ResultStatus = MutantStatus.NotRun
};
var basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
TestProjectAnalyzerResults = new List <ProjectAnalyzerResult> {
new ProjectAnalyzerResult(null, null)
{
AssemblyPath = "/bin/Debug/netcoreapp2.1/TestName.dll",
Properties = new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" }
}
}
},
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
AssemblyPath = "/bin/Debug/netcoreapp2.1/TestName.dll",
Properties = new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" }
}
},
ProjectContents = new FolderComposite()
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "SomeFile.cs",
Mutants = new Collection <Mutant>()
{
mutant, otherMutant
}
}
}
},
},
AssemblyReferences = new ReferenceProvider().GetReferencedAssemblies()
};
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
reporterMock.Setup(x => x.OnMutantTested(It.IsAny <Mutant>()));
reporterMock.Setup(x => x.OnAllMutantsTested(It.IsAny <ProjectComponent>()));
reporterMock.Setup(x => x.OnStartMutantTestRun(It.IsAny <IList <Mutant> >(), It.IsAny <IEnumerable <TestDescription> >()));
var runnerMock = new Mock <ITestRunner>();
runnerMock.Setup(x => x.DiscoverNumberOfTests()).Returns(1);
var executorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
executorMock.SetupGet(x => x.TestRunner).Returns(runnerMock.Object);
executorMock.Setup(x => x.Test(It.IsAny <IList <Mutant> >(), It.IsAny <int>(), It.IsAny <TestUpdateHandler>()));
var mutantFilterMock = new Mock <IMutantFilter>(MockBehavior.Loose);
var options = new StrykerOptions(fileSystem: new MockFileSystem(), basePath: basePath);
var target = new MutationTestProcess(input,
reporterMock.Object,
executorMock.Object,
mutantFilter: mutantFilterMock.Object,
options: options);
target.Test(options);
reporterMock.Verify(x => x.OnStartMutantTestRun(It.Is <IList <Mutant> >(y => y.Count == 1), It.IsAny <IEnumerable <TestDescription> >()), Times.Once);
executorMock.Verify(x => x.Test(new List <Mutant> {
otherMutant
}, It.IsAny <int>(), It.IsAny <TestUpdateHandler>()), Times.Once);
reporterMock.Verify(x => x.OnAllMutantsTested(It.IsAny <ProjectComponent>()), Times.Once);
}
public void ShouldNotTest_WhenThereAreNoTestableMutations()
{
var mutant = new Mutant()
{
Id = 1, ResultStatus = MutantStatus.Ignored
};
var mutant2 = new Mutant()
{
Id = 2, ResultStatus = MutantStatus.CompileError
};
string basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
ProjectContents = new FolderComposite()
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "SomeFile.cs",
Mutants = new Collection <Mutant>()
{
mutant, mutant2
}
}
}
},
},
AssemblyReferences = _assemblies
};
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
reporterMock.Setup(x => x.OnMutantTested(It.IsAny <Mutant>()));
reporterMock.Setup(x => x.OnAllMutantsTested(It.IsAny <ProjectComponent>()));
reporterMock.Setup(x => x.OnStartMutantTestRun(It.IsAny <IList <Mutant> >(), It.IsAny <IEnumerable <TestDescription> >()));
var runnerMock = new Mock <ITestRunner>();
runnerMock.Setup(x => x.DiscoverNumberOfTests()).Returns(1);
var executorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
executorMock.SetupGet(x => x.TestRunner).Returns(runnerMock.Object);
executorMock.Setup(x => x.Test(It.IsAny <IList <Mutant> >(), It.IsAny <int>(), It.IsAny <TestUpdateHandler>()));
var mutantFilterMock = new Mock <IMutantFilter>(MockBehavior.Loose);
var options = new StrykerOptions(fileSystem: new MockFileSystem(), basePath: basePath);
var target = new MutationTestProcess(input,
reporterMock.Object,
executorMock.Object,
mutantFilter: mutantFilterMock.Object,
options: options);
var testResult = target.Test(options);
executorMock.Verify(x => x.Test(It.IsAny <IList <Mutant> >(), It.IsAny <int>(), It.IsAny <TestUpdateHandler>()), Times.Never);
reporterMock.Verify(x => x.OnStartMutantTestRun(It.IsAny <IList <Mutant> >(), It.IsAny <IEnumerable <TestDescription> >()), Times.Never);
reporterMock.Verify(x => x.OnMutantTested(It.IsAny <Mutant>()), Times.Never);
reporterMock.Verify(x => x.OnAllMutantsTested(It.IsAny <ProjectComponent>()), Times.Once);
testResult.MutationScore.ShouldBe(double.NaN);
}
public void CompilingProcessTests_ShouldCallRollbackProcess_OnCompileError()
{
var syntaxTree = CSharpSyntaxTree.ParseText(@"using System;
namespace ExampleProject
{
public class Calculator
{
public int Subtract(string first, string second)
{
return first - second;
}
}
}");
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
Properties = new Dictionary <string, string>()
{
{ "AssemblyTitle", "AssemblyName" },
},
Resources = new List <ResourceDescription>()
},
TestProjectAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
Properties = new Dictionary <string, string>()
{
{ "AssemblyTitle", "AssemblyName" },
},
Resources = new List <ResourceDescription>()
}
},
AssemblyReferences = new List <PortableExecutableReference>()
{
MetadataReference.CreateFromFile(typeof(object).Assembly.Location)
}
};
var rollbackProcessMock = new Mock <IRollbackProcess>(MockBehavior.Strict);
rollbackProcessMock.Setup(x => x.Start(It.IsAny <CSharpCompilation>(), It.IsAny <ImmutableArray <Diagnostic> >(), It.IsAny <bool>()))
.Returns((CSharpCompilation compilation, ImmutableArray <Diagnostic> diagnostics, bool devMode) =>
new RollbackProcessResult()
{
Compilation = compilation
});
var target = new CompilingProcess(input, rollbackProcessMock.Object);
using (var ms = new MemoryStream())
{
Should.Throw <ApplicationException>(() => target.Compile(new Collection <SyntaxTree>()
{
syntaxTree
}, ms, false));
}
rollbackProcessMock.Verify(x => x.Start(It.IsAny <CSharpCompilation>(), It.IsAny <ImmutableArray <Diagnostic> >(), It.IsAny <bool>()),
Times.AtLeast(2));
}
public CoverageAnalyser(StrykerOptions options, IMutationTestExecutor mutationTestExecutor, MutationTestInput input)
{
_input = input;
_mutationTestExecutor = mutationTestExecutor;
_options = options;
_logger = ApplicationLogging.LoggerFactory.CreateLogger <CoverageAnalyser>();
}
public void MutationTestProcess_MutateShouldWriteToDisk_IfCompilationIsSuccessful()
{
string basePath = Path.Combine(_filesystemRoot, "ExampleProject.Test");
var input = new MutationTestInput()
{
ProjectInfo = new Core.Initialisation.ProjectInfo()
{
TestProjectPath = basePath,
ProjectContents = new FolderComposite()
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "SomeFile.cs",
SourceCode = _exampleFileContents
}
}
},
ProjectUnderTestAssemblyName = "ExampleProject",
ProjectUnderTestPath = Path.Combine(_filesystemRoot, "ExampleProject"),
TargetFramework = "netcoreapp2.0"
},
AssemblyReferences = new ReferenceProvider().GetReferencedAssemblies()
};
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}
};
// create mocks
var orchestratorMock = new Mock <IMutantOrchestrator>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var compilingProcessMock = new Mock <ICompilingProcess>(MockBehavior.Strict);
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(_filesystemRoot, "SomeFile.cs"), new MockFileData("SomeFile") },
});
fileSystem.AddDirectory(Path.Combine(_filesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0"));
// setup mocks
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(_exampleFileContents).GetRoot());
orchestratorMock.SetupAllProperties();
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()));
compilingProcessMock.Setup(x => x.Compile(It.IsAny <IEnumerable <SyntaxTree> >(), It.IsAny <MemoryStream>()))
.Returns(new CompilingProcessResult()
{
Success = true
});
var target = new MutationTestProcess(input,
reporterMock.Object,
Enumerable.Empty <MutatorType>(),
mutationTestExecutorMock.Object,
orchestratorMock.Object,
compilingProcessMock.Object,
fileSystem);
target.Mutate();
// Verify the created assembly is written to disk on the right location
Assert.True(fileSystem.FileExists(Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll")),
"The mutated Assembly was not written to disk, or not to the right location.");
}
public void MutationTestProcess_MutateShouldCallMutantOrchestrator()
{
var input = new MutationTestInput()
{
ProjectInfo = new Core.Initialisation.ProjectInfo()
{
TestProjectPath = Path.Combine(_filesystemRoot, "ExampleProject.Test"),
ProjectUnderTestPath = Path.Combine(_filesystemRoot, "ExampleProject"),
ProjectUnderTestAssemblyName = "ExampleProject",
TargetFramework = "netcoreapp2.0",
ProjectContents = new FolderComposite()
{
Name = Path.Combine(_filesystemRoot, "ExampleProject"),
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "Recursive.cs",
SourceCode = _exampleFileContents
}
}
},
},
AssemblyReferences = new ReferenceProvider().GetReferencedAssemblies()
};
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(_filesystemRoot, "ExampleProject", "Recursive.cs"), new MockFileData(_exampleFileContents) },
{ Path.Combine(_filesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") },
{ Path.Combine(_filesystemRoot, "ExampleProject.Test", "obj", "Release", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") }
});
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}
};
// create mocks
var orchestratorMock = new Mock <IMutantOrchestrator>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var compilingProcessMock = new Mock <ICompilingProcess>(MockBehavior.Strict);
// setup mocks
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()));
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(_exampleFileContents).GetRoot());
orchestratorMock.SetupAllProperties();
compilingProcessMock.Setup(x => x.Compile(It.IsAny <IEnumerable <SyntaxTree> >(), It.IsAny <MemoryStream>()))
.Returns(new CompilingProcessResult()
{
Success = true
});
var target = new MutationTestProcess(input,
reporterMock.Object,
Enumerable.Empty <MutatorType>(),
mutationTestExecutorMock.Object,
orchestratorMock.Object,
compilingProcessMock.Object,
fileSystem);
// start mutation process
target.Mutate();
// verify the right methods were called
orchestratorMock.Verify(x => x.Mutate(It.IsAny <SyntaxNode>()), Times.Once);
reporterMock.Verify(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()), Times.Once);
}
public void MutationTestProcess_ExcludeFilesToMutate_MutationCalledOnce()
{
string sourceFile2 = _sourceFile.Replace("Recursive.cs", "Recursive2.cs");
string sourceFile3 = _sourceFile.Replace("Recursive.cs", "Recursive3.cs");
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
TestProjectAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
AssemblyPath = "/bin/Debug/netcoreapp2.1/TestName.dll",
Properties = new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" }
}
},
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
AssemblyPath = "/bin/Debug/netcoreapp2.1/TestName.dll",
Properties = new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" }
}
},
ProjectContents = new FolderComposite()
{
Name = Path.Combine(_filesystemRoot, "ExampleProject"),
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "Recursive.cs",
SourceCode = _sourceFile
},
new FileLeaf()
{
Name = "Recursive2.cs",
SourceCode = sourceFile2,
IsExcluded = true
},
new FileLeaf()
{
Name = "Recursive3.cs",
SourceCode = sourceFile3
}
}
},
},
AssemblyReferences = _assemblies
};
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(_filesystemRoot, "ExampleProject", "Recursive.cs"), new MockFileData(_sourceFile) },
{ Path.Combine(_filesystemRoot, "ExampleProject", "Recursive2.cs"), new MockFileData(sourceFile2) },
{ Path.Combine(_filesystemRoot, "ExampleProject", "Recursive3.cs"), new MockFileData(sourceFile3) },
{ Path.Combine(_filesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") },
{ Path.Combine(_filesystemRoot, "ExampleProject.Test", "obj", "Release", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") }
});
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}
};
// create mocks
var orchestratorMock = new Mock <IMutantOrchestrator>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var compilingProcessMock = new Mock <ICompilingProcess>(MockBehavior.Strict);
// setup mocks
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()));
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(_sourceFile).GetRoot());
orchestratorMock.SetupAllProperties();
compilingProcessMock.Setup(x => x.Compile(It.IsAny <IEnumerable <SyntaxTree> >(), It.IsAny <MemoryStream>(), It.IsAny <bool>()))
.Returns(new CompilingProcessResult()
{
Success = true
});
var target = new MutationTestProcess(input,
reporterMock.Object,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
compilingProcessMock.Object,
fileSystem);
var options = new StrykerOptions(fileSystem: fileSystem);
// start mutation process
target.Mutate(options);
// verify the right methods were called
orchestratorMock.Verify(x => x.Mutate(It.IsAny <SyntaxNode>()), Times.Exactly(2));
reporterMock.Verify(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()), Times.Once);
}
public void Stryker_ShouldInvokeAllProcesses()
{
var projectOrchestratorMock = new Mock <IProjectOrchestrator>(MockBehavior.Strict);
var mutationTestProcessMock = new Mock <IMutationTestProcess>(MockBehavior.Strict);
var reporterFactoryMock = new Mock <IReporterFactory>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var fileSystemMock = new MockFileSystem();
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf()
{
Mutants = new List <Mutant> {
new Mutant {
Id = 1
}
}
});
var mutationTestInput = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
ProjectContents = folder
},
};
var options = new StrykerOptions(basePath: "c:/test", fileSystem: fileSystemMock);
projectOrchestratorMock.Setup(x => x.MutateProjects(options, It.IsAny <IReporter>()))
.Returns(new List <IMutationTestProcess>()
{
mutationTestProcessMock.Object
});
reporterFactoryMock.Setup(x => x.Create(It.IsAny <StrykerOptions>(), It.IsAny <IGitInfoProvider>())).Returns(reporterMock.Object);
reporterMock.Setup(x => x.OnStartMutantTestRun(It.IsAny <IEnumerable <IReadOnlyMutant> >()));
reporterMock.Setup(x => x.OnAllMutantsTested(It.IsAny <IReadOnlyProjectComponent>()));
mutationTestProcessMock.SetupGet(x => x.Input).Returns(mutationTestInput);
mutationTestProcessMock.Setup(x => x.GetCoverage());
mutationTestProcessMock.Setup(x => x.Test(It.IsAny <IEnumerable <Mutant> >()))
.Returns(new StrykerRunResult(It.IsAny <StrykerOptions>(), It.IsAny <double>()));
// Set up sequence-critical methods:
// * FilterMutants must be called before OnMutantsCreated to get valid reports
var seq = new MockSequence();
mutationTestProcessMock.InSequence(seq).Setup(x => x.FilterMutants());
reporterMock.InSequence(seq).Setup(x => x.OnMutantsCreated(It.IsAny <IReadOnlyProjectComponent>()));
var target = new StrykerRunner(projectOrchestratorMock.Object, reporterFactory: reporterFactoryMock.Object);
target.RunMutationTest(options);
projectOrchestratorMock.Verify(x => x.MutateProjects(options, It.IsAny <IReporter>()), Times.Once);
mutationTestProcessMock.Verify(x => x.GetCoverage(), Times.Once);
mutationTestProcessMock.Verify(x => x.Test(It.IsAny <IEnumerable <Mutant> >()), Times.Once);
reporterMock.Verify(x => x.OnMutantsCreated(It.IsAny <IReadOnlyProjectComponent>()), Times.Once);
reporterMock.Verify(x => x.OnStartMutantTestRun(It.IsAny <IEnumerable <IReadOnlyMutant> >()), Times.Once);
reporterMock.Verify(x => x.OnAllMutantsTested(It.IsAny <IReadOnlyProjectComponent>()), Times.Once);
}
public void CompilingProcessTests_SignedAssembliesMustBeSigned()
{
var syntaxTree = CSharpSyntaxTree.ParseText(@"
namespace ExampleProject
{
public class Calculator
{
public int Subtract(int first, int second)
{
return first - second;
}
}
}");
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
ProjectFilePath = "project.csproj",
Properties = new Dictionary <string, string>()
{
{ "AssemblyTitle", "TargetFileName" },
{ "TargetFileName", "TargetFileName.dll" },
{ "SignAssembly", "true" },
{ "AssemblyOriginatorKeyFile", Path.GetFullPath(Path.Combine("TestResources", "StrongNameKeyFile.snk")) }
},
Resources = new List <ResourceDescription>(),
},
TestProjectAnalyzerResults = new List <ProjectAnalyzerResult> {
new ProjectAnalyzerResult(null, null)
{
Properties = new Dictionary <string, string>()
{
{ "AssemblyTitle", "TargetFileName" },
},
Resources = new List <ResourceDescription>()
}
}
},
AssemblyReferences = new List <PortableExecutableReference>()
{
MetadataReference.CreateFromFile(typeof(object).Assembly.Location)
},
};
var rollbackProcessMock = new Mock <IRollbackProcess>(MockBehavior.Strict);
var target = new CompilingProcess(input, rollbackProcessMock.Object);
using (var ms = new MemoryStream())
{
var result = target.Compile(new Collection <SyntaxTree>()
{
syntaxTree
}, ms, null, false);
result.Success.ShouldBe(true);
var key = Assembly.Load(ms.ToArray()).GetName().GetPublicKey();
key.Length.ShouldBe(160, "Assembly was not signed");
ms.Length.ShouldBeGreaterThan(100, "No value was written to the MemoryStream by the compiler");
}
}
public void ShouldStop_WhenAllMutationsWereIgnored()
{
var projectOrchestratorMock = new Mock <IProjectOrchestrator>(MockBehavior.Strict);
var mutationTestProcessMock = new Mock <IMutationTestProcess>(MockBehavior.Strict);
var reporterFactoryMock = new Mock <IReporterFactory>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var inputsMock = new Mock <IStrykerInputs>(MockBehavior.Strict);
var fileSystemMock = new MockFileSystem();
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf
{
Mutants = new Collection <Mutant>()
{
new Mutant()
{
Id = 1, ResultStatus = MutantStatus.Ignored
}
}
});
var mutationTestInput = new MutationTestInput()
{
ProjectInfo = new ProjectInfo(new MockFileSystem())
{
ProjectContents = folder
}
};
inputsMock.Setup(x => x.ValidateAll()).Returns(new StrykerOptions
{
BasePath = "C:/test",
OptimizationMode = OptimizationModes.None,
LogOptions = new LogOptions()
});
projectOrchestratorMock.Setup(x => x.MutateProjects(It.IsAny <StrykerOptions>(), It.IsAny <IReporter>()))
.Returns(new List <IMutationTestProcess>()
{
mutationTestProcessMock.Object
});
mutationTestProcessMock.Setup(x => x.FilterMutants());
mutationTestProcessMock.SetupGet(x => x.Input).Returns(mutationTestInput);
reporterFactoryMock.Setup(x => x.Create(It.IsAny <StrykerOptions>(), It.IsAny <IGitInfoProvider>())).Returns(reporterMock.Object);
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <IReadOnlyProjectComponent>()));
reporterMock.Setup(x => x.OnStartMutantTestRun(It.IsAny <IEnumerable <IReadOnlyMutant> >()));
reporterMock.Setup(x => x.OnAllMutantsTested(It.IsAny <IReadOnlyProjectComponent>()));
var target = new StrykerRunner(reporterFactory: reporterFactoryMock.Object);
var result = target.RunMutationTest(inputsMock.Object, new LoggerFactory(), projectOrchestratorMock.Object);
result.MutationScore.ShouldBe(double.NaN);
reporterMock.Verify(x => x.OnStartMutantTestRun(It.IsAny <IList <Mutant> >()), Times.Never);
reporterMock.Verify(x => x.OnMutantTested(It.IsAny <Mutant>()), Times.Never);
reporterMock.Verify(x => x.OnAllMutantsTested(It.IsAny <IReadOnlyProjectComponent>()), Times.Once);
}
public void Stryker_ShouldInvokeAllProcesses()
{
var projectOrchestratorMock = new Mock <IProjectOrchestrator>(MockBehavior.Strict);
var mutationTestProcessMock = new Mock <IMutationTestProcess>(MockBehavior.Strict);
var reporterFactoryMock = new Mock <IReporterFactory>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var inputsMock = new Mock <IStrykerInputs>(MockBehavior.Strict);
var fileSystemMock = new MockFileSystem();
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf()
{
Mutants = new List <Mutant> {
new Mutant {
Id = 1
}
}
});
var projectInfo = Mock.Of <ProjectInfo>();
projectInfo.ProjectContents = folder;
Mock.Get(projectInfo).Setup(p => p.RestoreOriginalAssembly());
var mutationTestInput = new MutationTestInput()
{
ProjectInfo = projectInfo
};
inputsMock.Setup(x => x.ValidateAll()).Returns(new StrykerOptions
{
BasePath = "C:/test",
LogOptions = new LogOptions(),
OptimizationMode = OptimizationModes.SkipUncoveredMutants
});
projectOrchestratorMock.Setup(x => x.MutateProjects(It.IsAny <StrykerOptions>(), It.IsAny <IReporter>()))
.Returns(new List <IMutationTestProcess>()
{
mutationTestProcessMock.Object
});
reporterFactoryMock.Setup(x => x.Create(It.IsAny <StrykerOptions>(), It.IsAny <IGitInfoProvider>())).Returns(reporterMock.Object);
reporterMock.Setup(x => x.OnStartMutantTestRun(It.IsAny <IEnumerable <IReadOnlyMutant> >()));
reporterMock.Setup(x => x.OnAllMutantsTested(It.IsAny <IReadOnlyProjectComponent>()));
mutationTestProcessMock.SetupGet(x => x.Input).Returns(mutationTestInput);
mutationTestProcessMock.Setup(x => x.GetCoverage());
mutationTestProcessMock.Setup(x => x.Test(It.IsAny <IEnumerable <Mutant> >()))
.Returns(new StrykerRunResult(It.IsAny <StrykerOptions>(), It.IsAny <double>()));
mutationTestProcessMock.Setup(x => x.Restore());
// Set up sequence-critical methods:
// * FilterMutants must be called before OnMutantsCreated to get valid reports
var seq = new MockSequence();
mutationTestProcessMock.InSequence(seq).Setup(x => x.FilterMutants());
reporterMock.InSequence(seq).Setup(x => x.OnMutantsCreated(It.IsAny <IReadOnlyProjectComponent>()));
var target = new StrykerRunner(reporterFactory: reporterFactoryMock.Object);
target.RunMutationTest(inputsMock.Object, new LoggerFactory(), projectOrchestratorMock.Object);
projectOrchestratorMock.Verify(x => x.MutateProjects(It.Is <StrykerOptions>(x => x.BasePath == "C:/test"), It.IsAny <IReporter>()), Times.Once);
mutationTestProcessMock.Verify(x => x.GetCoverage(), Times.Once);
mutationTestProcessMock.Verify(x => x.Test(It.IsAny <IEnumerable <Mutant> >()), Times.Once);
reporterMock.Verify(x => x.OnMutantsCreated(It.IsAny <IReadOnlyProjectComponent>()), Times.Once);
reporterMock.Verify(x => x.OnStartMutantTestRun(It.IsAny <IEnumerable <IReadOnlyMutant> >()), Times.Once);
reporterMock.Verify(x => x.OnAllMutantsTested(It.IsAny <IReadOnlyProjectComponent>()), Times.Once);
}
public void ShouldCallMutantOrchestratorAndReporter()
{
var inputFile = new CsharpFileLeaf()
{
SourceCode = SourceFile,
SyntaxTree = CSharpSyntaxTree.ParseText(SourceFile)
};
var folder = new CsharpFolderComposite();
folder.Add(inputFile);
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(FilesystemRoot, "ExampleProject", "Recursive.cs"), new MockFileData(SourceFile) },
{ Path.Combine(FilesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") },
{ Path.Combine(FilesystemRoot, "ExampleProject.Test", "obj", "Release", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") }
});
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo(fileSystem)
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "AssemblyName", "AssemblyName" },
{ "Language", "C#" }
}).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "AssemblyName", "AssemblyName" },
{ "Language", "C#" }
}).Object
},
ProjectContents = folder
},
AssemblyReferences = _assemblies
};
var mutantToBeSkipped = new Mutant()
{
Mutation = new Mutation()
};
var mockMutants = new Collection <Mutant>()
{
new() { Mutation = new Mutation() }, mutantToBeSkipped
};
// create mocks
var options = new StrykerOptions()
{
DevMode = true,
ExcludedMutations = new Mutator[] { }
};
var orchestratorMock = new Mock <BaseMutantOrchestrator <SyntaxNode> >(MockBehavior.Strict, options);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var coverageAnalyzerMock = new Mock <ICoverageAnalyser>(MockBehavior.Strict);
// setup mocks
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(SourceFile).GetRoot());
orchestratorMock.SetupAllProperties();
var target = new MutationTestProcess(input,
reporterMock.Object,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
fSharpOrchestrator: null,
fileSystem,
new BroadcastMutantFilter(Enumerable.Empty <IMutantFilter>()),
coverageAnalyzerMock.Object,
options);
// start mutation process
target.Mutate();
target.FilterMutants();
// verify the right methods were called
orchestratorMock.Verify(x => x.Mutate(It.IsAny <SyntaxNode>()), Times.Once);
}
public void MutationTestProcess_MutateShouldWriteToDisk_IfCompilationIsSuccessful()
{
string file1 = @"using System;
namespace ExampleProject
{
public class Recursive
{
public int Fibinacci(int len)
{
return Fibonacci(0, 1, 1, len);
}
private int Fibonacci(int a, int b, int counter, int len)
{
if (counter <= len)
{
Console.Write(""{0} "", a);
return Fibonacci(b, a + b, counter + 1, len);
}
return 0;
}
}
}
";
string basePath = Path.Combine(_filesystemRoot, "ExampleProject.Test");
var input = new MutationTestInput()
{
ProjectInfo = new Core.Initialisation.ProjectInfo()
{
TestProjectPath = basePath,
ProjectContents = new FolderComposite()
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "SomeFile.cs",
SourceCode = file1
}
}
},
ProjectUnderTestAssemblyName = "ExampleProject",
ProjectUnderTestPath = Path.Combine(_filesystemRoot, "ExampleProject"),
TargetFramework = "netcoreapp2.0"
},
AssemblyReferences = new ReferenceProvider().GetReferencedAssemblies()
};
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}
};
// create mocks
var orchestratorMock = new Mock <IMutantOrchestrator>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var compilingProcessMock = new Mock <ICompilingProcess>(MockBehavior.Strict);
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(_filesystemRoot, "SomeFile.cs"), new MockFileData("SomeFile") },
});
fileSystem.AddDirectory(Path.Combine(_filesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0"));
// setup mocks
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(file1).GetRoot());
orchestratorMock.SetupAllProperties();
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()));
compilingProcessMock.Setup(x => x.Compile(It.IsAny <IEnumerable <SyntaxTree> >(), It.IsAny <MemoryStream>()))
.Returns(new CompilingProcessResult()
{
Success = true
});
var options = new StrykerOptions(Path.Combine(_filesystemRoot, "test"), "Console", "", 2000, null, false, 1, 80, 60, 0);
var target = new MutationTestProcess(input,
reporterMock.Object,
null,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
compilingProcessMock.Object,
fileSystem);
target.Mutate();
// Verify the created assembly is written to disk on the right location
Assert.True(fileSystem.FileExists(Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll")),
"The mutated Assembly was not written to disk, or not to the right location.");
}
public void ShouldThrowExceptionWhenOtherStatusThanNotRunIsPassed(MutantStatus status)
{
var mutant = new Mutant {
Id = 1, ResultStatus = status
};
var basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf
{
Mutants = new Collection <Mutant> {
mutant
}
});
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo(new MockFileSystem())
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "Language", "C#" }
}).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "Language", "C#" }
}).Object
},
ProjectContents = folder
},
AssemblyReferences = new ReferenceProvider().GetReferencedAssemblies()
};
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
reporterMock.Setup(x => x.OnMutantTested(It.IsAny <Mutant>()));
var runnerMock = new Mock <ITestRunner>();
runnerMock.Setup(x => x.DiscoverTests()).Returns(new TestSet());
var executorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
executorMock.SetupGet(x => x.TestRunner).Returns(runnerMock.Object);
executorMock.Setup(x => x.Test(It.IsAny <IList <Mutant> >(),
It.IsAny <ITimeoutValueCalculator>(),
It.IsAny <TestUpdateHandler>()));
var mutantFilterMock = new Mock <IMutantFilter>(MockBehavior.Loose);
var options = new StrykerOptions()
{
BasePath = basePath
};
var target = new MutationTestProcess(input,
reporterMock.Object,
executorMock.Object,
mutantFilter: mutantFilterMock.Object,
options: options);
Should.Throw <GeneralStrykerException>(() => target.Test(input.ProjectInfo.ProjectContents.Mutants));
}
public void ShouldCallExecutorForEveryMutant()
{
var mutant = new Mutant {
Id = 1
};
var otherMutant = new Mutant {
Id = 2
};
string basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
AssemblyPath = "/bin/Debug/netcoreapp2.1/TestName.dll",
Properties = new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" }
}
},
ProjectContents = new FolderComposite()
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "SomeFile.cs",
SourceCode = SourceFile,
Mutants = new List <Mutant>()
{
mutant, otherMutant
}
}
}
}
},
AssemblyReferences = _assemblies
};
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
reporterMock.Setup(x => x.OnMutantTested(It.IsAny <Mutant>()));
reporterMock.Setup(x => x.OnAllMutantsTested(It.IsAny <ProjectComponent>()));
reporterMock.Setup(x => x.OnStartMutantTestRun(It.IsAny <IList <Mutant> >(), It.IsAny <IEnumerable <TestDescription> >()));
var executorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
executorMock.SetupGet(x => x.TestRunner).Returns(Mock.Of <ITestRunner>());
executorMock.Setup(x => x.Test(It.IsAny <Mutant>(), It.IsAny <int>()));
var options = new StrykerOptions(fileSystem: new MockFileSystem(), basePath: basePath);
var target = new MutationTestProcess(input,
reporterMock.Object,
executorMock.Object,
options: options);
target.Test(options);
executorMock.Verify(x => x.Test(mutant, It.IsAny <int>()), Times.Once);
reporterMock.Verify(x => x.OnStartMutantTestRun(It.Is <IList <Mutant> >(y => y.Count == 2), It.IsAny <IEnumerable <TestDescription> >()), Times.Once);
reporterMock.Verify(x => x.OnMutantTested(mutant), Times.Once);
reporterMock.Verify(x => x.OnMutantTested(otherMutant), Times.Once);
reporterMock.Verify(x => x.OnAllMutantsTested(It.IsAny <ProjectComponent>()), Times.Once);
}
/// <summary>
/// Starts a mutation test run
/// </summary>
/// <exception cref="StrykerInputException">For managed exceptions</exception>
/// <param name="options">The user options</param>
/// <param name="initialLogMessages">
/// Allows to pass log messages that occured before the mutation test.
/// The messages will be written to the logger after it was configured.
/// </param>
public StrykerRunResult RunMutationTest(StrykerOptions options, IEnumerable <LogMessage> initialLogMessages = null)
{
// start stopwatch
var stopwatch = new Stopwatch();
stopwatch.Start();
// Create output dir with gitignore
_fileSystem.Directory.CreateDirectory(options.OutputPath);
_fileSystem.File.Create(Path.Combine(options.OutputPath, ".gitignore")).Close();
using (var file = _fileSystem.File.CreateText(Path.Combine(options.OutputPath, ".gitignore")))
{
file.WriteLine("*");
}
// setup logging
ApplicationLogging.ConfigureLogger(options.LogOptions, initialLogMessages);
var logger = ApplicationLogging.LoggerFactory.CreateLogger <StrykerRunner>();
logger.LogDebug("Stryker started with options: {0}",
JsonConvert.SerializeObject(options, new StringEnumConverter()));
try
{
// initialize
_reporter = ReporterFactory.Create(options);
_initialisationProcess = _initialisationProcess ?? new InitialisationProcess();
_input = _initialisationProcess.Initialize(options);
_mutationTestProcess = _mutationTestProcess ?? new MutationTestProcess(
mutationTestInput: _input,
reporter: _reporter,
mutationTestExecutor: new MutationTestExecutor(_input.TestRunner),
options: options);
// initial test
_input.TimeoutMs = _initialisationProcess.InitialTest(options, out var nbTests);
// mutate
_mutationTestProcess.Mutate();
if (options.Optimizations.HasFlag(OptimizationFlags.SkipUncoveredMutants) || options.Optimizations.HasFlag(OptimizationFlags.CoverageBasedTest))
{
logger.LogInformation($"Capture mutant coverage using '{options.OptimizationMode}' mode.");
// coverage
_mutationTestProcess.GetCoverage();
}
// test mutations and return results
return(_mutationTestProcess.Test(options));
}
catch (Exception ex) when(!(ex is StrykerInputException))
{
logger.LogError(ex, "An error occurred during the mutation test run ");
throw;
}
finally
{
// log duration
stopwatch.Stop();
logger.LogInformation("Time Elapsed {0}", stopwatch.Elapsed);
}
}
public void ShouldCallExecutorForEveryMutant()
{
var mutant = new Mutant {
Id = 1, MustRunAgainstAllTests = true
};
var otherMutant = new Mutant {
Id = 2, MustRunAgainstAllTests = true
};
string basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf()
{
SourceCode = SourceFile,
Mutants = new List <Mutant>()
{
mutant, otherMutant
}
});
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "Language", "C#" }
}).Object
,
ProjectContents = folder
},
AssemblyReferences = _assemblies
};
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
reporterMock.Setup(x => x.OnMutantTested(It.IsAny <Mutant>()));
var runnerMock = new Mock <ITestRunner>();
runnerMock.Setup(x => x.DiscoverNumberOfTests()).Returns(1);
var executorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
executorMock.SetupGet(x => x.TestRunner).Returns(runnerMock.Object);
executorMock.Setup(x => x.Test(It.IsAny <IList <Mutant> >(), It.IsAny <int>(), It.IsAny <TestUpdateHandler>()));
var mutantFilterMock = new Mock <IMutantFilter>(MockBehavior.Loose);
var options = new StrykerOptions(basePath: basePath, fileSystem: new MockFileSystem());
var target = new MutationTestProcess(input,
reporterMock.Object,
executorMock.Object,
mutantFilter: mutantFilterMock.Object,
options: options);
target.Test(input.ProjectInfo.ProjectContents.Mutants);
executorMock.Verify(x => x.Test(new List <Mutant> {
mutant
}, It.IsAny <int>(), It.IsAny <TestUpdateHandler>()), Times.Once);
}
public void MutateShouldWriteToDisk_IfCompilationIsSuccessful()
{
string basePath = Path.Combine(FilesystemRoot, "ExampleProject.Test");
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
TestProjectAnalyzerResults = new List <ProjectAnalyzerResult> {
new ProjectAnalyzerResult(null, null)
{
AssemblyPath = Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0", "TestName.dll"),
Properties = new Dictionary <string, string>()
{
{ "TargetDir", Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0") },
{ "AssemblyName", "TestName" },
}
}
},
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
AssemblyPath = Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll"),
Properties = new Dictionary <string, string>()
{
{ "TargetDir", Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0") },
{ "AssemblyName", "ExampleProject" }
}
},
ProjectContents = new FolderComposite()
{
Name = "ProjectRoot",
Children = new Collection <ProjectComponent>()
{
new FileLeaf
{
Name = "SomeFile.cs",
SourceCode = SourceFile,
SyntaxTree = CSharpSyntaxTree.ParseText(SourceFile)
}
}
}
},
AssemblyReferences = _assemblies
};
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}
};
// create mocks
var orchestratorMock = new Mock <IMutantOrchestrator>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var compilingProcessMock = new Mock <ICompilingProcess>(MockBehavior.Strict);
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(FilesystemRoot, "SomeFile.cs"), new MockFileData("SomeFile") },
});
fileSystem.AddDirectory(Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0"));
// setup mocks
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(SourceFile).GetRoot());
orchestratorMock.SetupAllProperties();
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()));
compilingProcessMock.Setup(x => x.Compile(It.IsAny <IEnumerable <SyntaxTree> >(), It.IsAny <MemoryStream>(), It.IsAny <MemoryStream>(), It.IsAny <bool>()))
.Returns(new CompilingProcessResult()
{
Success = true
});
var options = new StrykerOptions();
var target = new MutationTestProcess(input,
reporterMock.Object,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
compilingProcessMock.Object,
fileSystem,
options,
new BroadcastMutantFilter(Enumerable.Empty <IMutantFilter>()));
target.Mutate();
// Verify the created assembly is written to disk on the right location
string expectedPath = Path.Combine(basePath, "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll");
fileSystem.FileExists(expectedPath)
.ShouldBeTrue($"The mutated Assembly was not written to disk, or not to the right location ({expectedPath}).");
}
public void FilterMutantsShouldCallMutantFilters()
{
var inputFile = new CsharpFileLeaf()
{
SourceCode = SourceFile,
SyntaxTree = CSharpSyntaxTree.ParseText(SourceFile)
};
var folder = new CsharpFolderComposite();
folder.Add(inputFile);
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(FilesystemRoot, "ExampleProject", "Recursive.cs"), new MockFileData(SourceFile) },
{ Path.Combine(FilesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") },
{ Path.Combine(FilesystemRoot, "ExampleProject.Test", "obj", "Release", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") }
});
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo(fileSystem)
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "AssemblyName", "AssemblyName" },
{ "Language", "C#" }
}).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "TargetFileName", "TestName.dll" },
{ "AssemblyName", "AssemblyName" },
{ "Language", "C#" }
}).Object
},
ProjectContents = folder
},
AssemblyReferences = _assemblies
};
var mutantToBeSkipped = new Mutant()
{
Mutation = new Mutation()
};
var compileErrorMutant = new Mutant()
{
Mutation = new Mutation(), ResultStatus = MutantStatus.CompileError
};
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}, mutantToBeSkipped, compileErrorMutant
};
// create mocks
var options = new StrykerOptions()
{
DevMode = true,
ExcludedMutations = new Mutator[] { }
};
var orchestratorMock = new Mock <BaseMutantOrchestrator <SyntaxNode> >(MockBehavior.Strict, options);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var mutantFilterMock = new Mock <IMutantFilter>(MockBehavior.Strict);
var coverageAnalyzerMock = new Mock <ICoverageAnalyser>(MockBehavior.Strict);
// setup mocks
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <IReadOnlyProjectComponent>()));
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(SourceFile).GetRoot());
orchestratorMock.SetupAllProperties();
mutantFilterMock.SetupGet(x => x.DisplayName).Returns("Mock filter");
IEnumerable <Mutant> mutantsPassedToFilter = null;
mutantFilterMock.Setup(x => x.FilterMutants(It.IsAny <IEnumerable <Mutant> >(), It.IsAny <IReadOnlyFileLeaf>(), It.IsAny <StrykerOptions>()))
.Callback <IEnumerable <Mutant>, IReadOnlyFileLeaf, StrykerOptions>((mutants, _, __) => mutantsPassedToFilter = mutants)
.Returns((IEnumerable <Mutant> mutants, IReadOnlyFileLeaf file, StrykerOptions o) => mutants.Take(1));
var target = new MutationTestProcess(input,
reporterMock.Object,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
fSharpOrchestrator: null,
fileSystem,
new BroadcastMutantFilter(new[] { mutantFilterMock.Object }),
coverageAnalyzerMock.Object,
options);
// start mutation process
target.Mutate();
target.FilterMutants();
// verify that compiler error mutants are not passed to filter
mutantsPassedToFilter.ShouldNotContain(compileErrorMutant);
// verify that filtered mutants are skipped
inputFile.Mutants.ShouldContain(mutantToBeSkipped);
mutantToBeSkipped.ResultStatus.ShouldBe(MutantStatus.Ignored);
}
public void ShouldCallMutantOrchestratorAndReporter()
{
var inputFile = new FileLeaf()
{
Name = "Recursive.cs",
SourceCode = SourceFile,
SyntaxTree = CSharpSyntaxTree.ParseText(SourceFile)
};
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
TestProjectAnalyzerResults = new List <ProjectAnalyzerResult> {
new ProjectAnalyzerResult(null, null)
{
AssemblyPath = "/bin/Debug/netcoreapp2.1/TestName.dll",
Properties = new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "AssemblyName", "TestName" }
}
}
},
ProjectUnderTestAnalyzerResult = new ProjectAnalyzerResult(null, null)
{
AssemblyPath = "/bin/Debug/netcoreapp2.1/TestName.dll",
Properties = new Dictionary <string, string>()
{
{ "TargetDir", "/bin/Debug/netcoreapp2.1" },
{ "AssemblyName", "TestName" }
}
},
ProjectContents = new FolderComposite()
{
Name = Path.Combine(FilesystemRoot, "ExampleProject"),
Children = new Collection <ProjectComponent>()
{
inputFile,
}
},
},
AssemblyReferences = _assemblies
};
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(FilesystemRoot, "ExampleProject", "Recursive.cs"), new MockFileData(SourceFile) },
{ Path.Combine(FilesystemRoot, "ExampleProject.Test", "bin", "Debug", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") },
{ Path.Combine(FilesystemRoot, "ExampleProject.Test", "obj", "Release", "netcoreapp2.0", "ExampleProject.dll"), new MockFileData("Bytecode") }
});
var mutantToBeSkipped = new Mutant()
{
Mutation = new Mutation()
};
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}, mutantToBeSkipped
};
// create mocks
var orchestratorMock = new Mock <IMutantOrchestrator>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var compilingProcessMock = new Mock <ICompilingProcess>(MockBehavior.Strict);
// setup mocks
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()));
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(SourceFile).GetRoot());
orchestratorMock.SetupAllProperties();
compilingProcessMock.Setup(x => x.Compile(It.IsAny <IEnumerable <SyntaxTree> >(), It.IsAny <MemoryStream>(), It.IsAny <MemoryStream>(), true))
.Returns(new CompilingProcessResult()
{
Success = true
});
var options = new StrykerOptions(devMode: true, excludedMutations: new string[] { });
var target = new MutationTestProcess(input,
reporterMock.Object,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
compilingProcessMock.Object,
fileSystem,
options,
new BroadcastMutantFilter(Enumerable.Empty <IMutantFilter>()));
// start mutation process
target.Mutate();
// verify the right methods were called
orchestratorMock.Verify(x => x.Mutate(It.IsAny <SyntaxNode>()), Times.Once);
reporterMock.Verify(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()), Times.Once);
}
public void MutateShouldWriteToDisk_IfCompilationIsSuccessful()
{
var folder = new CsharpFolderComposite();
folder.Add(new CsharpFileLeaf
{
SourceCode = SourceFile,
SyntaxTree = CSharpSyntaxTree.ParseText(SourceFile)
});
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ Path.Combine(FilesystemRoot, "SomeFile.cs"), new MockFileData("SomeFile") },
});
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo(fileSystem)
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", Path.Combine(FilesystemRoot, "ProjectUnderTest", "bin", "Debug", "netcoreapp2.0") },
{ "TargetFileName", "ProjectUnderTest.dll" },
{ "AssemblyName", "ProjectUnderTest.dll" },
{ "Language", "C#" }
}).Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", Path.Combine(FilesystemRoot, "TestProject", "bin", "Debug", "netcoreapp2.0") },
{ "TargetFileName", "TestProject.dll" },
{ "Language", "C#" }
}).Object
},
ProjectContents = folder
},
AssemblyReferences = _assemblies
};
var mockMutants = new Collection <Mutant>()
{
new() { Mutation = new Mutation() }
};
// create mocks
var options = new StrykerOptions();
var orchestratorMock = new Mock <BaseMutantOrchestrator <SyntaxNode> >(MockBehavior.Strict, options);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var coverageAnalyzerMock = new Mock <ICoverageAnalyser>(MockBehavior.Strict);
fileSystem.AddDirectory(Path.Combine(FilesystemRoot, "TestProject", "bin", "Debug", "netcoreapp2.0"));
// setup mocks
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(SourceFile).GetRoot());
orchestratorMock.SetupAllProperties();
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <IReadOnlyProjectComponent>()));
var target = new MutationTestProcess(input,
reporterMock.Object,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
fSharpOrchestrator: null,
fileSystem,
new BroadcastMutantFilter(Enumerable.Empty <IMutantFilter>()),
coverageAnalyzerMock.Object,
options);
target.Mutate();
// Verify the created assembly is written to disk on the right location
string expectedPath = Path.Combine(FilesystemRoot, "TestProject", "bin", "Debug", "netcoreapp2.0", "ProjectUnderTest.dll");
fileSystem.ShouldContainFile(expectedPath);
}
public void MutationTestProcess_MutateShouldCallMutantOrchestrator()
{
Skip.IfNot(RunningOnWindows);
string file1 = @"using System;
namespace ExampleProject
{
public class Recursive
{
public int Fibinacci(int len)
{
return Fibonacci(0, 1, 1, len);
}
private int Fibonacci(int a, int b, int counter, int len)
{
if (counter <= len)
{
Console.Write(""{0} "", a);
return Fibonacci(b, a + b, counter + 1, len);
}
return 0;
}
}
}
";
var input = new MutationTestInput()
{
ProjectInfo = new Core.Initialisation.ProjectInfo()
{
TestProjectPath = @"c:\ExampleProject.Test",
ProjectUnderTestPath = @"c:\ExampleProject",
ProjectUnderTestAssemblyName = "ExampleProject",
TargetFramework = "netcoreapp2.0",
ProjectContents = new FolderComposite()
{
Name = @"c:\ExampleProject",
Children = new Collection <ProjectComponent>()
{
new FileLeaf()
{
Name = "Recursive.cs",
SourceCode = file1
}
}
},
},
AssemblyReferences = new ReferenceProvider().GetReferencedAssemblies()
};
var fileSystem = new MockFileSystem(new Dictionary <string, MockFileData>
{
{ @"c:\ExampleProject\Recursive.cs", new MockFileData(file1) },
{ @"c:\ExampleProject.Test\bin\Debug\netcoreapp2.0\ExampleProject.dll", new MockFileData("Bytecode") },
{ @"c:\ExampleProject.Test\obj\Release\netcoreapp2.0\ExampleProject.dll", new MockFileData("Bytecode") }
});
var mockMutants = new Collection <Mutant>()
{
new Mutant()
{
Mutation = new Mutation()
}
};
// create mocks
var orchestratorMock = new Mock <IMutantOrchestrator>(MockBehavior.Strict);
var reporterMock = new Mock <IReporter>(MockBehavior.Strict);
var mutationTestExecutorMock = new Mock <IMutationTestExecutor>(MockBehavior.Strict);
var compilingProcessMock = new Mock <ICompilingProcess>(MockBehavior.Strict);
// setup mocks
reporterMock.Setup(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()));
orchestratorMock.Setup(x => x.GetLatestMutantBatch()).Returns(mockMutants);
orchestratorMock.Setup(x => x.Mutate(It.IsAny <SyntaxNode>())).Returns(CSharpSyntaxTree.ParseText(file1).GetRoot());
orchestratorMock.SetupAllProperties();
compilingProcessMock.Setup(x => x.Compile(It.IsAny <IEnumerable <SyntaxTree> >(), It.IsAny <MemoryStream>()))
.Returns(new CompilingProcessResult()
{
Success = true
});
var options = new StrykerOptions("c:/test", "Console", "", 2000, null, false, 1);
var target = new MutationTestProcess(input,
reporterMock.Object,
null,
mutationTestExecutorMock.Object,
orchestratorMock.Object,
compilingProcessMock.Object,
fileSystem);
// start mutation process
target.Mutate();
// verify the right methods were called
orchestratorMock.Verify(x => x.Mutate(It.IsAny <SyntaxNode>()), Times.Once);
reporterMock.Verify(x => x.OnMutantsCreated(It.IsAny <ProjectComponent>()), Times.Once);
}
public void ShouldRollbackIssueInExpression()
{
var syntaxTree = CSharpSyntaxTree.ParseText(@"
using System;
using System.Collections.Generic;
using System.Linq;
namespace ExampleProject
{
public class Test
{
public void SomeLinq()
{
var list = new List<List<double>>();
int[] listProjected = list.Select(l => l.Count()).ToArray();
}
}
}");
var mutator = new CsharpMutantOrchestrator(options: new StrykerOptions(mutationLevel: MutationLevel.Complete.ToString(), devMode: true));
var helpers = new List <SyntaxTree>();
foreach (var(name, code) in CodeInjection.MutantHelpers)
{
helpers.Add(CSharpSyntaxTree.ParseText(code, path: name, encoding: Encoding.UTF32));
}
var mutant = mutator.Mutate(syntaxTree.GetRoot());
helpers.Add(mutant.SyntaxTree);
var references = new List <PortableExecutableReference>()
{
MetadataReference.CreateFromFile(typeof(object).Assembly.Location),
MetadataReference.CreateFromFile(typeof(List <string>).Assembly.Location),
MetadataReference.CreateFromFile(typeof(Enumerable).Assembly.Location),
MetadataReference.CreateFromFile(typeof(PipeStream).Assembly.Location),
};
Assembly.GetEntryAssembly().GetReferencedAssemblies().ToList().ForEach(a => references.Add(MetadataReference.CreateFromFile(Assembly.Load(a).Location)));
var input = new MutationTestInput()
{
ProjectInfo = new ProjectInfo()
{
ProjectUnderTestAnalyzerResult = TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "TargetDir", "" },
{ "AssemblyName", "AssemblyName" },
{ "TargetFileName", "TargetFileName.dll" },
{ "SignAssembly", "true" },
{ "AssemblyOriginatorKeyFile", Path.GetFullPath(Path.Combine("TestResources", "StrongNameKeyFile.snk")) }
},
projectFilePath: "TestResources").Object,
TestProjectAnalyzerResults = new List <IAnalyzerResult> {
TestHelper.SetupProjectAnalyzerResult(properties: new Dictionary <string, string>()
{
{ "AssemblyName", "AssemblyName" },
}).Object
}
},
AssemblyReferences = references
};
var rollbackProcess = new RollbackProcess();
var target = new CompilingProcess(input, rollbackProcess);
using (var ms = new MemoryStream())
{
var result = target.Compile(helpers, ms, null, true);
result.RollbackResult.RollbackedIds.Count().ShouldBe(1);
}
}
