public async Task <IList <BaselineInfo> > CreateBaselineAsync(MutationConfig config, CancellationToken cancellationToken = default(CancellationToken))
{
Log.Info("Creating baseline and verifying solution/tests..");
_fileSystem.Directory.DeleteDirectoryAndCheckForException(BaselineDirectoryPath);
try
{
await _compileMutationProjectsHandler.CompileMutationProjectsAsync(config, BaselineDirectoryPath, cancellationToken);
var baselineInfos = await _runUnitTestHandler.RunUnitTests(config, BaselineDirectoryPath, cancellationToken);
_logSummaryHandler.ShowBaselineSummary(baselineInfos);
Log.Info("Baseline completed.");
return(baselineInfos);
}
catch (OperationCanceledException)
{
Log.Info("Creating baseline was cancelled by request.");
throw;
}
finally
{
_fileSystem.Directory.DeleteDirectoryAndCheckForException(BaselineDirectoryPath);
}
}
public async void OpenProject(string path)
{
_loadingDisplayer.ShowLoading($"Opening project at {Path.GetFileName(path)}");
MutationConfig config = null;
try
{
config = await Task.Run(() => _mediator.Send(new OpenProjectCommand(path)));
_mutationModuleTabOpener.OpenOverviewTab(config);
}
catch (ValidationException ex)
{
CommonDialogDisplayer.ShowErrorDialog("Unexpected error", "Failed to open project.", ex.Message);
return;
}
catch (OpenProjectException ex)
{
CommonDialogDisplayer.ShowErrorDialog("Unexpected error", "Failed to open project.", ex.InnerException?.ToString());
return;
}
finally
{
_loadingDisplayer.HideLoading();
}
}
private IList <MutationDocument> CreateMutationsForDocument(MutationConfig config, Project currentProject, DocumentId documentId)
{
var mutations = new List <MutationDocument>();
try
{
var document = currentProject.GetDocument(documentId);
if (config.Filter != null && !config.Filter.ResourceAllowed(document.FilePath))
{
Log.Info($"Ignoring {document.Name}.");
return(mutations);
}
Log.Info($"Creating mutation for {document.Name}..");
var root = document.GetSyntaxRootAsync().Result;
foreach (var mutationOperator in config.Mutators)
{
var mutatedDocuments = mutationOperator.GetMutatedDocument(root, document);
mutations.AddRange(mutatedDocuments.Where(m =>
config.Filter == null ||
(config.Filter.ResourceLinesAllowed(document.FilePath, GetDocumentLine(m)) &&
config.Filter.CodeAllowed(document.FilePath, GetDocumentLine(m), m.MutationDetails.Original))));
}
}
catch (Exception ex)
{
Log.Error("Error when creating mutation: " + ex.Message);
}
return(mutations);
}
public async Task <IList <BaselineInfo> > RunUnitTests(MutationConfig config, string baselineDirectoryPath, CancellationToken cancellationToken = default(CancellationToken))
{
var baselineInfos = new List <BaselineInfo>();
foreach (var testProject in config.TestProjects)
{
cancellationToken.ThrowIfCancellationRequested();
var result = await RunTestAsync(testProject, config.DotNetPath, config.MaxTestTimeMin, baselineDirectoryPath, cancellationToken);
if (!result.IsSuccess)
{
var failedTests = result.TestResults.Where(t => !t.IsSuccess);
Log.Error("Unit tests failed with base line");
Log.Error($"Name: {result.Name}");
foreach (var failedTest in failedTests)
{
Log.Error(JObject.FromObject(new { TestName = failedTest.Name, Message = failedTest.InnerText }).ToString());
}
throw new BaselineException("Failed to run all unit tests with baseline which make mutation testing impossible. See log for more details.");
}
Log.Info($"..done ({result.TestResults.Count(t => t.IsSuccess)} tests passed).");
baselineInfos.Add(new BaselineInfo(testProject.Project.Name, result.ExecutionTime));
}
return(baselineInfos);
}
public void SetUp()
{
_fileSystem = new MockFileSystem();
_config = ConfigCreator.CreateConfig(_fileSystem);
_dependency = new TestRunnerDependencyFilesHandler(_fileSystem);
_mutationDocument = new MutationDocumentCreator().CreateMutations(_config).First();
}
public void MutationalEmptyTest()
{
var config = new MutationConfig();
var data = new byte[200];
for (byte x = 0; x < data.Length; x++)
{
data[x] = x;
}
// Without entry
using (var copy = new MemoryStream())
using (var stream = new FuzzingStream(config, data))
{
stream.CopyTo(copy, 64);
CollectionAssert.AreEqual(data, copy.ToArray());
}
// With entry
config.Mutations.Add(new MutationalEntry());
using (var copy = new MemoryStream())
using (var stream = new FuzzingStream(config, data))
{
stream.CopyTo(copy, 64);
CollectionAssert.AreEqual(data, copy.ToArray());
}
}
public void SetUp()
{
_baseDirectory = @"C:\Base";
_mockFileSystem = new MockFileSystem();
_config = ConfigCreator.CreateConfig(_mockFileSystem);
}
public void Initialize(MutationDocument document, MutationConfig config)
{
_config = config;
_document = document;
FileName = document.FileName;
Title = $"{document.FileName} - {document.MutationDetails.Location})";
ShowFullCode(false);
}
public MutationDocumentsOverviewView(MutationConfig config)
{
InitializeComponent();
var viewModel = DataContext as MutationDocumentsOverviewViewModel;
viewModel.Initialize(config);
}
public void SetUp()
{
_fileSystem = new MockFileSystem();
_fileConfig = ConfigCreator.CreateFileConfig();
_config = ConfigCreator.CreateConfig(_fileSystem);
_openProjectWorkspaceHandler = new OpenProjectWorkspaceHandler();
}
public MutationDocumentDetailsView(MutationDocument document, MutationConfig config)
{
InitializeComponent();
var dataContext = DataContext as MutationDocumentDetailsViewModel;
dataContext.Initialize(document, config);
}
public MutationFileDetailsView(FileMutationsModel file, MutationConfig config)
{
InitializeComponent();
var dataContext = DataContext as MutationFileDetailsViewModel;
dataContext.Initialize(file, config);
}
public void SetDocuments(IReadOnlyList <MutationDocument> documents, MutationConfig config)
{
_config = config;
RunningDocuments.AddRange(documents.Select(d => new TestRunDocument {
Document = d, Status = TestRunDocument.TestRunStatusEnum.Waiting
}));
MutationCount = documents.Count;
MutationsInQueueCount = MutationCount;
}
public void SetUp()
{
_fileSystem = new MockFileSystem();
_fileSystem.File.AppendAllText("myFile.sln", "test");
_config = ConfigCreator.CreateConfig(_fileSystem);
var solutionOpener = new SolutionOpenerStub(_config.Solution);
_openProjectSolutionExistHandler = new OpenProjectSolutionHandler(_fileSystem, solutionOpener);
}
public async Task <IList <BaselineInfo> > RunBaselineAsync(MutationConfig config, bool runBaseline, CancellationToken cancellationToken = default(CancellationToken))
{
var baselineInfos = new List <BaselineInfo>();
if (runBaseline)
{
return(await _baselineCreator.CreateBaselineAsync(config, cancellationToken));
}
return(baselineInfos);
}
public ExecuteMutationsCommand(
MutationConfig config,
IList <MutationDocument> mutationDocuments,
Action <MutationDocument> mutationDocumentStartedCallback = null,
Action <MutationDocumentResult> mutationDocumentCompledtedCallback = null)
{
Config = config;
MutationDocuments = mutationDocuments;
MutationDocumentStartedCallback = mutationDocumentStartedCallback;
MutationDocumentCompledtedCallback = mutationDocumentCompledtedCallback;
}
void NewFile()
{
saveToolStripMenuItem.Enabled = false;
_LastFile = "";
MutationConfig c = new MutationConfig();
c.Mutations.Add(new MutationalOffset());
_Cur = c;
propertyGrid1.SelectedObject = _Cur;
propertyGrid1.ExpandAllGridItems();
}
public void SetUp()
{
_fileSystem = new MockFileSystem();
_config = ConfigCreator.CreateConfig(_fileSystem);
_fileConfig = ConfigCreator.CreateFileConfig();
_openProjectCommandHandler = new OpenProjectCommandHandler(
new OpenProjectSolutionHandler(_fileSystem, new SolutionOpenerStub(_config.Solution)),
new OpenProjectMutatorsHandler(),
new OpenProjectGitFilterHandler(new MutationDocumentFilterItemGitDiffCreator(new GitDiffStub())),
new OpenProjectWorkspaceHandler(),
new OpenProjectBaselineHandler(BaselineCreatorCreator.CreatePositiveBaseline(_fileSystem)));
}
public void SetUp()
{
_fileSystem = new MockFileSystem();
_config = ConfigCreator.CreateConfig(_fileSystem);
_mutationDocuments = new MutationDocumentCreator().CreateMutations(_config);
var dependency = new TestRunnerDependencyFilesHandler(_fileSystem);
var compiler = ProjectCompilerCreator.CreatePositiveCompiler(_fileSystem);
var testRunner = TestRunnerClientCreator.CreateNegative();
_mutationExecutor = new MutationDocumentExecutor(compiler, dependency, testRunner, _fileSystem);
}
public IList <MutationDocument> CreateMutations(MutationConfig config, CancellationToken cancellationToken = default(CancellationToken))
{
try
{
Log.Info("Starting to analyze test..");
var mutations = new List <MutationDocument>();
foreach (var mutationProjectInfo in config.MutationProjects)
{
var currentProject = config.Solution.Projects.FirstOrDefault(p => p.Name == mutationProjectInfo.Project.Name);
if (currentProject == null)
{
Log.Error($"Could not find any project with the name {mutationProjectInfo.Project.Name}");
continue;
}
var documentIds = currentProject.DocumentIds;
Log.Info($"Starting to create mutations for {currentProject.Name}..");
foreach (var documentId in documentIds)
{
cancellationToken.ThrowIfCancellationRequested();
mutations.AddRange(CreateMutationsForDocument(config, currentProject, documentId));
}
}
if (!mutations.Any())
{
Log.Warn("Could not find a single mutation. Maybe check your filter?");
}
return(mutations);
}
catch (OperationCanceledException)
{
Log.Info("Cancellation requested");
return(new List <MutationDocument>());
}
catch (Exception ex)
{
Log.Error("Unknown exception when creating mutation documents", ex);
throw new MutationDocumentException("Unknown exception when creating mutation documents", ex);
}
}
/// <summary>
/// Add config
/// </summary>
/// <param name="file">File</param>
public void AddConfig(string file)
{
if (string.IsNullOrEmpty(file))
{
return;
}
if (_Invoker != null && _Invoker.InvokeRequired)
{
_Invoker.Invoke(new delAddCOnfig(AddConfig), new object[] { file });
return;
}
if (!File.Exists(file))
{
return;
}
switch (Path.GetExtension(file).ToLowerInvariant())
{
case ".fmut":
{
MutationConfig c = null;
try { c = MutationConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)); } catch { }
if (c != null)
{
lock (Configurations) Configurations.Add(new FuzzerStat <IFuzzingConfig>(c));
}
break;
}
case ".fpatch":
{
PatchConfig c = null;
try { c = PatchConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)); } catch { }
if (c != null)
{
lock (Configurations) Configurations.Add(new FuzzerStat <IFuzzingConfig>(c));
}
break;
}
}
}
public void LoadFile(string file)
{
_LastFile = null;
switch (Path.GetExtension(file).ToLowerInvariant())
{
case ".fmut":
{
LoadConfig(MutationConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)));
_LastFile = file;
break;
}
case ".fpatch":
{
LoadConfig(PatchConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)));
_LastFile = file;
break;
}
default: LoadConfig(null); break;
}
}
public async Task CompileMutationProjectsAsync(MutationConfig config, string baselineDirectoryPath, CancellationToken cancellationToken = default(CancellationToken))
{
_fileSystem.Directory.CreateDirectory(baselineDirectoryPath);
foreach (var mutationProject in config.MutationProjects)
{
cancellationToken.ThrowIfCancellationRequested();
var project = config.Solution.Projects.FirstOrDefault(p => p.Name == mutationProject.Project.Name);
if (project == null)
{
throw new BaselineException($"Could not find any project with the name {mutationProject.Project.Name} " +
$"in the solution. Currently the solution have these projects: {string.Join(", ", config.Solution.Projects.Select(p => p.Name))}");
}
Log.Info($"Starting to compile {project.Name}..");
var result = await _projectCompiler.CompileAsync(baselineDirectoryPath, project);
if (!result.IsSuccess)
{
Log.Info($".. compiled failed.");
foreach (var compilationError in result.Errors)
{
Log.Error($"{{ Error = \"{compilationError.Message}\", Location = \"{compilationError.Location}\"");
}
throw new BaselineException(
$"Failed to compile {project.Name} in base line.",
new CompilationException(result.Errors.Select(e => e.Message)));
}
Log.Info($".. compiled successfully.");
}
}
public void MutationalEntryPeerByteTest()
{
var config = new MutationConfig()
{
Description = "Test"
};
var entry = new MutationalEntry()
{
FuzzPercent = new FromToValue <double>(100),
ValidOffset = new FromToValue <long>(0, long.MaxValue),
MaxChanges = new FromToValue <ushort>(ushort.MaxValue),
FuzzPercentType = EFuzzingPercentType.PeerByte
};
// Config
config.Mutations.Add(entry);
entry.Changes.Add(new MutationalChange()
{
Weight = 5,
Description = "Add A",
Append = new MutationalFromTo((byte)'A'),
RemoveLength = new FromToValue <ushort>(1),
AppendIterations = new FromToValue <ushort>(1)
});
entry.Changes.Add(new MutationalChange()
{
// Remmove
Weight = 1,
Description = "Remove",
RemoveLength = new FromToValue <ushort>(1),
AppendIterations = new FromToValue <ushort>(1)
});
// 100%
var input = new ManualFuzzingInput(new byte[200]);
using (var stream = new FuzzingStream(config, input.GetStream()))
{
for (long x = 0; x < 200; x++)
{
Assert.IsNotNull(entry.Get(stream, x, 0));
}
}
// 0%
entry.FuzzPercent = new FromToValue <double>(0);
input = new ManualFuzzingInput(new byte[200]);
using (var stream = new FuzzingStream(config, input.GetStream()))
{
for (long x = 0; x < 200; x++)
{
Assert.IsNull(entry.Get(stream, x, 0));
}
}
// Argument excepcion
entry.FuzzPercentType = (EFuzzingPercentType)197;
Assert.Throws <ArgumentException>(() => entry.Get(null, 0, 0));
// Only offset 5
entry.FuzzPercentType = EFuzzingPercentType.PeerByte;
entry.FuzzPercent = new FromToValue <double>(100);
entry.ValidOffset = new FromToValue <long>(5);
input = new ManualFuzzingInput(new byte[100]);
using (var stream = new FuzzingStream(config, input.GetStream()))
{
for (long x = 0; x < 100; x++)
{
var next = entry.Get(stream, x, 0);
if (x == 5)
{
Assert.IsNotNull(next);
}
else
{
Assert.IsNull(next);
}
}
}
// Max changes 2
entry.Changes.RemoveAt(1);
entry.ValidOffset = new FromToValue <long>(0, long.MaxValue);
entry.MaxChanges = new FromToValue <ushort>(2);
input = new ManualFuzzingInput(new byte[100]);
using (var stream = new FuzzingStream(config, input.GetStream()))
{
stream.CopyTo(new MemoryStream(), 16);
Assert.AreEqual(2, stream.Log.Length);
Assert.AreEqual(0, stream.Log[0].Offset);
Assert.AreEqual(1, stream.Log[1].Offset);
}
}
public MutationDocumentsExecutionView(IReadOnlyList <MutationDocument> documents, MutationConfig config)
{
InitializeComponent();
var dataContext = DataContext as MutationDocumentsExecutionViewModel;
dataContext?.SetDocuments(documents, config);
}
public async Task <MutationDocumentResult> ExecuteMutationAsync(
MutationConfig config,
MutationDocument mutationDocument,
CancellationToken cancellationToken = default(CancellationToken))
{
var mutationResult = new MutationDocumentResult
{
Id = mutationDocument.Id,
MutationName = mutationDocument.MutationName,
ProjectName = mutationDocument.ProjectName,
FileName = mutationDocument.FileName,
Location = mutationDocument.MutationDetails.Location,
Orginal = mutationDocument.MutationDetails.Original.ToFullString(),
FullOrginal = mutationDocument.MutationDetails.FullOriginal.ToFullString(),
Mutation = mutationDocument.MutationDetails.Mutation.ToFullString(),
FullMutation = mutationDocument.MutationDetails.FullMutation.ToFullString(),
Category = mutationDocument.MutationDetails.Category
};
mutationResult.GenerateHash();
Log.Info($"Running mutation: \"{mutationDocument.MutationName}\"");
var results = new List <TestSuiteResult>();
// Create the temporary "head" mutation directory to run all tests
var mutationDirectoryPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "TestRun", mutationDocument.Id.ToString());
// Where we should save our compiled mutation
var mutationDllPath = Path.Combine(mutationDirectoryPath, $"{config.MutationProjects.FirstOrDefault(m => m.Project.Name == mutationDocument.ProjectName).Project.OutputFileName}");
_fileSystem.Directory.DeleteDirectoryAndCheckForException(mutationDirectoryPath);
try
{
cancellationToken.ThrowIfCancellationRequested();
_fileSystem.Directory.CreateDirectory(mutationDirectoryPath);
mutationResult.CompilationResult = await _compiler.CompileAsync(mutationDllPath, mutationDocument);
if (!mutationResult.CompilationResult.IsSuccess)
{
return(mutationResult);
}
foreach (var testProject in config.TestProjects)
{
cancellationToken.ThrowIfCancellationRequested();
if (ShouldIgnoreTestProject(mutationDocument, config.MutationProjects, testProject))
{
continue;
}
var baseline = config.BaselineInfos.FirstOrDefault(b => b.TestProjectName.Equals(testProject.Project.Name, StringComparison.OrdinalIgnoreCase));
var result = await RunTestAsync(
testProject,
mutationDirectoryPath,
mutationDllPath,
config.DotNetPath,
baseline?.GetTestProjectTimeout() ?? TimeSpan.FromMinutes(config.MaxTestTimeMin),
cancellationToken);
results.Add(result);
if (results.Any(r => !r.IsSuccess))
{
break;
}
}
var final = CombineResult(mutationDocument.FileName, results);
if (final.TestResults.Count == 0)
{
throw new MutationDocumentException($"Unknown error when running, we should not have 0 tests. Also make sure that you don't have bad project mapping. Error: \"{final.Name}\"");
}
Log.Info($"\"{mutationDocument.MutationName}\" done. Ran {final.TestResults.Count} tests and {final.TestResults.Count(t => !t.IsSuccess)} failed.");
mutationResult.FailedTests = final.TestResults.Where(t => !t.IsSuccess).ToList();
mutationResult.TestsRunCount = final.TestResults.Count;
}
catch (OperationCanceledException)
{
Log.Info("Cancellation requested (single mutation)");
mutationResult.UnexpectedError = "Mutation cancelled";
}
catch (Exception)
{
throw;
}
finally
{
_fileSystem.Directory.DeleteDirectoryAndCheckForException(mutationDirectoryPath);
}
return(mutationResult);
}
public void SetUp()
{
_fileSystem = new MockFileSystem();
_config = ConfigCreator.CreateConfig();
}
public void MutationalEntryPeerStreamTest()
{
var config = new MutationConfig()
{
Description = "Test"
};
var entry = new MutationalEntry()
{
FuzzPercent = new FromToValue <double>(100),
ValidOffset = new FromToValue <long>(0, long.MaxValue),
MaxChanges = new FromToValue <ushort>(50),
FuzzPercentType = EFuzzingPercentType.PeerStream
};
// Config
config.Mutations.Add(entry);
entry.Changes.Add(new MutationalChange()
{
Weight = 1,
Append = new MutationalFromTo(0x01),
RemoveLength = new FromToValue <ushort>(),
AppendIterations = new FromToValue <ushort>(1)
});
// 100% / 50 changes
var input = new ManualFuzzingInput(new byte[200]);
using (var copy = new MemoryStream())
using (var stream = new FuzzingStream(config, input.GetStream()))
{
stream.CopyTo(copy, 200);
Assert.AreEqual(50, copy.ToArray().Count(u => u == 0x01));
}
// 0%
entry.FuzzPercent = new FromToValue <double>(0);
input = new ManualFuzzingInput(new byte[200]);
using (var copy = new MemoryStream())
using (var stream = new FuzzingStream(config, input.GetStream()))
{
stream.CopyTo(copy, 200);
Assert.AreEqual(200, copy.ToArray().Count(u => u == 0x00));
}
// Only offset 5
entry.FuzzPercent = new FromToValue <double>(100);
entry.ValidOffset = new FromToValue <long>(5);
entry.MaxChanges = new FromToValue <ushort>(1);
input = new ManualFuzzingInput(new byte[200]);
using (var stream = new FuzzingStream(config, input.GetStream()))
{
for (long x = 0; x < 200; x++)
{
var next = entry.Get(stream, x, 0);
if (x == 5)
{
Assert.IsNotNull(next);
}
else
{
Assert.IsNull(next);
}
}
}
// Max changes 2
entry.ValidOffset = new FromToValue <long>(0, long.MaxValue);
entry.MaxChanges = new FromToValue <ushort>(2);
input = new ManualFuzzingInput(new byte[200]);
using (var stream = new FuzzingStream(config, input.GetStream()))
{
stream.CopyTo(new MemoryStream(), 200);
Assert.AreEqual(2, stream.Log.Length);
}
}
public void Initialize(FileMutationsModel file, MutationConfig config)
{
_config = config;
File = file;
FileName = File.FileName;
}
public void SetUp()
{
_config = ConfigCreator.CreateConfig();
_fileSystem = new MockFileSystem();
_testRunnerDependencyFilesHandler = new TestRunnerDependencyFilesHandler(_fileSystem);
}