Example #1
0
        public async Task <IList <BaselineInfo> > CreateBaselineAsync(MutationConfig config, CancellationToken cancellationToken = default(CancellationToken))
        {
            Log.Info("Creating baseline and verifying solution/tests..");
            _fileSystem.Directory.DeleteDirectoryAndCheckForException(BaselineDirectoryPath);

            try
            {
                await _compileMutationProjectsHandler.CompileMutationProjectsAsync(config, BaselineDirectoryPath, cancellationToken);

                var baselineInfos = await _runUnitTestHandler.RunUnitTests(config, BaselineDirectoryPath, cancellationToken);

                _logSummaryHandler.ShowBaselineSummary(baselineInfos);

                Log.Info("Baseline completed.");
                return(baselineInfos);
            }
            catch (OperationCanceledException)
            {
                Log.Info("Creating baseline was cancelled by request.");
                throw;
            }
            finally
            {
                _fileSystem.Directory.DeleteDirectoryAndCheckForException(BaselineDirectoryPath);
            }
        }
        public async void OpenProject(string path)
        {
            _loadingDisplayer.ShowLoading($"Opening project at {Path.GetFileName(path)}");
            MutationConfig config = null;

            try
            {
                config = await Task.Run(() => _mediator.Send(new OpenProjectCommand(path)));
                _mutationModuleTabOpener.OpenOverviewTab(config);
            }
            catch (ValidationException ex)
            {
                CommonDialogDisplayer.ShowErrorDialog("Unexpected error", "Failed to open project.", ex.Message);
                return;
            }
            catch (OpenProjectException ex)
            {
                CommonDialogDisplayer.ShowErrorDialog("Unexpected error", "Failed to open project.", ex.InnerException?.ToString());
                return;
            }
            finally
            {
                _loadingDisplayer.HideLoading();
            }
        }
        private IList <MutationDocument> CreateMutationsForDocument(MutationConfig config, Project currentProject, DocumentId documentId)
        {
            var mutations = new List <MutationDocument>();

            try
            {
                var document = currentProject.GetDocument(documentId);

                if (config.Filter != null && !config.Filter.ResourceAllowed(document.FilePath))
                {
                    Log.Info($"Ignoring {document.Name}.");
                    return(mutations);
                }

                Log.Info($"Creating mutation for {document.Name}..");

                var root = document.GetSyntaxRootAsync().Result;

                foreach (var mutationOperator in config.Mutators)
                {
                    var mutatedDocuments = mutationOperator.GetMutatedDocument(root, document);
                    mutations.AddRange(mutatedDocuments.Where(m =>
                                                              config.Filter == null ||
                                                              (config.Filter.ResourceLinesAllowed(document.FilePath, GetDocumentLine(m)) &&
                                                               config.Filter.CodeAllowed(document.FilePath, GetDocumentLine(m), m.MutationDetails.Original))));
                }
            }
            catch (Exception ex)
            {
                Log.Error("Error when creating mutation: " + ex.Message);
            }

            return(mutations);
        }
Example #4
0
        public async Task <IList <BaselineInfo> > RunUnitTests(MutationConfig config, string baselineDirectoryPath, CancellationToken cancellationToken = default(CancellationToken))
        {
            var baselineInfos = new List <BaselineInfo>();

            foreach (var testProject in config.TestProjects)
            {
                cancellationToken.ThrowIfCancellationRequested();

                var result = await RunTestAsync(testProject, config.DotNetPath, config.MaxTestTimeMin, baselineDirectoryPath, cancellationToken);

                if (!result.IsSuccess)
                {
                    var failedTests = result.TestResults.Where(t => !t.IsSuccess);
                    Log.Error("Unit tests failed with base line");
                    Log.Error($"Name: {result.Name}");

                    foreach (var failedTest in failedTests)
                    {
                        Log.Error(JObject.FromObject(new { TestName = failedTest.Name, Message = failedTest.InnerText }).ToString());
                    }

                    throw new BaselineException("Failed to run all unit tests with baseline which make mutation testing impossible. See log for more details.");
                }

                Log.Info($"..done ({result.TestResults.Count(t => t.IsSuccess)} tests passed).");
                baselineInfos.Add(new BaselineInfo(testProject.Project.Name, result.ExecutionTime));
            }

            return(baselineInfos);
        }
 public void SetUp()
 {
     _fileSystem       = new MockFileSystem();
     _config           = ConfigCreator.CreateConfig(_fileSystem);
     _dependency       = new TestRunnerDependencyFilesHandler(_fileSystem);
     _mutationDocument = new MutationDocumentCreator().CreateMutations(_config).First();
 }
Example #6
0
        public void MutationalEmptyTest()
        {
            var config = new MutationConfig();

            var data = new byte[200];

            for (byte x = 0; x < data.Length; x++)
            {
                data[x] = x;
            }

            // Without entry

            using (var copy = new MemoryStream())
                using (var stream = new FuzzingStream(config, data))
                {
                    stream.CopyTo(copy, 64);

                    CollectionAssert.AreEqual(data, copy.ToArray());
                }

            // With entry

            config.Mutations.Add(new MutationalEntry());

            using (var copy = new MemoryStream())
                using (var stream = new FuzzingStream(config, data))
                {
                    stream.CopyTo(copy, 64);

                    CollectionAssert.AreEqual(data, copy.ToArray());
                }
        }
        public void SetUp()
        {
            _baseDirectory = @"C:\Base";

            _mockFileSystem = new MockFileSystem();
            _config         = ConfigCreator.CreateConfig(_mockFileSystem);
        }
 public void Initialize(MutationDocument document, MutationConfig config)
 {
     _config   = config;
     _document = document;
     FileName  = document.FileName;
     Title     = $"{document.FileName} - {document.MutationDetails.Location})";
     ShowFullCode(false);
 }
Example #9
0
        public MutationDocumentsOverviewView(MutationConfig config)
        {
            InitializeComponent();

            var viewModel = DataContext as MutationDocumentsOverviewViewModel;

            viewModel.Initialize(config);
        }
Example #10
0
        public void SetUp()
        {
            _fileSystem = new MockFileSystem();
            _fileConfig = ConfigCreator.CreateFileConfig();
            _config     = ConfigCreator.CreateConfig(_fileSystem);

            _openProjectWorkspaceHandler = new OpenProjectWorkspaceHandler();
        }
        public MutationDocumentDetailsView(MutationDocument document, MutationConfig config)
        {
            InitializeComponent();

            var dataContext = DataContext as MutationDocumentDetailsViewModel;

            dataContext.Initialize(document, config);
        }
Example #12
0
        public MutationFileDetailsView(FileMutationsModel file, MutationConfig config)
        {
            InitializeComponent();

            var dataContext = DataContext as MutationFileDetailsViewModel;

            dataContext.Initialize(file, config);
        }
 public void SetDocuments(IReadOnlyList <MutationDocument> documents, MutationConfig config)
 {
     _config = config;
     RunningDocuments.AddRange(documents.Select(d => new TestRunDocument {
         Document = d, Status = TestRunDocument.TestRunStatusEnum.Waiting
     }));
     MutationCount         = documents.Count;
     MutationsInQueueCount = MutationCount;
 }
        public void SetUp()
        {
            _fileSystem = new MockFileSystem();
            _fileSystem.File.AppendAllText("myFile.sln", "test");
            _config = ConfigCreator.CreateConfig(_fileSystem);

            var solutionOpener = new SolutionOpenerStub(_config.Solution);

            _openProjectSolutionExistHandler = new OpenProjectSolutionHandler(_fileSystem, solutionOpener);
        }
Example #15
0
        public async Task <IList <BaselineInfo> > RunBaselineAsync(MutationConfig config, bool runBaseline, CancellationToken cancellationToken = default(CancellationToken))
        {
            var baselineInfos = new List <BaselineInfo>();

            if (runBaseline)
            {
                return(await _baselineCreator.CreateBaselineAsync(config, cancellationToken));
            }

            return(baselineInfos);
        }
Example #16
0
 public ExecuteMutationsCommand(
     MutationConfig config,
     IList <MutationDocument> mutationDocuments,
     Action <MutationDocument> mutationDocumentStartedCallback          = null,
     Action <MutationDocumentResult> mutationDocumentCompledtedCallback = null)
 {
     Config            = config;
     MutationDocuments = mutationDocuments;
     MutationDocumentStartedCallback    = mutationDocumentStartedCallback;
     MutationDocumentCompledtedCallback = mutationDocumentCompledtedCallback;
 }
Example #17
0
        void NewFile()
        {
            saveToolStripMenuItem.Enabled = false;
            _LastFile = "";

            MutationConfig c = new MutationConfig();

            c.Mutations.Add(new MutationalOffset());
            _Cur = c;
            propertyGrid1.SelectedObject = _Cur;
            propertyGrid1.ExpandAllGridItems();
        }
Example #18
0
        public void SetUp()
        {
            _fileSystem = new MockFileSystem();
            _config     = ConfigCreator.CreateConfig(_fileSystem);
            _fileConfig = ConfigCreator.CreateFileConfig();

            _openProjectCommandHandler = new OpenProjectCommandHandler(
                new OpenProjectSolutionHandler(_fileSystem, new SolutionOpenerStub(_config.Solution)),
                new OpenProjectMutatorsHandler(),
                new OpenProjectGitFilterHandler(new MutationDocumentFilterItemGitDiffCreator(new GitDiffStub())),
                new OpenProjectWorkspaceHandler(),
                new OpenProjectBaselineHandler(BaselineCreatorCreator.CreatePositiveBaseline(_fileSystem)));
        }
        public void SetUp()
        {
            _fileSystem        = new MockFileSystem();
            _config            = ConfigCreator.CreateConfig(_fileSystem);
            _mutationDocuments = new MutationDocumentCreator().CreateMutations(_config);

            var dependency = new TestRunnerDependencyFilesHandler(_fileSystem);

            var compiler   = ProjectCompilerCreator.CreatePositiveCompiler(_fileSystem);
            var testRunner = TestRunnerClientCreator.CreateNegative();

            _mutationExecutor = new MutationDocumentExecutor(compiler, dependency, testRunner, _fileSystem);
        }
        public IList <MutationDocument> CreateMutations(MutationConfig config, CancellationToken cancellationToken = default(CancellationToken))
        {
            try
            {
                Log.Info("Starting to analyze test..");

                var mutations = new List <MutationDocument>();

                foreach (var mutationProjectInfo in config.MutationProjects)
                {
                    var currentProject = config.Solution.Projects.FirstOrDefault(p => p.Name == mutationProjectInfo.Project.Name);

                    if (currentProject == null)
                    {
                        Log.Error($"Could not find any project with the name {mutationProjectInfo.Project.Name}");
                        continue;
                    }

                    var documentIds = currentProject.DocumentIds;

                    Log.Info($"Starting to create mutations for {currentProject.Name}..");
                    foreach (var documentId in documentIds)
                    {
                        cancellationToken.ThrowIfCancellationRequested();
                        mutations.AddRange(CreateMutationsForDocument(config, currentProject, documentId));
                    }
                }

                if (!mutations.Any())
                {
                    Log.Warn("Could not find a single mutation. Maybe check your filter?");
                }

                return(mutations);
            }
            catch (OperationCanceledException)
            {
                Log.Info("Cancellation requested");
                return(new List <MutationDocument>());
            }
            catch (Exception ex)
            {
                Log.Error("Unknown exception when creating mutation documents", ex);
                throw new MutationDocumentException("Unknown exception when creating mutation documents", ex);
            }
        }
Example #21
0
        /// <summary>
        /// Add config
        /// </summary>
        /// <param name="file">File</param>
        public void AddConfig(string file)
        {
            if (string.IsNullOrEmpty(file))
            {
                return;
            }

            if (_Invoker != null && _Invoker.InvokeRequired)
            {
                _Invoker.Invoke(new delAddCOnfig(AddConfig), new object[] { file });
                return;
            }

            if (!File.Exists(file))
            {
                return;
            }

            switch (Path.GetExtension(file).ToLowerInvariant())
            {
            case ".fmut":
            {
                MutationConfig c = null;
                try { c = MutationConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)); } catch { }
                if (c != null)
                {
                    lock (Configurations) Configurations.Add(new FuzzerStat <IFuzzingConfig>(c));
                }
                break;
            }

            case ".fpatch":
            {
                PatchConfig c = null;
                try { c = PatchConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)); } catch { }
                if (c != null)
                {
                    lock (Configurations) Configurations.Add(new FuzzerStat <IFuzzingConfig>(c));
                }
                break;
            }
            }
        }
Example #22
0
        public void LoadFile(string file)
        {
            _LastFile = null;
            switch (Path.GetExtension(file).ToLowerInvariant())
            {
            case ".fmut":
            {
                LoadConfig(MutationConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)));
                _LastFile = file;
                break;
            }

            case ".fpatch":
            {
                LoadConfig(PatchConfig.FromJson(File.ReadAllText(file, Encoding.UTF8)));
                _LastFile = file;
                break;
            }

            default: LoadConfig(null); break;
            }
        }
Example #23
0
        public async Task CompileMutationProjectsAsync(MutationConfig config, string baselineDirectoryPath, CancellationToken cancellationToken = default(CancellationToken))
        {
            _fileSystem.Directory.CreateDirectory(baselineDirectoryPath);

            foreach (var mutationProject in config.MutationProjects)
            {
                cancellationToken.ThrowIfCancellationRequested();

                var project = config.Solution.Projects.FirstOrDefault(p => p.Name == mutationProject.Project.Name);
                if (project == null)
                {
                    throw new BaselineException($"Could not find any project with the name {mutationProject.Project.Name} " +
                                                $"in the solution. Currently the solution have these projects: {string.Join(", ", config.Solution.Projects.Select(p => p.Name))}");
                }

                Log.Info($"Starting to compile {project.Name}..");

                var result = await _projectCompiler.CompileAsync(baselineDirectoryPath, project);

                if (!result.IsSuccess)
                {
                    Log.Info($".. compiled failed.");

                    foreach (var compilationError in result.Errors)
                    {
                        Log.Error($"{{ Error = \"{compilationError.Message}\", Location = \"{compilationError.Location}\"");
                    }

                    throw new BaselineException(
                              $"Failed to compile {project.Name} in base line.",
                              new CompilationException(result.Errors.Select(e => e.Message)));
                }

                Log.Info($".. compiled successfully.");
            }
        }
Example #24
0
        public void MutationalEntryPeerByteTest()
        {
            var config = new MutationConfig()
            {
                Description = "Test"
            };
            var entry = new MutationalEntry()
            {
                FuzzPercent     = new FromToValue <double>(100),
                ValidOffset     = new FromToValue <long>(0, long.MaxValue),
                MaxChanges      = new FromToValue <ushort>(ushort.MaxValue),
                FuzzPercentType = EFuzzingPercentType.PeerByte
            };

            // Config

            config.Mutations.Add(entry);
            entry.Changes.Add(new MutationalChange()
            {
                Weight           = 5,
                Description      = "Add A",
                Append           = new MutationalFromTo((byte)'A'),
                RemoveLength     = new FromToValue <ushort>(1),
                AppendIterations = new FromToValue <ushort>(1)
            });
            entry.Changes.Add(new MutationalChange()
            {
                // Remmove
                Weight           = 1,
                Description      = "Remove",
                RemoveLength     = new FromToValue <ushort>(1),
                AppendIterations = new FromToValue <ushort>(1)
            });

            // 100%

            var input = new ManualFuzzingInput(new byte[200]);

            using (var stream = new FuzzingStream(config, input.GetStream()))
            {
                for (long x = 0; x < 200; x++)
                {
                    Assert.IsNotNull(entry.Get(stream, x, 0));
                }
            }

            // 0%

            entry.FuzzPercent = new FromToValue <double>(0);

            input = new ManualFuzzingInput(new byte[200]);
            using (var stream = new FuzzingStream(config, input.GetStream()))
            {
                for (long x = 0; x < 200; x++)
                {
                    Assert.IsNull(entry.Get(stream, x, 0));
                }
            }

            // Argument excepcion

            entry.FuzzPercentType = (EFuzzingPercentType)197;
            Assert.Throws <ArgumentException>(() => entry.Get(null, 0, 0));

            // Only offset 5

            entry.FuzzPercentType = EFuzzingPercentType.PeerByte;
            entry.FuzzPercent     = new FromToValue <double>(100);
            entry.ValidOffset     = new FromToValue <long>(5);

            input = new ManualFuzzingInput(new byte[100]);
            using (var stream = new FuzzingStream(config, input.GetStream()))
            {
                for (long x = 0; x < 100; x++)
                {
                    var next = entry.Get(stream, x, 0);

                    if (x == 5)
                    {
                        Assert.IsNotNull(next);
                    }
                    else
                    {
                        Assert.IsNull(next);
                    }
                }
            }

            // Max changes 2

            entry.Changes.RemoveAt(1);
            entry.ValidOffset = new FromToValue <long>(0, long.MaxValue);
            entry.MaxChanges  = new FromToValue <ushort>(2);
            input             = new ManualFuzzingInput(new byte[100]);

            using (var stream = new FuzzingStream(config, input.GetStream()))
            {
                stream.CopyTo(new MemoryStream(), 16);

                Assert.AreEqual(2, stream.Log.Length);
                Assert.AreEqual(0, stream.Log[0].Offset);
                Assert.AreEqual(1, stream.Log[1].Offset);
            }
        }
        public MutationDocumentsExecutionView(IReadOnlyList <MutationDocument> documents, MutationConfig config)
        {
            InitializeComponent();

            var dataContext = DataContext as MutationDocumentsExecutionViewModel;

            dataContext?.SetDocuments(documents, config);
        }
Example #26
0
        public async Task <MutationDocumentResult> ExecuteMutationAsync(
            MutationConfig config,
            MutationDocument mutationDocument,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            var mutationResult = new MutationDocumentResult
            {
                Id           = mutationDocument.Id,
                MutationName = mutationDocument.MutationName,
                ProjectName  = mutationDocument.ProjectName,
                FileName     = mutationDocument.FileName,
                Location     = mutationDocument.MutationDetails.Location,
                Orginal      = mutationDocument.MutationDetails.Original.ToFullString(),
                FullOrginal  = mutationDocument.MutationDetails.FullOriginal.ToFullString(),
                Mutation     = mutationDocument.MutationDetails.Mutation.ToFullString(),
                FullMutation = mutationDocument.MutationDetails.FullMutation.ToFullString(),
                Category     = mutationDocument.MutationDetails.Category
            };

            mutationResult.GenerateHash();

            Log.Info($"Running mutation: \"{mutationDocument.MutationName}\"");

            var results = new List <TestSuiteResult>();

            // Create the temporary "head" mutation directory to run all tests
            var mutationDirectoryPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "TestRun", mutationDocument.Id.ToString());

            // Where we should save our compiled mutation
            var mutationDllPath = Path.Combine(mutationDirectoryPath, $"{config.MutationProjects.FirstOrDefault(m => m.Project.Name == mutationDocument.ProjectName).Project.OutputFileName}");

            _fileSystem.Directory.DeleteDirectoryAndCheckForException(mutationDirectoryPath);

            try
            {
                cancellationToken.ThrowIfCancellationRequested();

                _fileSystem.Directory.CreateDirectory(mutationDirectoryPath);

                mutationResult.CompilationResult = await _compiler.CompileAsync(mutationDllPath, mutationDocument);

                if (!mutationResult.CompilationResult.IsSuccess)
                {
                    return(mutationResult);
                }

                foreach (var testProject in config.TestProjects)
                {
                    cancellationToken.ThrowIfCancellationRequested();

                    if (ShouldIgnoreTestProject(mutationDocument, config.MutationProjects, testProject))
                    {
                        continue;
                    }

                    var baseline = config.BaselineInfos.FirstOrDefault(b => b.TestProjectName.Equals(testProject.Project.Name, StringComparison.OrdinalIgnoreCase));
                    var result   = await RunTestAsync(
                        testProject,
                        mutationDirectoryPath,
                        mutationDllPath,
                        config.DotNetPath,
                        baseline?.GetTestProjectTimeout() ?? TimeSpan.FromMinutes(config.MaxTestTimeMin),
                        cancellationToken);

                    results.Add(result);

                    if (results.Any(r => !r.IsSuccess))
                    {
                        break;
                    }
                }

                var final = CombineResult(mutationDocument.FileName, results);

                if (final.TestResults.Count == 0)
                {
                    throw new MutationDocumentException($"Unknown error when running, we should not have 0 tests. Also make sure that you don't have bad project mapping. Error: \"{final.Name}\"");
                }

                Log.Info($"\"{mutationDocument.MutationName}\" done. Ran {final.TestResults.Count} tests and {final.TestResults.Count(t => !t.IsSuccess)} failed.");

                mutationResult.FailedTests   = final.TestResults.Where(t => !t.IsSuccess).ToList();
                mutationResult.TestsRunCount = final.TestResults.Count;
            }
            catch (OperationCanceledException)
            {
                Log.Info("Cancellation requested (single mutation)");
                mutationResult.UnexpectedError = "Mutation cancelled";
            }
            catch (Exception)
            {
                throw;
            }
            finally
            {
                _fileSystem.Directory.DeleteDirectoryAndCheckForException(mutationDirectoryPath);
            }

            return(mutationResult);
        }
 public void SetUp()
 {
     _fileSystem = new MockFileSystem();
     _config     = ConfigCreator.CreateConfig();
 }
Example #28
0
        public void MutationalEntryPeerStreamTest()
        {
            var config = new MutationConfig()
            {
                Description = "Test"
            };
            var entry = new MutationalEntry()
            {
                FuzzPercent     = new FromToValue <double>(100),
                ValidOffset     = new FromToValue <long>(0, long.MaxValue),
                MaxChanges      = new FromToValue <ushort>(50),
                FuzzPercentType = EFuzzingPercentType.PeerStream
            };

            // Config

            config.Mutations.Add(entry);
            entry.Changes.Add(new MutationalChange()
            {
                Weight           = 1,
                Append           = new MutationalFromTo(0x01),
                RemoveLength     = new FromToValue <ushort>(),
                AppendIterations = new FromToValue <ushort>(1)
            });

            // 100% / 50 changes

            var input = new ManualFuzzingInput(new byte[200]);

            using (var copy = new MemoryStream())
                using (var stream = new FuzzingStream(config, input.GetStream()))
                {
                    stream.CopyTo(copy, 200);
                    Assert.AreEqual(50, copy.ToArray().Count(u => u == 0x01));
                }

            // 0%

            entry.FuzzPercent = new FromToValue <double>(0);

            input = new ManualFuzzingInput(new byte[200]);
            using (var copy = new MemoryStream())
                using (var stream = new FuzzingStream(config, input.GetStream()))
                {
                    stream.CopyTo(copy, 200);
                    Assert.AreEqual(200, copy.ToArray().Count(u => u == 0x00));
                }

            // Only offset 5

            entry.FuzzPercent = new FromToValue <double>(100);
            entry.ValidOffset = new FromToValue <long>(5);
            entry.MaxChanges  = new FromToValue <ushort>(1);

            input = new ManualFuzzingInput(new byte[200]);
            using (var stream = new FuzzingStream(config, input.GetStream()))
            {
                for (long x = 0; x < 200; x++)
                {
                    var next = entry.Get(stream, x, 0);

                    if (x == 5)
                    {
                        Assert.IsNotNull(next);
                    }
                    else
                    {
                        Assert.IsNull(next);
                    }
                }
            }

            // Max changes 2

            entry.ValidOffset = new FromToValue <long>(0, long.MaxValue);
            entry.MaxChanges  = new FromToValue <ushort>(2);

            input = new ManualFuzzingInput(new byte[200]);
            using (var stream = new FuzzingStream(config, input.GetStream()))
            {
                stream.CopyTo(new MemoryStream(), 200);

                Assert.AreEqual(2, stream.Log.Length);
            }
        }
Example #29
0
 public void Initialize(FileMutationsModel file, MutationConfig config)
 {
     _config  = config;
     File     = file;
     FileName = File.FileName;
 }
Example #30
0
 public void SetUp()
 {
     _config     = ConfigCreator.CreateConfig();
     _fileSystem = new MockFileSystem();
     _testRunnerDependencyFilesHandler = new TestRunnerDependencyFilesHandler(_fileSystem);
 }