public long Create(long?projectId, string name) { if (string.IsNullOrEmpty(name)) { throw new ArgumentNullException(nameof(name)); } using (var telemetryScope = _telemetryScopeProvider.Create <Queries>(TelemetryOperationNames.Query.Create)) { try { var canCreateQuery = _userAuthorityValidator.HasUserAuthorities( _userPrincipal.Info.Id, new[] { Authorities.UI.Queries.CreateQuery }, projectId); if (!canCreateQuery) { throw new UnauthorizedAccessException(); } var query = new Queries { Comment = null, CreatedById = _userPrincipal.Info.Id, CreatedUtc = _timeService.GetUtc(), JsonQuery = null, ModifiedById = _userPrincipal.Info.Id, ModifiedUtc = _timeService.GetUtc(), Name = name, Privacy = (int)QueryPrivacyType.Private, ProjectId = projectId, Query = null, TargetCultureId = null, Visibility = (int)QueryVisibilityType.Closed }; telemetryScope.SetEntity(query); _queryRepository.Insert(query); _queryRepository.Save(); telemetryScope.WriteSuccess(); return(query.Id); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }
public long Add(long?projectId, string name, string desc, string rule, bool isSystem) { if (string.IsNullOrEmpty(name)) { throw new ArgumentNullException(nameof(name)); } if (string.IsNullOrEmpty(rule)) { throw new ArgumentNullException(nameof(rule)); } using (var telemetryScope = _telemetryScopeProvider.Create <Reports>(TelemetryOperationNames.Report.Create)) { try { if (_reportRepository.Get(projectId, name).Any()) { throw new ReportAlreadyExistsException(name); } if (!_reportAuthorityValidator.CanCreate(_userPrincipal.Info.Id, projectId)) { throw new UnauthorizedAccessException(); } var report = new Reports { DisplayName = name, Created = _timeService.GetUtc(), CreatedById = _userPrincipal.Info.Id, Description = desc, Modified = _timeService.GetUtc(), ModifiedById = _userPrincipal.Info.Id, ProjectId = projectId, Rule = rule, IsSystem = isSystem }; telemetryScope.SetEntity(report); _reportRepository.Insert(report); _reportRepository.Save(); telemetryScope.WriteSuccess(); return(report.Id); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }
public ReportFile Build( long reportId, long userId, IReadOnlyDictionary <string, object> parameters, ReportFileType reportFileType) { using (var telemetryScope = _telemetryScopeProvider.Create <Reports>(TelemetryOperationNames.Report.Generate)) { try { var report = GetReport(reportId); telemetryScope.SetEntity(report); var reportBundle = GetReportBundle(userId, parameters, report); var reportFile = TranslateReport(reportFileType, reportBundle); SaveReportFileInTempStorage(userId, reportBundle, reportFile); telemetryScope.WriteSuccess(); return(reportFile); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }
public void Handle(Event eventToHandle) { using (var telemetryScope = _telemetryScopeProvider.Create <Tasks>(TelemetryOperationNames.Task.Create)) { try { var projectId = long.Parse(eventToHandle.Data[Variables.ProjectId]); var task = new Tasks { Created = _timeService.GetUtc(), CreatedById = _userPrincipal.Info.Id, Finished = null, Modified = _timeService.GetUtc(), ModifiedById = _userPrincipal.Info.Id, ProjectId = projectId, Repository = eventToHandle.Data[Variables.Branch], SdlStatus = (int)SdlPolicyStatus.Unknown }; telemetryScope.SetEntity(task); _taskRepository.Insert(task); // TODO добавить копирования настроек с проекта _eventProvider.Publish( new Event { Key = EventKeys.ScanTask.Created, Data = new Dictionary <string, string> { { Variables.ProjectId, projectId.ToString() }, { Variables.TaskId, task.Id.ToString() } } }); telemetryScope.WriteSuccess(); _taskRepository.Save(); _log.Debug(Resources.TaskCreated.FormatWith(task.Id, projectId, task.Repository)); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }
private QueryResult ExecutePlainTextQuery(long?userId, Queries query, KeyValuePair <string, string>[] parameters) { userId = userId ?? _userPrincipal.Info.Id; var queryResult = new QueryResult(); using (var telemetryScope = _telemetryScopeProvider.Create <Queries>(TelemetryOperationNames.Query.Execute)) { telemetryScope.SetEntity(query); try { var queryText = ApplyQueryParameters(query.Query, parameters); var queryExpr = _dslParser.DataQueryParse(queryText); queryResult.Columns = _dslDataQueryEvaluator.Evaluate(queryExpr, userId.Value); queryResult.Items = _queryBuilder.ExecuteTable(queryExpr).ToArray(); // generalize array telemetryScope.WriteSuccess(); } catch (DataQueryCompilationException ex) { telemetryScope.WriteException(ex); queryResult.Exceptions = ex.Errors .Select( _ => new QueryException { Message = $"{_.Item1}: {_.Item2}" }); } catch (Exception ex) { telemetryScope.WriteException(ex); queryResult.Exceptions = new[] { new QueryException { Message = ex.Message, StackTrace = ex.StackTrace } }; } } return(queryResult); }
protected override void ProcessAuthorized(CreateTaskCommand command) { using (var telemetryScope = _telemetryScopeProvider.Create <Tasks>(TelemetryOperationNames.Task.Create)) { try { var currentDateTime = _timeService.GetUtc(); var task = new Tasks { Created = currentDateTime, CreatedById = _userPrincipal.Info.Id, Finished = null, Modified = currentDateTime, ModifiedById = _userPrincipal.Info.Id, ProjectId = command.ProjectId, Repository = command.Repository, SdlStatus = (int)SdlPolicyStatus.Unknown }; telemetryScope.SetEntity(task); _repositoryTasks.Insert(task); command.CreatedTaskId = task.Id; _eventProvider.Publish( new Event { Key = EventKeys.ScanTask.Created, Data = new Dictionary <string, string> { { Variables.ProjectId, command.ProjectId.ToString() }, { Variables.TaskId, task.Id.ToString() } } }); _repositoryTasks.Save(); telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }
protected override void UpdateProject(Projects project, UpdateProjectSettingsCommand command) { using (var telemetryScope = _telemetryScopeProvider.Create <Projects>(TelemetryOperationNames.Prroject.Update)) { try { telemetryScope.SetEntity(project); project.Alias = command.Alias; project.CommitToIt = command.CommitToIt; project.CommitToVcs = command.CommitToVcs; project.DefaultBranchName = command.DefaultBranchName; project.Description = command.Description; project.DisplayName = command.DisplayName; project.VcsSyncEnabled = command.VcsSyncEnabled; if (project.EnablePoll && !command.EnablePoll) { Global.JobScheduler.DeleteJob("PollJob.Project_{0}".FormatWith(project.Id)); } if (!project.EnablePoll && command.EnablePoll && command.PollTimeout.HasValue) { ScheduleJob(project.Id, command.PollTimeout.Value); } if (project.EnablePoll && command.EnablePoll && command.PollTimeout.HasValue && (project.PollTimeout != command.PollTimeout)) { Global.JobScheduler.DeleteJob("PollJob.Project_{0}".FormatWith(project.Id)); ScheduleJob(project.Id, command.PollTimeout.Value); } project.EnablePoll = command.EnablePoll; project.PollTimeout = command.PollTimeout; telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }
/// <summary> /// Executes the job. /// </summary> /// <returns> /// Positive value to repeat the task, negative value or 0 - to finish the task and to wait time interval before the /// next run. /// </returns> protected override int Process() { var task = GetNextTask(); if (task == null) { return(0); } using (var telemetryScope = _telemetryScopeProvider.Create <Tasks>(TelemetryOperationNames.Task.Update)) { try { telemetryScope.SetEntity(task); MakeStart(task); ProcessTask(task); MakeEnd(task); telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); try { MakeFail(task); } catch (Exception internalException) { telemetryScope.WriteException(internalException); } } } return(1); }
protected override void ProcessAuthorized(StopTaskCommand command) { using (var telemetryScope = _telemetryScopeProvider.Create <Tasks>(TelemetryOperationNames.Task.Stop)) { try { var task = _repositoryTasks.GetById(command.TaskId); telemetryScope.SetEntity(task); task.Modified = _timeService.GetUtc(); task.ModifiedById = _userPrincipal.Info.Id; task.SdlPolicyStatus = SdlPolicyStatus.Unknown; task.Cancel(); _eventProvider.Publish( new Event { Key = EventKeys.ScanTask.Finished, Data = new Dictionary <string, string> { { Variables.TaskId, task.Id.ToString() }, { Variables.ProjectId, task.ProjectId.ToString() } } }); _repositoryTasks.Save(); telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }
protected override void ExecuteStage(PostProcessingBundle bundle) { var fpAnnotation = bundle.IssueVulnerabilityLinks .Select(x => x.IssueAnnotation) .Concat(bundle.RemainingIssues) .Where(x => x.State == IssueAnnotationState.FalsePositive) .ToArray(); var fixedAnnotation = bundle.RemainingIssues .Concat(bundle.Changes.Where(x => x.Type == FileChangeAnnotationType.Update).Select(x => x.Annotation)) .Where(x => x.State == IssueAnnotationState.Fixed) .ToArray(); var todoVulnerabilities = bundle.Changes.Where(x => x.Annotation.State == IssueAnnotationState.Todo).ToArray(); var reopenVulnerabilities = bundle.Changes.Where(x => x.Annotation.State == IssueAnnotationState.Reopen).ToArray(); var groups = fpAnnotation.Select(x => new { Type = x.LongName, IssueId = x.Id }) .Concat( fixedAnnotation.Select( x => new { Type = x.LongName, IssueId = x.Id })) .Concat( todoVulnerabilities.Concat(reopenVulnerabilities) .Select(x => new { Type = x.Annotation.LongName, IssueId = x.Annotation.Id })) .Distinct() .ToArray(); foreach (var group in groups) { var branchName = _branchNameBuilder.Build(new BranchNameBuilderInfo { Type = group.Type }); var template = _templateProvider.Get(TemplateNames.IssueBody); var fp = fpAnnotation.Where(x => x.LongName == group.Type).GroupBy( x => new { x.File, x.LineStart }).Select(x => x.First()).ToArray(); var fixedItems = fixedAnnotation.Where(x => x.LongName == group.Type).GroupBy( x => new { x.File, x.LineStart }).Select(x => x.First()).ToArray(); var todoItems = todoVulnerabilities .Where(x => x.Annotation.LongName == group.Type) .GroupBy( x => new { x.Vulnerability.File, x.Vulnerability.NumberLine }).Select(x => x.First()).ToArray(); var reopenItems = reopenVulnerabilities .Where(x => x.Annotation.LongName == group.Type) .GroupBy( x => new { x.Vulnerability.File, x.Vulnerability.NumberLine }).Select(x => x.First()).ToArray(); template.Body.Add( new Dictionary <string, object> { { "FP", fp }, { "Fixed", fixedItems }, { "Todo", todoItems }, { "Reopen", reopenItems }, // TODO: remove code block cause incorrect read out strongly typed GitHub settings //{ // "RepoOwner", settings[GitHubItSettingKeys.RepositoryOwner.ToString()] //}, //{ // "RepoName", settings[GitHubItSettingKeys.RepositoryName.ToString()] //}, { "RepoBranch", branchName }, { "TotalCount", fp.Length + fixedItems.Length + todoItems.Length + reopenItems.Length } }); var body = template.Body.Render(); var issue = bundle.Issues.FirstOrDefault(x => x.Id == group.IssueId) ?? bundle.IssueTrackerPlugin.GetIssue(group.IssueId); var title = _issueNameBuilder.Build( new IssueNameBuilderInfo { IssueTypeName = group.Type }); if (fp.Any() || fixedItems.Any() || todoItems.Any() || reopenItems.Any()) { title += Resources.IssueNameLeft.FormatWith(todoItems.Length + reopenItems.Length); } using (var telemetryScope = _telemetryScopeProvider.Create <ItPluginInfo>(TelemetryOperationNames.ItPlugin.Update)) { try { telemetryScope.SetEntity( new ItPluginInfo { Plugins = bundle.Task.Projects.Plugins, TaskId = bundle.Task.Id }); bundle.IssueTrackerPlugin.UpdateIssue( new UpdateIssueRequest { Id = issue.Id, Title = title, Description = body, Status = todoVulnerabilities.Concat(reopenVulnerabilities).Any(x => x.Annotation.LongName == group.Type) ? IssueStatus.Open : IssueStatus.Closed }); telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } } }
protected override void ExecuteStage(PostProcessingBundle bundle) { var vulnerabilityTypeGroups = bundle.VulnerabilitiesInfo .Where(x => bundle.IssueVulnerabilityLinks.All(y => y.VulnerabilityInfo != x)) .GroupBy(x => x.Type); // new foreach (var vulnerabilityGroup in vulnerabilityTypeGroups) { var issueTitle = _issueNameBuilder.Build( new IssueNameBuilderInfo { IssueTypeName = vulnerabilityGroup.Key }); Issue itIssue = null; if (bundle.Task.Projects.CommitToIt) { itIssue = bundle.Issues.FirstOrDefault( x => x.Title.StartsWith( issueTitle, StringComparison.InvariantCultureIgnoreCase)); if (itIssue == null) { using (var telemetryScope = _telemetryScopeProvider.Create <ItPluginInfo>(TelemetryOperationNames.ItPlugin.Create)) { try { telemetryScope.SetEntity( new ItPluginInfo { Plugins = bundle.Task.Projects.Plugins, TaskId = bundle.Task.Id }); itIssue = bundle.IssueTrackerPlugin.CreateIssue( bundle.Task.Repository, new CreateIssueRequest { Title = issueTitle, Description = string.Empty }); telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } } } //bundle.Issues.Add(itIssue); TODO: wtf its here? foreach (var vulnerabilityInfo in vulnerabilityGroup) { vulnerabilityInfo.IssueNumber = itIssue?.Id; vulnerabilityInfo.IssueUrl = itIssue?.Url; var additionalContent = string.Empty; if (!string.IsNullOrWhiteSpace(vulnerabilityInfo.Exploit)) { additionalContent += "\n" + vulnerabilityInfo.Exploit; } if (!string.IsNullOrWhiteSpace(vulnerabilityInfo.AdditionalExploitConditions)) { additionalContent += "\n" + vulnerabilityInfo.AdditionalExploitConditions; } var issue = new IssueAnnotation { LineStart = vulnerabilityInfo.NumberLine, File = vulnerabilityInfo.File, LongName = vulnerabilityInfo.Type, Severity = "High", Id = itIssue?.Id, IssuePath = itIssue?.Url, AdditionalContent = additionalContent }; bundle.Changes.Add(new FileChangeAnnotation { Annotation = issue, Vulnerability = vulnerabilityInfo, Type = FileChangeAnnotationType.Add }); } } }
protected override void ExecuteStage(PostProcessingBundle bundle) { var branches = bundle.VersionControlPlugin.GetBranches().ToList(); var mainBranch = branches.First(x => x.Name == bundle.Task.Repository); var annotationGroups = bundle.Changes .Where(x => x.Type != FileChangeAnnotationType.None) .GroupBy(x => x.Annotation.LongName); foreach (var typeGroup in annotationGroups) { var branchName = _branchNameBuilder.Build(new BranchNameBuilderInfo { Type = typeGroup.Key }); var branch = branches.FirstOrDefault( x => string.Equals( x.Name, branchName, StringComparison.CurrentCultureIgnoreCase)); if (branch == null) { using ( var telemetryScope = _telemetryScopeProvider.Create <VcsPluginInfo>(TelemetryOperationNames.VcsPlugin.CreateBranch) ) { try { telemetryScope.SetEntity( new VcsPluginInfo { Plugin = bundle.Task.Projects.Plugins1, // VCS plugin =) CreatedBranchName = branchName, TaskId = bundle.Task.Id }); branch = bundle.VersionControlPlugin.CreateBranch(bundle.Task.FolderPath, branchName, mainBranch.Id); telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } branches.Add(branch); } foreach (var fileGroup in typeGroup.GroupBy(x => x.Annotation.File)) { var content = File.ReadAllText(Path.Combine(bundle.Task.FolderPath, fileGroup.Key)); var ending = content.GetLineSplitter(); var linesContent = content.Split(new[] { ending }, StringSplitOptions.None).ToList(); var rowShift = 0; foreach (var fileUpdate in fileGroup.OrderBy(x => x.Annotation.LineStart)) { var removeCount = 0; if (fileUpdate.Type == FileChangeAnnotationType.Update) { removeCount = fileUpdate.Annotation.LineEnd - fileUpdate.Annotation.LineStart + 1; for (var index = 0; index < removeCount; index++) { linesContent.RemoveAt(fileUpdate.Annotation.LineStart + rowShift - 1); } } var serializedIssueAnnotation = _issueAnnotationSerializer.Serialize(fileUpdate.Annotation); var annotation = _issueAnnotationFormatter.Format(serializedIssueAnnotation); var lines = annotation.Split('\n'); var rawLine = linesContent[fileUpdate.Annotation.LineStart + rowShift - 1]; var startLineShift = rawLine.Substring(0, rawLine.Length - rawLine.TrimStart(' ', '\t').Length); for (var index = 0; index < lines.Length; index++) { linesContent.Insert( fileUpdate.Annotation.LineStart + rowShift + index - 1, startLineShift + lines[index].Trim()); } rowShift = rowShift - removeCount + lines.Length; if (fileUpdate.Vulnerability != null) { fileUpdate.Vulnerability.NumberLine = fileUpdate.Annotation.LineStart + rowShift; } } var memoryStream = new MemoryStream(); var streamWriter = new StreamWriter(memoryStream); streamWriter.Write(string.Join(ending, linesContent)); streamWriter.Flush(); streamWriter.Close(); var commitTemplate = _templateProvider.Get(TemplateNames.CommitName); commitTemplate.Body.Add( new Dictionary <string, object> { { "Group", _vulnerabilityShortTypeResolver.Resolve(typeGroup.Key) }, { "File", Path.GetFileName(fileGroup.Key) } }); var bodyCommit = commitTemplate.Body.Render(); _log.Debug($"Commit to {branch.Id} {fileGroup.Key}"); var fileContent = memoryStream.ToArray(); using (var telemetryScope = _telemetryScopeProvider.Create <VcsPluginInfo>(TelemetryOperationNames.VcsPlugin.Commit)) { try { telemetryScope.SetEntity( new VcsPluginInfo { Plugin = bundle.Task.Projects.Plugins1, // VCS plugin CommittedSourcesSize = fileContent.Length, TaskId = bundle.Task.Id }); bundle.VersionControlPlugin.Commit( bundle.Task.FolderPath, branch.Id, bodyCommit, fileGroup.Key, fileContent); telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } } } bundle.VersionControlPlugin.CleanUp(bundle.Task.FolderPath); }
/// <summary> /// Handles the specified event to handle. /// </summary> /// <param name="eventToHandle">The event to handle.</param> public void Handle(Event eventToHandle) { PublishPolicyEvent(eventToHandle, EventKeys.Policy.CheckStarted); Tasks task = null; using (var telemetryScope = _telemetryScopeProvider.Create <Tasks>(TelemetryOperationNames.Task.Update)) { try { var data = eventToHandle.Data; if ((data == null) || !data.ContainsKey(Variables.TaskId)) { PublishPolicyEvent(eventToHandle, EventKeys.Policy.CheckFinished); return; } var taskId = long.Parse(data[Variables.TaskId]); task = _taskRepository.GetById(taskId); telemetryScope.SetEntity(task); var projectId = long.Parse(data[Variables.ProjectId]); var rules = _policyRuleRepository.Get(projectId).ToArray(); var policyRules = rules.Select(_ => _ruleParser.ParsePolicyRule(_)).ToArray(); var success = true; var violatedRules = new List <IPolicyRule>(); // ReSharper disable once LoopCanBePartlyConvertedToQuery foreach (var policyRule in policyRules) { var ruleResult = _ruleExecutorDirector.Execute <IPolicyRule, PolicyRuleResult>(policyRule, data); if (ruleResult.Success) { continue; } success = false; violatedRules.Add(policyRule); } if (!success) { task.ViolatePolicy( Resources.PolicyViolated .FormatWith(violatedRules.Select(_ => _.Rule.Name).ToCommaSeparatedString())); } else { task.SuccessPolicy(); } _taskRepository.Save(); PublishPolicyEvent(eventToHandle, EventKeys.Policy.CheckFinished); PublishPolicyEvent(eventToHandle, success ? EventKeys.Policy.Successful : EventKeys.Policy.Violation); telemetryScope.WriteSuccess(); } catch (Exception exception) { _logger.Error(Resources.PolicyCheckingError, exception); try { task?.ErrorPolicy(Resources.PolicyError.FormatWith(exception.Message, exception.Format())); _taskRepository.Save(); } catch (Exception innerException) { _logger.Error(Resources.PolicyCheckingError, innerException); } PublishPolicyEvent(eventToHandle, EventKeys.Policy.CheckFinished); PublishPolicyEvent(eventToHandle, EventKeys.Policy.Error); telemetryScope.WriteException(exception); } } }
/// <summary> /// Processes the task. /// </summary> /// <param name="task">The task.</param> protected override void ProcessTask([NotNull] Tasks task) { var project = _projectRepository.GetById(task.ProjectId); if (project.VcsPluginId == null) { throw new Exception("Failed to preprocess task. Vcs plugin is eq NULL."); } var plugin = _backendPluginProvider.GetPlugin <IVersionControlPlugin>( project.VcsPluginId.Value, task.CreatedById, task.ProjectId); if (plugin == null) { throw new Exception(Resources.VcsPluginNotFound); } var folderPath = _configurationProvider.GetValue(ConfigurationKeys.AppSettings.TempDirPath); var repoFolder = Path.Combine(task.Repository, task.Id.ToString()); var path = Path.Combine(folderPath, GetSafeRepoName(repoFolder)); if (Directory.Exists(path)) { Directory.Delete(path, true); } Directory.CreateDirectory(path); using (var telemetryScope = _telemetryScopeProvider.Create <VcsPluginInfo>( TelemetryOperationNames.VcsPlugin.Checkout)) { try { var vcsPlugin = new VcsPluginInfo { Plugin = project.Plugins1, TaskId = task.Id }; telemetryScope.SetEntity(vcsPlugin); plugin.GetSources(task.Repository, path); var folderSize = _fileSystemInfoProvider.CalculateDirectorySize(path); vcsPlugin.DownloadedSourcesSize = folderSize; task.FolderPath = path; task.FolderSize = folderSize; telemetryScope.WriteSuccess(); } catch (Exception ex) { telemetryScope.WriteException(ex); throw; } } }