private IssueAnnotation[] ProcessRemainingIssues(PostProcessingBundle bundle)
{
var remainingIssues = bundle.IssueAnnotations
.Where(x => bundle.IssueVulnerabilityLinks.All(y => y.IssueAnnotation != x)).ToList();
// fixed
// ReSharper disable once LoopCanBePartlyConvertedToQuery
foreach (var issueAnnotation in remainingIssues.ToArray())
{
if ((issueAnnotation.State == IssueAnnotationState.Fixed) ||
(issueAnnotation.State == IssueAnnotationState.FalsePositive))
{
continue;
}
remainingIssues.Remove(issueAnnotation);
_issueAnnotationWorkflow.ChangeState(issueAnnotation, IssueAnnotationState.Fixed);
bundle.Changes.Add(new FileChangeAnnotation
{
Annotation = issueAnnotation,
Type = FileChangeAnnotationType.Update
});
}
return(remainingIssues.ToArray());
}
protected override void ExecuteStage(PostProcessingBundle bundle) =>
bundle.VulnerabilitiesInfo = bundle.VulnerabilitiesInfo
.GroupBy(x => new { x.Place, x.Type })
.Select(
x =>
x.Any(y => !string.IsNullOrWhiteSpace(y.Exploit))
? x.FirstOrDefault(y => !string.IsNullOrWhiteSpace(y.Exploit))
: x.First())
.Select(x =>
{
var splittedPlace = x.Place.Split(':');
string file;
if (splittedPlace.Length > 2)
{
file = string.Join(":", splittedPlace.Take(splittedPlace.Length - 3));
x.Position = int.Parse(splittedPlace[splittedPlace.Length - 2]);
}
else
{
file = x.Place;
}
file = file.Substring(bundle.Task.FolderPath.Length).Trim('\\');
x.File = x.SourceFile = file;
return(x);
}).ToArray();
protected override void ExecuteStage(PostProcessingBundle bundle)
{
if (bundle.Task.Projects.VcsPluginId == null)
{
throw new Exception();
}
bundle.VersionControlPlugin = _backendPluginProvider.GetPlugin <IVersionControlPlugin>(
bundle.Task.Projects.VcsPluginId.Value,
bundle.Task.CreatedById,
bundle.Task.ProjectId);
}
protected override void ExecuteStage(PostProcessingBundle bundle)
{
if (bundle.Task.Projects.ItPluginId == null)
{
throw new Exception("Project IT plugin is not initialized.");
}
bundle.IssueTrackerPlugin = _backendPluginProvider.GetPlugin <IIssueTrackerPlugin>(
bundle.Task.Projects.ItPluginId.Value,
bundle.Task.CreatedById,
bundle.Task.ProjectId);
}
public void Execute([NotNull] PostProcessingBundle bundle)
{
if (bundle == null)
{
throw new ArgumentNullException(nameof(bundle));
}
if (PreCondition.Invoke(bundle.Task))
{
ExecuteStage(bundle);
}
ExecuteSuccessor(bundle);
}
protected override void ExecuteStage(PostProcessingBundle bundle)
{
if (!_branchNameBuilder.IsOurBranch(bundle.Task.Repository))
{
return;
}
var branchVulnerabilityTypeName = _branchNameBuilder.GetInfo(bundle.Task.Repository);
bundle.VulnerabilitiesInfo = bundle.VulnerabilitiesInfo
.Where(x => x.Type.ToLower() == branchVulnerabilityTypeName.Type)
.ToArray();
bundle.IssueAnnotations = bundle.IssueAnnotations
.Where(x => x.LongName.ToLower() == branchVulnerabilityTypeName.Type)
.ToArray();
}
protected override void ExecuteStage([NotNull] PostProcessingBundle bundle) => bundle.VulnerabilitiesInfo = _vulnerabilityInfoProvider.GetByTask(bundle.Task);
protected override void ExecuteStage(PostProcessingBundle bundle)
{
var fpAnnotation =
bundle.IssueVulnerabilityLinks
.Select(x => x.IssueAnnotation)
.Concat(bundle.RemainingIssues)
.Where(x => x.State == IssueAnnotationState.FalsePositive)
.ToArray();
var fixedAnnotation = bundle.RemainingIssues
.Concat(bundle.Changes.Where(x => x.Type == FileChangeAnnotationType.Update).Select(x => x.Annotation))
.Where(x => x.State == IssueAnnotationState.Fixed)
.ToArray();
var todoVulnerabilities = bundle.Changes.Where(x => x.Annotation.State == IssueAnnotationState.Todo).ToArray();
var reopenVulnerabilities = bundle.Changes.Where(x => x.Annotation.State == IssueAnnotationState.Reopen).ToArray();
var groups =
fpAnnotation.Select(x => new { Type = x.LongName, IssueId = x.Id })
.Concat(
fixedAnnotation.Select(
x => new
{
Type = x.LongName,
IssueId = x.Id
}))
.Concat(
todoVulnerabilities.Concat(reopenVulnerabilities)
.Select(x => new { Type = x.Annotation.LongName, IssueId = x.Annotation.Id }))
.Distinct()
.ToArray();
foreach (var group in groups)
{
var branchName = _branchNameBuilder.Build(new BranchNameBuilderInfo {
Type = group.Type
});
var template = _templateProvider.Get(TemplateNames.IssueBody);
var fp = fpAnnotation.Where(x => x.LongName == group.Type).GroupBy(
x => new
{
x.File,
x.LineStart
}).Select(x => x.First()).ToArray();
var fixedItems = fixedAnnotation.Where(x => x.LongName == group.Type).GroupBy(
x => new
{
x.File,
x.LineStart
}).Select(x => x.First()).ToArray();
var todoItems = todoVulnerabilities
.Where(x => x.Annotation.LongName == group.Type)
.GroupBy(
x => new
{
x.Vulnerability.File,
x.Vulnerability.NumberLine
}).Select(x => x.First()).ToArray();
var reopenItems = reopenVulnerabilities
.Where(x => x.Annotation.LongName == group.Type)
.GroupBy(
x => new
{
x.Vulnerability.File,
x.Vulnerability.NumberLine
}).Select(x => x.First()).ToArray();
template.Body.Add(
new Dictionary <string, object>
{
{
"FP", fp
},
{
"Fixed", fixedItems
},
{
"Todo", todoItems
},
{
"Reopen", reopenItems
},
// TODO: remove code block cause incorrect read out strongly typed GitHub settings
//{
// "RepoOwner", settings[GitHubItSettingKeys.RepositoryOwner.ToString()]
//},
//{
// "RepoName", settings[GitHubItSettingKeys.RepositoryName.ToString()]
//},
{
"RepoBranch", branchName
},
{
"TotalCount", fp.Length + fixedItems.Length + todoItems.Length + reopenItems.Length
}
});
var body = template.Body.Render();
var issue = bundle.Issues.FirstOrDefault(x => x.Id == group.IssueId) ??
bundle.IssueTrackerPlugin.GetIssue(group.IssueId);
var title = _issueNameBuilder.Build(
new IssueNameBuilderInfo
{
IssueTypeName = group.Type
});
if (fp.Any() || fixedItems.Any() || todoItems.Any() || reopenItems.Any())
{
title += Resources.IssueNameLeft.FormatWith(todoItems.Length + reopenItems.Length);
}
using (var telemetryScope = _telemetryScopeProvider.Create <ItPluginInfo>(TelemetryOperationNames.ItPlugin.Update))
{
try
{
telemetryScope.SetEntity(
new ItPluginInfo
{
Plugins = bundle.Task.Projects.Plugins,
TaskId = bundle.Task.Id
});
bundle.IssueTrackerPlugin.UpdateIssue(
new UpdateIssueRequest
{
Id = issue.Id,
Title = title,
Description = body,
Status =
todoVulnerabilities.Concat(reopenVulnerabilities).Any(x => x.Annotation.LongName == group.Type)
? IssueStatus.Open
: IssueStatus.Closed
});
telemetryScope.WriteSuccess();
}
catch (Exception ex)
{
telemetryScope.WriteException(ex);
throw;
}
}
}
}
protected override void ExecuteStage(PostProcessingBundle bundle)
{
var vulnerabilityTypeGroups = bundle.VulnerabilitiesInfo
.Where(x => bundle.IssueVulnerabilityLinks.All(y => y.VulnerabilityInfo != x))
.GroupBy(x => x.Type);
// new
foreach (var vulnerabilityGroup in vulnerabilityTypeGroups)
{
var issueTitle = _issueNameBuilder.Build(
new IssueNameBuilderInfo
{
IssueTypeName = vulnerabilityGroup.Key
});
Issue itIssue = null;
if (bundle.Task.Projects.CommitToIt)
{
itIssue = bundle.Issues.FirstOrDefault(
x =>
x.Title.StartsWith(
issueTitle,
StringComparison.InvariantCultureIgnoreCase));
if (itIssue == null)
{
using (var telemetryScope = _telemetryScopeProvider.Create <ItPluginInfo>(TelemetryOperationNames.ItPlugin.Create))
{
try
{
telemetryScope.SetEntity(
new ItPluginInfo
{
Plugins = bundle.Task.Projects.Plugins,
TaskId = bundle.Task.Id
});
itIssue = bundle.IssueTrackerPlugin.CreateIssue(
bundle.Task.Repository,
new CreateIssueRequest
{
Title = issueTitle,
Description = string.Empty
});
telemetryScope.WriteSuccess();
}
catch (Exception ex)
{
telemetryScope.WriteException(ex);
throw;
}
}
}
}
//bundle.Issues.Add(itIssue); TODO: wtf its here?
foreach (var vulnerabilityInfo in vulnerabilityGroup)
{
vulnerabilityInfo.IssueNumber = itIssue?.Id;
vulnerabilityInfo.IssueUrl = itIssue?.Url;
var additionalContent = string.Empty;
if (!string.IsNullOrWhiteSpace(vulnerabilityInfo.Exploit))
{
additionalContent += "\n" + vulnerabilityInfo.Exploit;
}
if (!string.IsNullOrWhiteSpace(vulnerabilityInfo.AdditionalExploitConditions))
{
additionalContent += "\n" + vulnerabilityInfo.AdditionalExploitConditions;
}
var issue = new IssueAnnotation
{
LineStart = vulnerabilityInfo.NumberLine,
File = vulnerabilityInfo.File,
LongName = vulnerabilityInfo.Type,
Severity = "High",
Id = itIssue?.Id,
IssuePath = itIssue?.Url,
AdditionalContent = additionalContent
};
bundle.Changes.Add(new FileChangeAnnotation
{
Annotation = issue,
Vulnerability = vulnerabilityInfo,
Type = FileChangeAnnotationType.Add
});
}
}
}
protected override void ExecuteStage(PostProcessingBundle bundle) => bundle.Issues = bundle.IssueTrackerPlugin.GetIssues().ToList();
protected override void ExecuteStage(PostProcessingBundle bundle)
{
var branches = bundle.VersionControlPlugin.GetBranches().ToList();
var mainBranch = branches.First(x => x.Name == bundle.Task.Repository);
var annotationGroups = bundle.Changes
.Where(x => x.Type != FileChangeAnnotationType.None)
.GroupBy(x => x.Annotation.LongName);
foreach (var typeGroup in annotationGroups)
{
var branchName = _branchNameBuilder.Build(new BranchNameBuilderInfo {
Type = typeGroup.Key
});
var branch = branches.FirstOrDefault(
x => string.Equals(
x.Name,
branchName,
StringComparison.CurrentCultureIgnoreCase));
if (branch == null)
{
using (
var telemetryScope = _telemetryScopeProvider.Create <VcsPluginInfo>(TelemetryOperationNames.VcsPlugin.CreateBranch)
)
{
try
{
telemetryScope.SetEntity(
new VcsPluginInfo
{
Plugin = bundle.Task.Projects.Plugins1, // VCS plugin =)
CreatedBranchName = branchName,
TaskId = bundle.Task.Id
});
branch = bundle.VersionControlPlugin.CreateBranch(bundle.Task.FolderPath, branchName, mainBranch.Id);
telemetryScope.WriteSuccess();
}
catch (Exception ex)
{
telemetryScope.WriteException(ex);
throw;
}
}
branches.Add(branch);
}
foreach (var fileGroup in typeGroup.GroupBy(x => x.Annotation.File))
{
var content = File.ReadAllText(Path.Combine(bundle.Task.FolderPath, fileGroup.Key));
var ending = content.GetLineSplitter();
var linesContent = content.Split(new[] { ending }, StringSplitOptions.None).ToList();
var rowShift = 0;
foreach (var fileUpdate in fileGroup.OrderBy(x => x.Annotation.LineStart))
{
var removeCount = 0;
if (fileUpdate.Type == FileChangeAnnotationType.Update)
{
removeCount = fileUpdate.Annotation.LineEnd - fileUpdate.Annotation.LineStart + 1;
for (var index = 0; index < removeCount; index++)
{
linesContent.RemoveAt(fileUpdate.Annotation.LineStart + rowShift - 1);
}
}
var serializedIssueAnnotation = _issueAnnotationSerializer.Serialize(fileUpdate.Annotation);
var annotation = _issueAnnotationFormatter.Format(serializedIssueAnnotation);
var lines = annotation.Split('\n');
var rawLine = linesContent[fileUpdate.Annotation.LineStart + rowShift - 1];
var startLineShift = rawLine.Substring(0, rawLine.Length - rawLine.TrimStart(' ', '\t').Length);
for (var index = 0; index < lines.Length; index++)
{
linesContent.Insert(
fileUpdate.Annotation.LineStart + rowShift + index - 1,
startLineShift + lines[index].Trim());
}
rowShift = rowShift - removeCount + lines.Length;
if (fileUpdate.Vulnerability != null)
{
fileUpdate.Vulnerability.NumberLine = fileUpdate.Annotation.LineStart + rowShift;
}
}
var memoryStream = new MemoryStream();
var streamWriter = new StreamWriter(memoryStream);
streamWriter.Write(string.Join(ending, linesContent));
streamWriter.Flush();
streamWriter.Close();
var commitTemplate = _templateProvider.Get(TemplateNames.CommitName);
commitTemplate.Body.Add(
new Dictionary <string, object>
{
{ "Group", _vulnerabilityShortTypeResolver.Resolve(typeGroup.Key) },
{ "File", Path.GetFileName(fileGroup.Key) }
});
var bodyCommit = commitTemplate.Body.Render();
_log.Debug($"Commit to {branch.Id} {fileGroup.Key}");
var fileContent = memoryStream.ToArray();
using (var telemetryScope = _telemetryScopeProvider.Create <VcsPluginInfo>(TelemetryOperationNames.VcsPlugin.Commit))
{
try
{
telemetryScope.SetEntity(
new VcsPluginInfo
{
Plugin = bundle.Task.Projects.Plugins1, // VCS plugin
CommittedSourcesSize = fileContent.Length,
TaskId = bundle.Task.Id
});
bundle.VersionControlPlugin.Commit(
bundle.Task.FolderPath,
branch.Id,
bodyCommit,
fileGroup.Key,
fileContent);
telemetryScope.WriteSuccess();
}
catch (Exception ex)
{
telemetryScope.WriteException(ex);
throw;
}
}
}
}
bundle.VersionControlPlugin.CleanUp(bundle.Task.FolderPath);
}
protected override void ExecuteStage(PostProcessingBundle bundle) => bundle.IssueAnnotations = GetIssuesByTask(bundle.Task);
protected override void ExecuteStage(PostProcessingBundle bundle)
{
var pairs = new List <IssueVulnerabilityLink>();
var vulnGroups = bundle.VulnerabilitiesInfo
.GroupBy(x => x.Type)
.ToDictionary(x => x.Key, x => x.ToArray());
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var issueAnnotation in bundle.IssueAnnotations)
{
if (!vulnGroups.ContainsKey(issueAnnotation.LongName))
{
continue;
}
var vulns = vulnGroups[issueAnnotation.LongName];
var vulnerability =
vulns
.Where(x => x.File == issueAnnotation.File)
.FirstOrDefault(x => issueAnnotation.LineEnd + 1 == x.NumberLine);
if (vulnerability == null)
{
continue;
}
vulnerability.IssueNumber = issueAnnotation.Id;
vulnerability.IssueUrl = issueAnnotation.IssuePath;
pairs.Add(new IssueVulnerabilityLink(issueAnnotation, vulnerability));
}
// reopen or fp
// ReSharper disable once LoopCanBePartlyConvertedToQuery
foreach (var pair in pairs)
{
// ignoring false positives
if (pair.IssueAnnotation.State == IssueAnnotationState.FalsePositive)
{
continue;
}
// Don't process not finished issues
if ((pair.IssueAnnotation.State != IssueAnnotationState.Verify) &&
(pair.IssueAnnotation.State != IssueAnnotationState.Fixed))
{
bundle.Changes.Add(new FileChangeAnnotation
{
Annotation = pair.IssueAnnotation,
Vulnerability = pair.VulnerabilityInfo,
Type =
pair.IssueAnnotation.State == IssueAnnotationState.Todo
? FileChangeAnnotationType.None
: FileChangeAnnotationType.Update
});
continue;
}
_issueAnnotationWorkflow.ChangeState(pair.IssueAnnotation, IssueAnnotationState.Reopen);
bundle.Changes.Add(new FileChangeAnnotation
{
Annotation = pair.IssueAnnotation,
Vulnerability = pair.VulnerabilityInfo,
Type = FileChangeAnnotationType.Update
});
}
bundle.IssueVulnerabilityLinks = pairs.ToArray();
}
protected override void ExecuteStage(PostProcessingBundle bundle) => bundle.RemainingIssues = ProcessRemainingIssues(bundle);
private void ExecuteSuccessor(PostProcessingBundle bundle) => _successor?.Execute(bundle);
protected abstract void ExecuteStage(PostProcessingBundle bundle);