public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
// Make sure we're running a step that this applies to
if (JobStep.Name.IndexOf("Copyright", StringComparison.OrdinalIgnoreCase) == -1)
{
return(false);
}
// Find any files in compiler output format
HashSet <string> SourceFileNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (Match FileMatch in Regex.Matches(Diagnostic.Message, @"^\s*(?:WARNING|ERROR):\s*([^ ]+\.[a-zA-Z]+):", RegexOptions.Multiline))
{
if (FileMatch.Success)
{
SourceFileNames.Add(FileMatch.Groups[1].Value);
}
}
// If we found any source files, create a diagnostic category for them
if (SourceFileNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, Diagnostic.Message, Diagnostic.Url));
Issue.FileNames.UnionWith(SourceFileNames);
Issues.Add(Issue);
return(true);
}
// Otherwise pass
return(false);
}
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
// Find any files in compiler output format
HashSet <string> SourceFileNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (Match FileMatch in Regex.Matches(Diagnostic.Message, @"^\s*(?:In file included from\s*)?((?:[A-Za-z]:)?[^\s(:]+)[\(:]\d", RegexOptions.Multiline))
{
if (FileMatch.Success)
{
string FileName = GetNormalizedFileName(FileMatch.Groups[1].Value, JobStep.BaseDirectory);
if (SourceFileExtensions.Any(x => FileName.EndsWith(x, StringComparison.OrdinalIgnoreCase)))
{
SourceFileNames.Add(FileName);
}
}
}
// If we found any source files, create a diagnostic category for them
if (SourceFileNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, ShortenPaths(Diagnostic.Message), Diagnostic.Url));
Issue.FileNames.UnionWith(SourceFileNames);
Issues.Add(Issue);
return(true);
}
// Otherwise pass
return(false);
}
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
string DefaultProject = String.Format("{0} (Unmatched)", Job.Project);
BuildHealthIssue Issue = new BuildHealthIssue(DefaultProject, Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, JobStep.Url, Diagnostic.Message, Diagnostic.Url));
Issue.Identifiers.Add(Diagnostic.Message);
Issues.Add(Issue);
return(true);
}
/// <summary>
/// Creates fingerprints from any matching diagnostics
/// </summary>
/// <param name="Job">The job that was run</param>
/// <param name="JobStep">The job step that was run</param>
/// <param name="Diagnostics">List of diagnostics that were produced by the build. Items should be removed from this list if they match.</param>
/// <param name="Issues">List which receives all the matched issues.</param>
public virtual void Match(InputJob Job, InputJobStep JobStep, List <InputDiagnostic> Diagnostics, List <BuildHealthIssue> Issues)
{
for (int Idx = 0; Idx < Diagnostics.Count; Idx++)
{
InputDiagnostic Diagnostic = Diagnostics[Idx];
if (TryMatch(Job, JobStep, Diagnostic, Issues))
{
Diagnostics.RemoveAt(Idx);
Idx--;
}
}
}
/// <summary>
/// Adds diagnostics from a job step into the issue database
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
void AddStep(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep)
{
// Create a lazily evaluated list of changes that are responsible for any errors
Lazy <IReadOnlyList <ChangeInfo> > LazyChanges = new Lazy <IReadOnlyList <ChangeInfo> >(() => FindChanges(Perforce, State, InputJob));
// Create issues for any diagnostics in this step
List <BuildHealthIssue> InputIssues = new List <BuildHealthIssue>();
foreach (PatternMatcher Matcher in Matchers)
{
Matcher.Match(InputJob, InputJobStep, InputJobStep.Diagnostics, InputIssues);
}
// Merge the issues together
List <BuildHealthIssue> NewIssues = new List <BuildHealthIssue>();
foreach (BuildHealthIssue InputIssue in InputIssues)
{
BuildHealthIssue OutputIssue = MergeIntoExistingIssue(Perforce, State, InputJob, InputJobStep, InputIssue, LazyChanges);
if (OutputIssue == null)
{
NewIssues.Add(InputIssue);
State.Issues.Add(InputIssue);
OutputIssue = InputIssue;
}
AddFailureToIssue(OutputIssue, InputJob, InputJobStep, InputIssue.Diagnostics[0].ErrorUrl, State);
}
// Update the watchers for any new issues
foreach (BuildHealthIssue NewIssue in NewIssues)
{
IReadOnlyList <ChangeInfo> Changes = LazyChanges.Value;
if (Changes != null)
{
// Find the pattern matcher for this issue
PatternMatcher Matcher = CategoryToMatcher[NewIssue.Category];
// Update the causers
List <ChangeInfo> Causers = Matcher.FindCausers(Perforce, NewIssue, Changes);
foreach (ChangeInfo Causer in Causers)
{
NewIssue.SourceChanges.UnionWith(Causer.SourceChanges);
NewIssue.PendingWatchers.Add(Causer.Record.User);
}
}
}
}
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
HashSet <string> FileNames = new HashSet <string>();
foreach (Match Match in Regex.Matches(Diagnostic.Message, @"^\s*Log[a-zA-Z0-9]+:\s+(?:Error:|Warning:)\s+((?:[a-zA-Z]:)?[^:]+(?:.uasset|.umap)):\s*(.*)"))
{
FileNames.Add(GetNormalizedFileName(Match.Groups[1].Value, JobStep.BaseDirectory));
}
if (FileNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Job.Project, Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, JobStep.Url, Diagnostic.Message, Diagnostic.Url));
Issue.FileNames.UnionWith(FileNames);
Issues.Add(Issue);
return(true);
}
return(false);
}
/// <summary>
/// Adds diagnostics from a job step into the issue database
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
void AddStep(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep)
{
// Create all the fingerprints for failures in this step
List <BuildHealthIssue> InputIssues = new List <BuildHealthIssue>();
if (InputJobStep.Diagnostics != null)
{
List <InputDiagnostic> Diagnostics = new List <InputDiagnostic>(InputJobStep.Diagnostics);
foreach (PatternMatcher PatternMatcher in Matchers)
{
PatternMatcher.Match(InputJob, InputJobStep, Diagnostics, InputIssues);
}
}
// Add all the fingerprints to issues
foreach (BuildHealthIssue InputIssue in InputIssues)
{
BuildHealthIssue Issue = FindOrAddIssueForFingerprint(Perforce, State, InputJob, InputJobStep, InputIssue);
AddFailureToIssue(Issue, InputJob, InputJobStep, InputIssue.Diagnostics[0].ErrorUrl, State);
}
}
/// <summary>
/// Adds a new build history for a stream
/// </summary>
/// <param name="Issue">The issue to add a build to</param>
/// <param name="InputJob">The job containing the error</param>
/// <param name="InputJobStep">The job step containing the error</param>
/// <param name="InputErrorUrl">Url of the error</param>
/// <param name="State">Current persistent state. Used to find previous build history.</param>
void AddFailureToIssue(BuildHealthIssue Issue, InputJob InputJob, InputJobStep InputJobStep, string InputErrorUrl, BuildHealthState State)
{
// Find or add a step name to history mapping
Dictionary <string, BuildHealthJobHistory> StepNameToHistory;
if (!Issue.Streams.TryGetValue(InputJob.Stream, out StepNameToHistory))
{
StepNameToHistory = new Dictionary <string, BuildHealthJobHistory>();
Issue.Streams.Add(InputJob.Stream, StepNameToHistory);
}
// Find or add a history for this step
BuildHealthJobHistory History;
if (!StepNameToHistory.TryGetValue(InputJobStep.Name, out History))
{
History = new BuildHealthJobHistory(State.FindBuildBefore(InputJob.Stream, InputJob.Change, InputJobStep.Name));
StepNameToHistory.Add(InputJobStep.Name, History);
}
// Add the new build
History.AddFailedBuild(CreateBuildForJobStep(InputJob, InputJobStep, InputErrorUrl));
}
/// <summary>
/// Find all changes PLUS all robomerge source changes
/// </summary>
/// <param name="Perforce">The Perforce connection to use</param>
/// <param name="State">State of </param>
/// <param name="InputJob">The job that failed</param>
/// <returns>Set of changelist numbers</returns>
IReadOnlyList <ChangeInfo> FindChanges(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob)
{
// List of changes since the last successful build in this stream
IReadOnlyList <ChangeInfo> Changes = null;
// Find the previous changelist that was built in this stream
List <BuildHealthJobStep> StreamBuilds;
if (State.Streams.TryGetValue(InputJob.Stream, out StreamBuilds))
{
// Find the last change submitted to this stream before it started failing
int LastChange = -1;
for (int Idx = 0; Idx < StreamBuilds.Count && StreamBuilds[Idx].Change < InputJob.Change; Idx++)
{
LastChange = StreamBuilds[Idx].Change;
}
// Allow adding to any open issue that contains changes merged from other branches
if (LastChange != -1)
{
// Query for all the changes since then
Changes = FindChanges(Perforce, InputJob.Stream, LastChange, InputJob.Change);
}
}
return(Changes);
}
/// <summary>
/// Creates a TrackedBuild instance for the given jobstep
/// </summary>
/// <param name="InputJob">The job to create a build for</param>
/// <param name="InputJobStep">The step to create a build for</param>
/// <param name="InputErrorUrl">The error Url</param>
/// <returns>New build instance</returns>
BuildHealthJobStep CreateBuildForJobStep(InputJob InputJob, InputJobStep InputJobStep, string InputErrorUrl)
{
return(new BuildHealthJobStep(InputJob.Change, InputJob.Name, InputJob.Url, InputJobStep.Name, InputJobStep.Url, InputErrorUrl));
}
/// <summary>
/// Finds or adds an issue for a particular issue
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
BuildHealthIssue MergeIntoExistingIssue(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep, BuildHealthIssue InputIssue, Lazy <IReadOnlyList <ChangeInfo> > LazyChanges)
{
// Find the pattern matcher for this fingerprint
PatternMatcher Matcher = CategoryToMatcher[InputIssue.Category];
// Check if it can be added to an existing open issue
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check this issue already exists in the current stream
Dictionary <string, BuildHealthJobHistory> StepNameToHistory;
if (!Issue.Streams.TryGetValue(InputJob.Stream, out StepNameToHistory))
{
continue;
}
// Check that this issue has not already been closed
BuildHealthJobHistory History;
if (StepNameToHistory.TryGetValue(InputJobStep.Name, out History))
{
if (!History.CanAddFailedBuild(InputJob.Change))
{
continue;
}
}
else
{
if (!StepNameToHistory.Values.Any(x => x.CanAddFailedBuild(InputJob.Change)))
{
continue;
}
}
// Try to merge the fingerprint
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Add the new build
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
// Check if this issue can be merged with an issue built in another stream
IReadOnlyList <ChangeInfo> Changes = LazyChanges.Value;
if (Changes != null && Changes.Count > 0)
{
SortedSet <int> SourceChanges = new SortedSet <int>(Changes.SelectMany(x => x.SourceChanges));
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check if this issue does not already contain this stream, but contains one of the causing changes
if (Issue.Streams.ContainsKey(InputJob.Stream))
{
continue;
}
if (!SourceChanges.Any(x => Issue.SourceChanges.Contains(x)))
{
continue;
}
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Merge the issue
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
}
// Check if it can be merged into an issue that's been created for this job. We only do this after exhausting all other options.
foreach (BuildHealthIssue Issue in State.Issues)
{
if (Issue.InitialJobUrl == InputIssue.InitialJobUrl && Matcher.CanMergeInitialJob(InputIssue, Issue))
{
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
}
return(null);
}
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
List <string> SymbolMatches = new List <string>();
// Mac link error:
// Undefined symbols for architecture arm64:
// "Foo::Bar() const", referenced from:
if (Regex.IsMatch(Diagnostic.Message, "^Undefined symbols"))
{
foreach (string Line in Diagnostic.Message.Split('\n'))
{
Match SymbolMatch = Regex.Match(Line, "^ \"(.+)\"");
if (SymbolMatch.Success)
{
SymbolMatches.Add(SymbolMatch.Groups[1].Value);
}
}
}
// Android link error:
// Foo.o:(.data.rel.ro + 0x5d88): undefined reference to `Foo::Bar()'
Match UndefinedReference = Regex.Match(Diagnostic.Message, ": undefined reference to [`']([^`']+)");
if (UndefinedReference.Success)
{
SymbolMatches.Add(UndefinedReference.Groups[1].Value);
}
// LLD link error:
// ld.lld.exe: error: undefined symbol: Foo::Bar() const
Match LldMatch = Regex.Match(Diagnostic.Message, "error: undefined symbol:\\s*(.+)");
if (LldMatch.Success)
{
SymbolMatches.Add(LldMatch.Groups[1].Value);
}
// Link error:
// Link: error: L0039: reference to undefined symbol `Foo::Bar() const' in file
Match LinkMatch = Regex.Match(Diagnostic.Message, ": reference to undefined symbol [`']([^`']+)");
if (LinkMatch.Success)
{
SymbolMatches.Add(LinkMatch.Groups[1].Value);
}
// Microsoft linker error:
// Foo.cpp.obj : error LNK2001: unresolved external symbol \"private: virtual void __cdecl UAssetManager::InitializeAssetBundlesFromMetadata_Recursive(class UStruct const *,void const *,struct FAssetBundleData &,class FName,class TSet<void const *,struct DefaultKeyFuncs<void const *,0>,class FDefaultSetAllocator> &)const \" (?InitializeAssetBundlesFromMetadata_Recursive@UAssetManager@@EEBAXPEBVUStruct@@PEBXAEAUFAssetBundleData@@VFName@@AEAV?$TSet@PEBXU?$DefaultKeyFuncs@PEBX$0A@@@VFDefaultSetAllocator@@@@@Z)",
Match MicrosoftMatch = Regex.Match(Diagnostic.Message, ": unresolved external symbol \"([^\"]*)\"");
if (MicrosoftMatch.Success)
{
SymbolMatches.Add(MicrosoftMatch.Groups[1].Value);
}
// Clean up all the symbol names
SortedSet <string> SymbolNames = new SortedSet <string>(StringComparer.Ordinal);
foreach (string SymbolMatch in SymbolMatches)
{
string SymbolName = SymbolMatch;
// Remove any __declspec qualifiers
SymbolName = Regex.Replace(SymbolName, "(?<![^a-zA-Z_])__declspec\\([^\\)]+\\)", "");
// Remove any argument lists for functions (anything after the first paren)
SymbolName = Regex.Replace(SymbolName, "\\(.*$", "");
// Remove any decorators and type information (greedy match up to the last space)
SymbolName = Regex.Replace(SymbolName, "^.* ", "");
// Add it to the list
SymbolNames.Add(SymbolName);
}
// If we found any symbol names, create a fingerprint for them
if (SymbolNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Job.Project, Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, JobStep.Url, Diagnostic.Message, Diagnostic.Url));
Issue.Identifiers.UnionWith(SymbolNames);
Issues.Add(Issue);
return(true);
}
// Otherwise pass
return(false);
}
/// <summary>
/// Finds or adds an issue for a particular fingerprint
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
BuildHealthIssue FindOrAddIssueForFingerprint(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep, BuildHealthIssue InputIssue)
{
// Find the pattern matcher for this fingerprint
PatternMatcher Matcher = CategoryToMatcher[InputIssue.Category];
// Check if it can be added to an existing open issue
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check this issue already exists in the current stream
Dictionary <string, BuildHealthJobHistory> StepNameToHistory;
if (!Issue.Streams.TryGetValue(InputJob.Stream, out StepNameToHistory))
{
continue;
}
// Check that this issue has not already been closed
BuildHealthJobHistory History;
if (StepNameToHistory.TryGetValue(InputJobStep.Name, out History))
{
if (!History.CanAddFailedBuild(InputJob.Change))
{
continue;
}
}
else
{
if (!StepNameToHistory.Values.Any(x => x.CanAddFailedBuild(InputJob.Change)))
{
continue;
}
}
// Try to merge the fingerprint
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Add the new build
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
// List of changes since the last successful build in this stream
IReadOnlyList <ChangeInfo> Changes = null;
// Find the previous changelist that was built in this stream
List <BuildHealthJobStep> StreamBuilds;
if (State.Streams.TryGetValue(InputJob.Stream, out StreamBuilds))
{
// Find the last change submitted to this stream before it started failing
int LastChange = -1;
for (int Idx = 0; Idx < StreamBuilds.Count && StreamBuilds[Idx].Change < InputJob.Change; Idx++)
{
LastChange = StreamBuilds[Idx].Change;
}
// Allow adding to any open issue that contains changes merged from other branches
if (LastChange != -1)
{
// Query for all the changes since then
Changes = FindChanges(Perforce, InputJob.Stream, LastChange, InputJob.Change);
if (Changes.Count > 0)
{
SortedSet <int> SourceChanges = new SortedSet <int>(Changes.SelectMany(x => x.SourceChanges));
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check if this issue does not already contain this stream, but contains one of the causing changes
if (Issue.Streams.ContainsKey(InputJob.Stream))
{
continue;
}
if (!SourceChanges.Any(x => Issue.SourceChanges.Contains(x)))
{
continue;
}
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Merge the issue
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
}
}
}
// Create new issues for everything else in this stream
if (Changes != null)
{
List <ChangeInfo> Causers = Matcher.FindCausers(Perforce, InputIssue, Changes);
foreach (ChangeInfo Causer in Causers)
{
InputIssue.SourceChanges.UnionWith(Causer.SourceChanges);
InputIssue.PendingWatchers.Add(Causer.Record.User);
}
}
State.Issues.Add(InputIssue);
return(InputIssue);
}
/// <summary> /// Tries to create a fingerprint from an individual diagnostic. /// </summary> /// <param name="Job">The job that was run</param> /// <param name="JobStep">The job step that was run</param> /// <param name="Diagnostic">A diagnostic from the given job step</param> /// <param name="Issues">List which receives all the matched issues.</param> /// <returns>True if this diagnostic should be removed (usually because a fingerprint was created)</returns> public abstract bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues);