public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
// Make sure we're running a step that this applies to
if (JobStep.Name.IndexOf("Copyright", StringComparison.OrdinalIgnoreCase) == -1)
{
return(false);
}
// Find any files in compiler output format
HashSet <string> SourceFileNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (Match FileMatch in Regex.Matches(Diagnostic.Message, @"^\s*(?:WARNING|ERROR):\s*([^ ]+\.[a-zA-Z]+):", RegexOptions.Multiline))
{
if (FileMatch.Success)
{
SourceFileNames.Add(FileMatch.Groups[1].Value);
}
}
// If we found any source files, create a diagnostic category for them
if (SourceFileNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, Diagnostic.Message, Diagnostic.Url));
Issue.FileNames.UnionWith(SourceFileNames);
Issues.Add(Issue);
return(true);
}
// Otherwise pass
return(false);
}
/// <summary>
/// Filters all the likely causers from the list of changes since an issue was created
/// </summary>
/// <param name="Perforce">The perforce connection</param>
/// <param name="Issue">The build issue</param>
/// <param name="Changes">List of changes since the issue first occurred.</param>
/// <returns>List of changes which are causers for the issue</returns>
public virtual List <ChangeInfo> FindCausers(PerforceConnection Perforce, BuildHealthIssue Issue, IReadOnlyList <ChangeInfo> Changes)
{
List <ChangeInfo> Causers = new List <ChangeInfo>();
SortedSet <string> FileNamesWithoutPath = GetFileNamesWithoutPath(Issue.FileNames);
if (FileNamesWithoutPath.Count > 0)
{
foreach (ChangeInfo Change in Changes)
{
DescribeRecord DescribeRecord = GetDescribeRecord(Perforce, Change);
if (ContainsFileNames(DescribeRecord, FileNamesWithoutPath))
{
Causers.Add(Change);
}
}
}
if (Causers.Count > 0)
{
return(Causers);
}
else
{
return(new List <ChangeInfo>(Changes));
}
}
public override List <ChangeInfo> FindCausers(PerforceConnection Perforce, BuildHealthIssue Issue, IReadOnlyList <ChangeInfo> Changes)
{
SortedSet <string> FileNamesWithoutPath = new SortedSet <string>(StringComparer.Ordinal);
foreach (string Identifier in Issue.Identifiers)
{
Match Match = Regex.Match(Identifier, "^[AUFS]([A-Z][A-Za-z_]*)::");
if (Match.Success)
{
FileNamesWithoutPath.Add(Match.Groups[1].Value + ".h");
FileNamesWithoutPath.Add(Match.Groups[1].Value + ".cpp");
}
}
if (FileNamesWithoutPath.Count > 0)
{
List <ChangeInfo> Causers = new List <ChangeInfo>();
foreach (ChangeInfo Change in Changes)
{
DescribeRecord DescribeRecord = GetDescribeRecord(Perforce, Change);
if (ContainsFileNames(DescribeRecord, FileNamesWithoutPath))
{
Causers.Add(Change);
}
}
if (Causers.Count > 0)
{
return(Causers);
}
}
return(base.FindCausers(Perforce, Issue, Changes));
}
/// <summary>
/// Finds any suspected causers for a particular failure. Excludes any changes that don't contain code.
/// </summary>
/// <param name="Perforce">The perforce connection</param>
/// <param name="Issue">The build issue</param>
/// <param name="Changes">List of changes since the issue first occurred.</param>
/// <returns>List of changes which are causers for the issue</returns>
public override List <ChangeInfo> FindCausers(PerforceConnection Perforce, BuildHealthIssue Issue, IReadOnlyList <ChangeInfo> Changes)
{
List <ChangeInfo> Causers = base.FindCausers(Perforce, Issue, Changes);
Causers.RemoveAll(x => !ContainsAnyFileWithExtension(Perforce, x, CodeExtensions));
return(Causers);
}
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
// Find any files in compiler output format
HashSet <string> SourceFileNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (Match FileMatch in Regex.Matches(Diagnostic.Message, @"^\s*(?:In file included from\s*)?((?:[A-Za-z]:)?[^\s(:]+)[\(:]\d", RegexOptions.Multiline))
{
if (FileMatch.Success)
{
string FileName = GetNormalizedFileName(FileMatch.Groups[1].Value, JobStep.BaseDirectory);
if (SourceFileExtensions.Any(x => FileName.EndsWith(x, StringComparison.OrdinalIgnoreCase)))
{
SourceFileNames.Add(FileName);
}
}
}
// If we found any source files, create a diagnostic category for them
if (SourceFileNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, ShortenPaths(Diagnostic.Message), Diagnostic.Url));
Issue.FileNames.UnionWith(SourceFileNames);
Issues.Add(Issue);
return(true);
}
// Otherwise pass
return(false);
}
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
string DefaultProject = String.Format("{0} (Unmatched)", Job.Project);
BuildHealthIssue Issue = new BuildHealthIssue(DefaultProject, Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, JobStep.Url, Diagnostic.Message, Diagnostic.Url));
Issue.Identifiers.Add(Diagnostic.Message);
Issues.Add(Issue);
return(true);
}
public override string GetSummary(BuildHealthIssue Issue)
{
if (Issue.FileNames.Count == 0)
{
return("Content errors");
}
else
{
return(String.Format("Content errors in {0}", String.Join(", ", GetFileNamesWithoutPath(Issue.FileNames))));
}
}
public override string GetSummary(BuildHealthIssue Issue)
{
if (Issue.Identifiers.Count == 0)
{
return("Compile errors");
}
else
{
return(String.Format("Compile errors in {0}", String.Join(", ", Issue.Identifiers)));
}
}
public override string GetSummary(BuildHealthIssue Issue)
{
if (Issue.Identifiers.Count == 1)
{
return(String.Format("Undefined symbol '{0}'", Issue.Identifiers.First()));
}
else
{
return(String.Format("Undefined symbols - '{0}'", String.Join("', '", Issue.Identifiers)));
}
}
public override string GetSummary(BuildHealthIssue Issue)
{
string Message = Issue.Diagnostics[0].Message;
int Idx = Message.IndexOf('\n');
if (Idx >= 0)
{
Message = Message.Substring(0, Idx).Trim();
}
return(Message);
}
public override string GetSummary(BuildHealthIssue Issue)
{
SortedSet <string> ShortFileNames = GetSourceFileNames(Issue.FileNames);
if (ShortFileNames.Count == 0)
{
return("Missing copyright notices");
}
else
{
return(String.Format("Missing copyright in {0}", String.Join(", ", ShortFileNames)));
}
}
/// <summary>
/// Determines if one issue can be merged into another
/// </summary>
/// <param name="Source">The source issue</param>
/// <param name="Target">The target issue</param>
public virtual bool CanMerge(BuildHealthIssue Source, BuildHealthIssue Target)
{
// Make sure the categories match
if (Source.Category != Target.Category)
{
return(false);
}
// Check that a filename or message matches
if (!Source.FileNames.Any(x => Target.FileNames.Contains(x)) && !Source.Identifiers.Any(x => Target.Identifiers.Contains(x)))
{
return(false);
}
return(true);
}
/// <summary>
/// Adds diagnostics from a job step into the issue database
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
void AddStep(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep)
{
// Create a lazily evaluated list of changes that are responsible for any errors
Lazy <IReadOnlyList <ChangeInfo> > LazyChanges = new Lazy <IReadOnlyList <ChangeInfo> >(() => FindChanges(Perforce, State, InputJob));
// Create issues for any diagnostics in this step
List <BuildHealthIssue> InputIssues = new List <BuildHealthIssue>();
foreach (PatternMatcher Matcher in Matchers)
{
Matcher.Match(InputJob, InputJobStep, InputJobStep.Diagnostics, InputIssues);
}
// Merge the issues together
List <BuildHealthIssue> NewIssues = new List <BuildHealthIssue>();
foreach (BuildHealthIssue InputIssue in InputIssues)
{
BuildHealthIssue OutputIssue = MergeIntoExistingIssue(Perforce, State, InputJob, InputJobStep, InputIssue, LazyChanges);
if (OutputIssue == null)
{
NewIssues.Add(InputIssue);
State.Issues.Add(InputIssue);
OutputIssue = InputIssue;
}
AddFailureToIssue(OutputIssue, InputJob, InputJobStep, InputIssue.Diagnostics[0].ErrorUrl, State);
}
// Update the watchers for any new issues
foreach (BuildHealthIssue NewIssue in NewIssues)
{
IReadOnlyList <ChangeInfo> Changes = LazyChanges.Value;
if (Changes != null)
{
// Find the pattern matcher for this issue
PatternMatcher Matcher = CategoryToMatcher[NewIssue.Category];
// Update the causers
List <ChangeInfo> Causers = Matcher.FindCausers(Perforce, NewIssue, Changes);
foreach (ChangeInfo Causer in Causers)
{
NewIssue.SourceChanges.UnionWith(Causer.SourceChanges);
NewIssue.PendingWatchers.Add(Causer.Record.User);
}
}
}
}
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
HashSet <string> FileNames = new HashSet <string>();
foreach (Match Match in Regex.Matches(Diagnostic.Message, @"^\s*Log[a-zA-Z0-9]+:\s+(?:Error:|Warning:)\s+((?:[a-zA-Z]:)?[^:]+(?:.uasset|.umap)):\s*(.*)"))
{
FileNames.Add(GetNormalizedFileName(Match.Groups[1].Value, JobStep.BaseDirectory));
}
if (FileNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Job.Project, Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, JobStep.Url, Diagnostic.Message, Diagnostic.Url));
Issue.FileNames.UnionWith(FileNames);
Issues.Add(Issue);
return(true);
}
return(false);
}
/// <summary>
/// Merge one fingerprint with another
/// </summary>
/// <param name="Source">The source fingerprint</param>
/// <param name="Target">The fingerprint to merge into</param>
public virtual void Merge(BuildHealthIssue Source, BuildHealthIssue Target)
{
HashSet <string> TargetMessages = new HashSet <string>(Target.Diagnostics.Select(x => x.Message), StringComparer.Ordinal);
foreach (BuildHealthDiagnostic SourceDiagnostic in Source.Diagnostics)
{
if (Target.Diagnostics.Count >= 50)
{
break;
}
if (!TargetMessages.Contains(SourceDiagnostic.Message))
{
Target.Diagnostics.Add(SourceDiagnostic);
}
}
Target.FileNames.UnionWith(Source.FileNames);
Target.Identifiers.UnionWith(Source.Identifiers);
}
/// <summary>
/// Adds diagnostics from a job step into the issue database
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
void AddStep(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep)
{
// Create all the fingerprints for failures in this step
List <BuildHealthIssue> InputIssues = new List <BuildHealthIssue>();
if (InputJobStep.Diagnostics != null)
{
List <InputDiagnostic> Diagnostics = new List <InputDiagnostic>(InputJobStep.Diagnostics);
foreach (PatternMatcher PatternMatcher in Matchers)
{
PatternMatcher.Match(InputJob, InputJobStep, Diagnostics, InputIssues);
}
}
// Add all the fingerprints to issues
foreach (BuildHealthIssue InputIssue in InputIssues)
{
BuildHealthIssue Issue = FindOrAddIssueForFingerprint(Perforce, State, InputJob, InputJobStep, InputIssue);
AddFailureToIssue(Issue, InputJob, InputJobStep, InputIssue.Diagnostics[0].ErrorUrl, State);
}
}
/// <summary>
/// Adds a new build history for a stream
/// </summary>
/// <param name="Issue">The issue to add a build to</param>
/// <param name="InputJob">The job containing the error</param>
/// <param name="InputJobStep">The job step containing the error</param>
/// <param name="InputErrorUrl">Url of the error</param>
/// <param name="State">Current persistent state. Used to find previous build history.</param>
void AddFailureToIssue(BuildHealthIssue Issue, InputJob InputJob, InputJobStep InputJobStep, string InputErrorUrl, BuildHealthState State)
{
// Find or add a step name to history mapping
Dictionary <string, BuildHealthJobHistory> StepNameToHistory;
if (!Issue.Streams.TryGetValue(InputJob.Stream, out StepNameToHistory))
{
StepNameToHistory = new Dictionary <string, BuildHealthJobHistory>();
Issue.Streams.Add(InputJob.Stream, StepNameToHistory);
}
// Find or add a history for this step
BuildHealthJobHistory History;
if (!StepNameToHistory.TryGetValue(InputJobStep.Name, out History))
{
History = new BuildHealthJobHistory(State.FindBuildBefore(InputJob.Stream, InputJob.Change, InputJobStep.Name));
StepNameToHistory.Add(InputJobStep.Name, History);
}
// Add the new build
History.AddFailedBuild(CreateBuildForJobStep(InputJob, InputJobStep, InputErrorUrl));
}
/// <summary>
/// Finds or adds an issue for a particular issue
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
BuildHealthIssue MergeIntoExistingIssue(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep, BuildHealthIssue InputIssue, Lazy <IReadOnlyList <ChangeInfo> > LazyChanges)
{
// Find the pattern matcher for this fingerprint
PatternMatcher Matcher = CategoryToMatcher[InputIssue.Category];
// Check if it can be added to an existing open issue
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check this issue already exists in the current stream
Dictionary <string, BuildHealthJobHistory> StepNameToHistory;
if (!Issue.Streams.TryGetValue(InputJob.Stream, out StepNameToHistory))
{
continue;
}
// Check that this issue has not already been closed
BuildHealthJobHistory History;
if (StepNameToHistory.TryGetValue(InputJobStep.Name, out History))
{
if (!History.CanAddFailedBuild(InputJob.Change))
{
continue;
}
}
else
{
if (!StepNameToHistory.Values.Any(x => x.CanAddFailedBuild(InputJob.Change)))
{
continue;
}
}
// Try to merge the fingerprint
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Add the new build
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
// Check if this issue can be merged with an issue built in another stream
IReadOnlyList <ChangeInfo> Changes = LazyChanges.Value;
if (Changes != null && Changes.Count > 0)
{
SortedSet <int> SourceChanges = new SortedSet <int>(Changes.SelectMany(x => x.SourceChanges));
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check if this issue does not already contain this stream, but contains one of the causing changes
if (Issue.Streams.ContainsKey(InputJob.Stream))
{
continue;
}
if (!SourceChanges.Any(x => Issue.SourceChanges.Contains(x)))
{
continue;
}
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Merge the issue
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
}
// Check if it can be merged into an issue that's been created for this job. We only do this after exhausting all other options.
foreach (BuildHealthIssue Issue in State.Issues)
{
if (Issue.InitialJobUrl == InputIssue.InitialJobUrl && Matcher.CanMergeInitialJob(InputIssue, Issue))
{
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
}
return(null);
}
/// <summary>
/// Main command entry point
/// </summary>
/// <param name="Arguments">The command line arguments</param>
public override void Exec(CommandLineArguments Arguments)
{
// Parse the arguments
bool bClean = Arguments.HasOption("-Clean");
string PerforcePort = Arguments.GetStringOrDefault("-P4Port=", null);
string PerforceUser = Arguments.GetStringOrDefault("-P4User="******"-InputFile=", null);
FileReference StateFile = Arguments.GetFileReference("-StateFile=");
string ServerUrl = Arguments.GetStringOrDefault("-Server=", null);
bool bKeepHistory = Arguments.HasOption("-KeepHistory");
bool bReadOnly = Arguments.HasOption("-ReadOnly");
DirectoryReference SaveUnmatchedDir = Arguments.GetDirectoryReferenceOrDefault("-SaveUnmatched=", null);
Arguments.CheckAllArgumentsUsed();
// Build a mapping from category to matching
Dictionary <string, PatternMatcher> CategoryNameToMatcher = new Dictionary <string, PatternMatcher>();
foreach (PatternMatcher Matcher in Matchers)
{
CategoryNameToMatcher[Matcher.Category] = Matcher;
}
// Complete any interrupted operation to update the state file
CompleteStateTransaction(StateFile);
// Read the persistent data file
BuildHealthState State;
if (!bClean && FileReference.Exists(StateFile))
{
Log.TraceInformation("Reading persistent data from {0}", StateFile);
State = DeserializeJson <BuildHealthState>(StateFile);
}
else
{
Log.TraceInformation("Creating new persistent data");
State = new BuildHealthState();
}
// Fixup any issues loaded from disk
foreach (BuildHealthIssue Issue in State.Issues)
{
if (Issue.References == null)
{
Issue.References = new SortedSet <string>();
}
}
// Create the Perforce connection
PerforceConnection Perforce = new PerforceConnection(PerforcePort, PerforceUser, null);
// Process the input data
if (InputFile != null)
{
// Parse the input file
Log.TraceInformation("Reading build results from {0}", InputFile);
InputData InputData = DeserializeJson <InputData>(InputFile);
// Parse all the builds and add them to the persistent data
List <InputJob> InputJobs = InputData.Jobs.OrderBy(x => x.Change).ThenBy(x => x.Stream).ToList();
Stopwatch Timer = Stopwatch.StartNew();
foreach (InputJob InputJob in InputJobs)
{
// Add a new build for each job step
foreach (InputJobStep InputJobStep in InputJob.Steps)
{
BuildHealthJobStep NewBuild = new BuildHealthJobStep(InputJob.Change, InputJob.Name, InputJob.Url, InputJobStep.Name, InputJobStep.Url, null);
State.AddBuild(InputJob.Stream, NewBuild);
}
// Add all the job steps
List <InputJobStep> InputJobSteps = InputJob.Steps.OrderBy(x => x.Name).ToList();
foreach (InputJobStep InputJobStep in InputJobSteps)
{
if (InputJobStep.Diagnostics != null && InputJobStep.Diagnostics.Count > 0)
{
AddStep(Perforce, State, InputJob, InputJobStep);
}
}
// Remove any steps which are empty
InputJob.Steps.RemoveAll(x => x.Diagnostics == null || x.Diagnostics.Count == 0);
}
InputJobs.RemoveAll(x => x.Steps.Count == 0);
Log.TraceInformation("Added jobs in {0}s", Timer.Elapsed.TotalSeconds);
// If there are any unmatched issues, save out the current state and remaining input
if (SaveUnmatchedDir != null && InputJobs.Count > 0)
{
DirectoryReference.CreateDirectory(SaveUnmatchedDir);
if (FileReference.Exists(StateFile))
{
FileReference.Copy(StateFile, FileReference.Combine(SaveUnmatchedDir, "State.json"), true);
}
SerializeJson(FileReference.Combine(SaveUnmatchedDir, "Input.json"), InputData);
}
// Try to find the next successful build for each stream, so we can close it as part of updating the server
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
BuildHealthIssue Issue = State.Issues[Idx];
foreach (string Stream in Issue.Streams.Keys)
{
Dictionary <string, BuildHealthJobHistory> StepNameToHistory = Issue.Streams[Stream];
foreach (string StepName in StepNameToHistory.Keys)
{
BuildHealthJobHistory IssueHistory = StepNameToHistory[StepName];
if (IssueHistory.FailedBuilds.Count > 0 && IssueHistory.NextSuccessfulBuild == null)
{
// Find the successful build after this change
BuildHealthJobStep LastFailedBuild = IssueHistory.FailedBuilds[IssueHistory.FailedBuilds.Count - 1];
IssueHistory.NextSuccessfulBuild = State.FindBuildAfter(Stream, LastFailedBuild.Change, StepName);
}
}
}
}
// Find the change two days before the latest change being added
if (InputData.Jobs.Count > 0 && !bKeepHistory)
{
// Find all the unique change numbers for each stream
SortedSet <int> ChangeNumbers = new SortedSet <int>();
foreach (List <BuildHealthJobStep> Builds in State.Streams.Values)
{
ChangeNumbers.UnionWith(Builds.Select(x => x.Change));
}
// Get the latest change record
int LatestChangeNumber = InputData.Jobs.Min(x => x.Change);
ChangeRecord LatestChangeRecord = Perforce.GetChange(GetChangeOptions.None, LatestChangeNumber).Data;
// Step forward through all the changelists until we get to one we don't want to delete
int DeleteChangeNumber = -1;
foreach (int ChangeNumber in ChangeNumbers)
{
ChangeRecord ChangeRecord = Perforce.GetChange(GetChangeOptions.None, ChangeNumber).Data;
if (ChangeRecord.Date > LatestChangeRecord.Date - TimeSpan.FromDays(2))
{
break;
}
DeleteChangeNumber = ChangeNumber;
}
// Remove any builds we no longer want to track
foreach (List <BuildHealthJobStep> Builds in State.Streams.Values)
{
Builds.RemoveAll(x => x.Change <= DeleteChangeNumber);
}
}
}
// Mark any issues as resolved
foreach (BuildHealthIssue Issue in State.Issues)
{
if (Issue.IsResolved())
{
if (!Issue.ResolvedAt.HasValue)
{
Issue.ResolvedAt = DateTime.UtcNow;
}
}
else
{
if (Issue.ResolvedAt.HasValue)
{
Issue.ResolvedAt = null;
}
}
}
// If we're in read-only mode, don't write anything out
if (bReadOnly)
{
return;
}
// Save the persistent data
Log.TraceInformation("Writing persistent data to {0}", StateFile);
DirectoryReference.CreateDirectory(StateFile.Directory);
WriteState(StateFile, State);
// Synchronize with the server
if (ServerUrl != null)
{
// Post any issue updates
foreach (BuildHealthIssue Issue in State.Issues)
{
PatternMatcher Matcher;
if (!CategoryNameToMatcher.TryGetValue(Issue.Category, out Matcher))
{
continue;
}
string Summary = Matcher.GetSummary(Issue);
if (Issue.Id == -1)
{
Log.TraceInformation("Adding issue: {0}", Issue);
if (Issue.PendingWatchers.Count == 0)
{
Log.TraceWarning("(No possible causers)");
}
CommandTypes.AddIssue IssueBody = new CommandTypes.AddIssue();
IssueBody.Project = Issue.Project;
IssueBody.Summary = Summary;
if (Issue.PendingWatchers.Count == 1)
{
IssueBody.Owner = Issue.PendingWatchers.First();
}
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues", ServerUrl), "POST", IssueBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add issue");
}
Issue.Id = ParseHttpResponse <CommandTypes.AddIssueResponse>(Response).Id;
}
Issue.PostedSummary = Summary;
WriteState(StateFile, State);
}
else if (Issue.PostedSummary == null || !String.Equals(Issue.PostedSummary, Summary, StringComparison.Ordinal))
{
Log.TraceInformation("Updating issue {0}", Issue.Id);
CommandTypes.UpdateIssue IssueBody = new CommandTypes.UpdateIssue();
IssueBody.Summary = Summary;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}", ServerUrl, Issue.Id), "PUT", IssueBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add issue");
}
}
Issue.PostedSummary = Summary;
WriteState(StateFile, State);
}
}
// Add any new builds associated with issues
Dictionary <string, long> JobStepUrlToId = new Dictionary <string, long>(StringComparer.Ordinal);
foreach (BuildHealthIssue Issue in State.Issues)
{
foreach (KeyValuePair <string, Dictionary <string, BuildHealthJobHistory> > StreamPair in Issue.Streams)
{
foreach (BuildHealthJobHistory StreamHistory in StreamPair.Value.Values)
{
foreach (BuildHealthJobStep Build in StreamHistory.Builds)
{
if (!Build.bPostedToServer)
{
Log.TraceInformation("Adding {0} to issue {1}", Build.JobStepUrl, Issue.Id);
CommandTypes.AddBuild AddBuild = new CommandTypes.AddBuild();
AddBuild.Stream = StreamPair.Key;
AddBuild.Change = Build.Change;
AddBuild.JobName = Build.JobName;
AddBuild.JobUrl = Build.JobUrl;
AddBuild.JobStepName = Build.JobStepName;
AddBuild.JobStepUrl = Build.JobStepUrl;
AddBuild.ErrorUrl = Build.ErrorUrl;
AddBuild.Outcome = (Build == StreamHistory.PrevSuccessfulBuild || Build == StreamHistory.NextSuccessfulBuild)? CommandTypes.Outcome.Success : CommandTypes.Outcome.Error;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/builds", ServerUrl, Issue.Id), "POST", AddBuild))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add build");
}
Build.Id = ParseHttpResponse <CommandTypes.AddBuildResponse>(Response).Id;
}
Build.bPostedToServer = true;
WriteState(StateFile, State);
}
if (Build.Id != -1)
{
JobStepUrlToId[Build.JobStepUrl] = Build.Id;
}
}
}
}
}
// Add any new diagnostics
foreach (BuildHealthIssue Issue in State.Issues)
{
foreach (BuildHealthDiagnostic Diagnostic in Issue.Diagnostics)
{
if (!Diagnostic.bPostedToServer)
{
string Summary = Diagnostic.Message;
const int MaxLength = 40;
if (Summary.Length > MaxLength)
{
Summary = Summary.Substring(0, MaxLength).TrimEnd();
}
Log.TraceInformation("Adding diagnostic '{0}' to issue {1}", Summary, Issue.Id);
CommandTypes.AddDiagnostic AddDiagnostic = new CommandTypes.AddDiagnostic();
long BuildId;
if (Diagnostic.JobStepUrl != null && JobStepUrlToId.TryGetValue(Diagnostic.JobStepUrl, out BuildId))
{
AddDiagnostic.BuildId = BuildId;
}
else
{
Console.WriteLine("ERROR");
}
AddDiagnostic.Message = Diagnostic.Message;
AddDiagnostic.Url = Diagnostic.ErrorUrl;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/diagnostics", ServerUrl, Issue.Id), "POST", AddDiagnostic))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add build");
}
}
Diagnostic.bPostedToServer = true;
WriteState(StateFile, State);
}
}
}
// Close any issues which are complete
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
BuildHealthIssue Issue = State.Issues[Idx];
if (Issue.ResolvedAt.HasValue != Issue.bPostedResolved)
{
Log.TraceInformation("Setting issue {0} resolved flag to {1}", Issue.Id, Issue.ResolvedAt.HasValue);
CommandTypes.UpdateIssue UpdateBody = new CommandTypes.UpdateIssue();
UpdateBody.Resolved = Issue.ResolvedAt.HasValue;
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}", ServerUrl, Issue.Id), "PUT", UpdateBody))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to delete issue");
}
}
Issue.bPostedResolved = Issue.ResolvedAt.HasValue;
WriteState(StateFile, State);
}
}
// Update watchers on any open builds
foreach (BuildHealthIssue Issue in State.Issues)
{
while (Issue.PendingWatchers.Count > 0)
{
CommandTypes.Watcher Watcher = new CommandTypes.Watcher();
Watcher.UserName = Issue.PendingWatchers.First();
using (HttpWebResponse Response = SendHttpRequest(String.Format("{0}/api/issues/{1}/watchers", ServerUrl, Issue.Id), "POST", Watcher))
{
int ResponseCode = (int)Response.StatusCode;
if (!(ResponseCode >= 200 && ResponseCode <= 299))
{
throw new Exception("Unable to add watcher");
}
}
Issue.PendingWatchers.Remove(Watcher.UserName);
Issue.Watchers.Add(Watcher.UserName);
WriteState(StateFile, State);
}
}
}
// Remove any issues which have been resolved for 24 hours. We have to keep information about issues that have been fixed for some time; we may be updating the same job
// multiple times while other steps are running, and we don't want to keep opening new issues for it. Also, it can take time for changes to propagate between streams.
DateTime RemoveIssueTime = DateTime.UtcNow - TimeSpan.FromHours(24.0);
for (int Idx = 0; Idx < State.Issues.Count; Idx++)
{
BuildHealthIssue Issue = State.Issues[Idx];
if (Issue.ResolvedAt.HasValue && Issue.ResolvedAt.Value < RemoveIssueTime)
{
State.Issues.RemoveAt(Idx--);
WriteState(StateFile, State);
continue;
}
}
// TODO: VERIFY ISSUES ARE CLOSED
}
/// <summary> /// Gets the summary for an issue /// </summary> /// <param name="Issue">The issue to summarize</param> /// <returns>The summary text for this issue</returns> public abstract string GetSummary(BuildHealthIssue Issue);
public override bool TryMatch(InputJob Job, InputJobStep JobStep, InputDiagnostic Diagnostic, List <BuildHealthIssue> Issues)
{
List <string> SymbolMatches = new List <string>();
// Mac link error:
// Undefined symbols for architecture arm64:
// "Foo::Bar() const", referenced from:
if (Regex.IsMatch(Diagnostic.Message, "^Undefined symbols"))
{
foreach (string Line in Diagnostic.Message.Split('\n'))
{
Match SymbolMatch = Regex.Match(Line, "^ \"(.+)\"");
if (SymbolMatch.Success)
{
SymbolMatches.Add(SymbolMatch.Groups[1].Value);
}
}
}
// Android link error:
// Foo.o:(.data.rel.ro + 0x5d88): undefined reference to `Foo::Bar()'
Match UndefinedReference = Regex.Match(Diagnostic.Message, ": undefined reference to [`']([^`']+)");
if (UndefinedReference.Success)
{
SymbolMatches.Add(UndefinedReference.Groups[1].Value);
}
// LLD link error:
// ld.lld.exe: error: undefined symbol: Foo::Bar() const
Match LldMatch = Regex.Match(Diagnostic.Message, "error: undefined symbol:\\s*(.+)");
if (LldMatch.Success)
{
SymbolMatches.Add(LldMatch.Groups[1].Value);
}
// Link error:
// Link: error: L0039: reference to undefined symbol `Foo::Bar() const' in file
Match LinkMatch = Regex.Match(Diagnostic.Message, ": reference to undefined symbol [`']([^`']+)");
if (LinkMatch.Success)
{
SymbolMatches.Add(LinkMatch.Groups[1].Value);
}
// Microsoft linker error:
// Foo.cpp.obj : error LNK2001: unresolved external symbol \"private: virtual void __cdecl UAssetManager::InitializeAssetBundlesFromMetadata_Recursive(class UStruct const *,void const *,struct FAssetBundleData &,class FName,class TSet<void const *,struct DefaultKeyFuncs<void const *,0>,class FDefaultSetAllocator> &)const \" (?InitializeAssetBundlesFromMetadata_Recursive@UAssetManager@@EEBAXPEBVUStruct@@PEBXAEAUFAssetBundleData@@VFName@@AEAV?$TSet@PEBXU?$DefaultKeyFuncs@PEBX$0A@@@VFDefaultSetAllocator@@@@@Z)",
Match MicrosoftMatch = Regex.Match(Diagnostic.Message, ": unresolved external symbol \"([^\"]*)\"");
if (MicrosoftMatch.Success)
{
SymbolMatches.Add(MicrosoftMatch.Groups[1].Value);
}
// Clean up all the symbol names
SortedSet <string> SymbolNames = new SortedSet <string>(StringComparer.Ordinal);
foreach (string SymbolMatch in SymbolMatches)
{
string SymbolName = SymbolMatch;
// Remove any __declspec qualifiers
SymbolName = Regex.Replace(SymbolName, "(?<![^a-zA-Z_])__declspec\\([^\\)]+\\)", "");
// Remove any argument lists for functions (anything after the first paren)
SymbolName = Regex.Replace(SymbolName, "\\(.*$", "");
// Remove any decorators and type information (greedy match up to the last space)
SymbolName = Regex.Replace(SymbolName, "^.* ", "");
// Add it to the list
SymbolNames.Add(SymbolName);
}
// If we found any symbol names, create a fingerprint for them
if (SymbolNames.Count > 0)
{
BuildHealthIssue Issue = new BuildHealthIssue(Job.Project, Category, Job.Url, new BuildHealthDiagnostic(JobStep.Name, JobStep.Url, Diagnostic.Message, Diagnostic.Url));
Issue.Identifiers.UnionWith(SymbolNames);
Issues.Add(Issue);
return(true);
}
// Otherwise pass
return(false);
}
/// <summary>
/// Determines if an issue can be merged into another issue that occurred at the same initial job
/// </summary>
/// <param name="Source">The source issue</param>
/// <param name="Target">The target issue</param>
/// <returns>True if the two new issues can be merged</returns>
public virtual bool CanMergeInitialJob(BuildHealthIssue Source, BuildHealthIssue Target)
{
return(Source.Category == Target.Category);
}
public override List <ChangeInfo> FindCausers(PerforceConnection Perforce, BuildHealthIssue Issue, IReadOnlyList <ChangeInfo> Changes)
{
return(new List <ChangeInfo>());
}
/// <summary>
/// Finds or adds an issue for a particular fingerprint
/// </summary>
/// <param name="Perforce">Perforce connection used to find possible causers</param>
/// <param name="State">The current set of tracked issues</param>
/// <param name="Build">The new build</param>
/// <param name="PreviousChange">The last changelist that was built before this one</param>
/// <param name="InputJob">Job containing the step to add</param>
/// <param name="InputJobStep">The job step to add</param>
BuildHealthIssue FindOrAddIssueForFingerprint(PerforceConnection Perforce, BuildHealthState State, InputJob InputJob, InputJobStep InputJobStep, BuildHealthIssue InputIssue)
{
// Find the pattern matcher for this fingerprint
PatternMatcher Matcher = CategoryToMatcher[InputIssue.Category];
// Check if it can be added to an existing open issue
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check this issue already exists in the current stream
Dictionary <string, BuildHealthJobHistory> StepNameToHistory;
if (!Issue.Streams.TryGetValue(InputJob.Stream, out StepNameToHistory))
{
continue;
}
// Check that this issue has not already been closed
BuildHealthJobHistory History;
if (StepNameToHistory.TryGetValue(InputJobStep.Name, out History))
{
if (!History.CanAddFailedBuild(InputJob.Change))
{
continue;
}
}
else
{
if (!StepNameToHistory.Values.Any(x => x.CanAddFailedBuild(InputJob.Change)))
{
continue;
}
}
// Try to merge the fingerprint
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Add the new build
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
// List of changes since the last successful build in this stream
IReadOnlyList <ChangeInfo> Changes = null;
// Find the previous changelist that was built in this stream
List <BuildHealthJobStep> StreamBuilds;
if (State.Streams.TryGetValue(InputJob.Stream, out StreamBuilds))
{
// Find the last change submitted to this stream before it started failing
int LastChange = -1;
for (int Idx = 0; Idx < StreamBuilds.Count && StreamBuilds[Idx].Change < InputJob.Change; Idx++)
{
LastChange = StreamBuilds[Idx].Change;
}
// Allow adding to any open issue that contains changes merged from other branches
if (LastChange != -1)
{
// Query for all the changes since then
Changes = FindChanges(Perforce, InputJob.Stream, LastChange, InputJob.Change);
if (Changes.Count > 0)
{
SortedSet <int> SourceChanges = new SortedSet <int>(Changes.SelectMany(x => x.SourceChanges));
foreach (BuildHealthIssue Issue in State.Issues)
{
// Check if this issue does not already contain this stream, but contains one of the causing changes
if (Issue.Streams.ContainsKey(InputJob.Stream))
{
continue;
}
if (!SourceChanges.Any(x => Issue.SourceChanges.Contains(x)))
{
continue;
}
if (!Matcher.CanMerge(InputIssue, Issue))
{
continue;
}
// Merge the issue
Matcher.Merge(InputIssue, Issue);
return(Issue);
}
}
}
}
// Create new issues for everything else in this stream
if (Changes != null)
{
List <ChangeInfo> Causers = Matcher.FindCausers(Perforce, InputIssue, Changes);
foreach (ChangeInfo Causer in Causers)
{
InputIssue.SourceChanges.UnionWith(Causer.SourceChanges);
InputIssue.PendingWatchers.Add(Causer.Record.User);
}
}
State.Issues.Add(InputIssue);
return(InputIssue);
}
