/// <inheritdoc />
public override void ProcessFingerprintComputed(ProcessFingerprintComputationEventData data)
{
var semistableHash = PipTable.GetPipSemiStableHash(data.PipId);
if (!m_targetSemistableHashSet.Contains(semistableHash))
{
return;
}
var formattedHash = Pip.FormatSemiStableHash(semistableHash);
string outputFile = GetOutputFileFromFormattedPipHash(formattedHash);
using (var sw = new StreamWriter(outputFile, append: true))
{
var pipInfo = m_model.GetPipInfo(data.PipId);
pipInfo.SetFingerprintComputation(data, CurrentEventWorkerId);
sw.WriteLine(I($"Fingerprint kind: {data.Kind}"));
WriteWeakFingerprintData(pipInfo, sw);
foreach (var strongComputation in data.StrongFingerprintComputations)
{
pipInfo.StrongFingerprintComputation = strongComputation;
WriteStrongFingerprintData(pipInfo, sw);
}
}
}
public override void Prepare()
{
foreach (var semistableHash in m_targetSemistableHashSet)
{
var formattedHash = Pip.FormatSemiStableHash(semistableHash);
string outputFile = GetOutputFileFromFormattedPipHash(formattedHash);
FileUtilities.DeleteFile(outputFile);
}
}
private void LogAzureDevOpsIssue(EventWrittenEventArgs eventData, string eventType)
{
var builder = new StringBuilder();
builder.Append("##vso[task.logIssue type=");
builder.Append(eventType);
var message = eventData.Message;
var args = eventData.Payload == null?CollectionUtilities.EmptyArray <object>() : eventData.Payload.ToArray();
string body;
// see if this event provides provenance info
if (message.StartsWith(EventConstants.ProvenancePrefix, StringComparison.Ordinal))
{
Contract.Assume(args.Length >= 3, "Provenance prefix contains 3 formatting tokens.");
// file
builder.Append(";sourcepath=");
builder.Append(args[0]);
//line
builder.Append(";linenumber=");
builder.Append(args[1]);
//column
builder.Append(";columnnumber=");
builder.Append(args[2]);
//code
builder.Append(";code=DX");
builder.Append(eventData.EventId.ToString("D4"));
}
// construct a short message for ADO console
if (eventData.EventId == (int)EventId.PipProcessError)
{
args[0] = Pip.FormatSemiStableHash((long)args[0]);
message = "[{0}, {10}, {2}] - failed with exit code {8}, {9}\r\n{5}\r\n{6}\r\n{7}";
}
body = string.Format(CultureInfo.CurrentCulture, message, args);
builder.Append(";]");
// substitute newlines in the message
var encodedBody = body.Replace("\r\n", $"%0D%0A##[{eventType}]")
.Replace("\r", $"%0D##[{eventType}]")
.Replace("\n", $"%0A##[{eventType}]");
builder.Append(encodedBody);
m_console.WriteOutputLine(MessageLevel.Info, builder.ToString());
}
private void LogAzureDevOpsIssue(EventWrittenEventArgs eventData, string eventType)
{
using (var pooledInstance = Pools.StringBuilderPool.GetInstance())
{
var builder = pooledInstance.Instance;
builder.Append("##vso[task.logIssue type=");
builder.Append(eventType);
int dxCode = eventData.EventId;
var message = eventData.Message;
var args = eventData.Payload == null?CollectionUtilities.EmptyArray <object>() : eventData.Payload.ToArray();
string body;
var newArgs = args;
// construct a short message for ADO console
if ((eventData.EventId == (int)LogEventId.PipProcessError) ||
(eventData.EventId == (int)SharedLogEventId.DistributionWorkerForwardedError && (int)args[1] == (int)LogEventId.PipProcessError))
{
dxCode = (int)LogEventId.PipProcessError;
var pipProcessError = new PipProcessErrorEventFields(eventData.Payload, eventData.EventId != (int)LogEventId.PipProcessError);
args[0] = Pip.FormatSemiStableHash(pipProcessError.PipSemiStableHash);
args[1] = pipProcessError.ShortPipDescription;
args[2] = pipProcessError.PipSpecPath;
args[3] = pipProcessError.ExitCode;
args[4] = pipProcessError.OptionalMessage;
args[5] = pipProcessError.OutputToLog;
args[6] = pipProcessError.MessageAboutPathsToLog;
args[7] = pipProcessError.PathsToLog;
message = "[{0}, {1}, {2}] - failed with exit code {3}, {4}\r\n{5}\r\n{6}\r\n{7}";
}
else if (eventData.EventId == (int)SharedLogEventId.DistributionWorkerForwardedError || eventData.EventId == (int)SharedLogEventId.DistributionWorkerForwardedWarning)
{
message = "{0}";
}
body = string.Format(CultureInfo.CurrentCulture, message, args);
builder.Append(";]");
// DX code
builder.Append("DX");
builder.Append(dxCode.ToString("D4"));
builder.Append(' ');
// substitute newlines in the message
var encodedBody = body.Replace("\r\n", $"%0D%0A##[{eventType}]")
.Replace("\r", $"%0D##[{eventType}]")
.Replace("\n", $"%0A##[{eventType}]");
builder.Append(encodedBody);
m_console.WriteOutputLine(MessageLevel.Info, builder.ToString());
}
}
public XElement CreateRow(string key, IEnumerable <PipId> values)
{
var allPips = values
.Select(pipId => m_pipTable.HydratePip(pipId, PipQueryContext.ViewerAnalyzer))
.Select(
pip => new
{
PipType = pip.PipType.ToString(),
Hash = Pip.FormatSemiStableHash(pip.Provenance?.SemiStableHash ?? 0),
FullName = pip.Provenance?.OutputValueSymbol.ToString(m_symbolTable),
Spec = pip.Provenance?.Token.Path.ToString(m_pathTable),
Details = GetPipDetails(pip),
})
.OrderBy(obj => obj.PipType)
.ThenBy(obj => obj.FullName)
.Select(obj => string.Format("[{0}] <{1}> {2} - {3} {4}", obj.Hash, obj.PipType, obj.FullName, obj.Spec, obj.Details));
return(CreateRow(key, allPips));
}
public void ExtractTests()
{
var id = 0x123456789ABCDEF0L;
var formatted = Pip.FormatSemiStableHash(id);
var text = $"A pip id inside a text: {formatted}";
var matches = Pip.ExtractSemistableHashes(text);
Assert.Equal(1, matches.Count);
text = $"A text without pip ids";
matches = Pip.ExtractSemistableHashes(text);
AssertExtractMatches(id, 0, matches);
text = @$ "One pipId " "{formatted}" " then another ({formatted}), and two more {formatted}{formatted}";
matches = Pip.ExtractSemistableHashes(text);
AssertExtractMatches(id, 4, matches);
text = $"Almost matches: {formatted.Substring(5)}, {formatted}2930";
matches = Pip.ExtractSemistableHashes(text);
AssertExtractMatches(id, 0, matches);
}
/// <inheritdoc />
protected override void OnError(EventWrittenEventArgs eventData)
{
LogIssueWithLimit(ref m_errorCount, eventData, "error");
switch (eventData.EventId)
{
case (int)LogEventId.PipProcessError:
{
addPipErrors(new PipProcessErrorEventFields(eventData.Payload, false));
}
break;
case (int)SharedLogEventId.DistributionWorkerForwardedError:
{
var actualEventId = (int)eventData.Payload[1];
if (actualEventId == (int)LogEventId.PipProcessError)
{
addPipErrors(new PipProcessErrorEventFields(eventData.Payload, true));
}
}
break;
}
void addPipErrors(PipProcessErrorEventFields pipProcessErrorEventFields)
{
m_buildViewModel.BuildSummary.AddPipError(new BuildSummaryPipDiagnostic
{
SemiStablePipId = Pip.FormatSemiStableHash(pipProcessErrorEventFields.PipSemiStableHash),
PipDescription = pipProcessErrorEventFields.PipDescription,
SpecPath = pipProcessErrorEventFields.PipSpecPath,
ToolName = pipProcessErrorEventFields.PipExe,
ExitCode = pipProcessErrorEventFields.ExitCode,
Output = pipProcessErrorEventFields.OutputToLog,
},
MaxErrorsToIncludeInSummary);
}
}
private void WriteDetailedJsonFile()
{
WriteLineIndented("{");
IncrementIndent();
WriteLineIndented("\"processes\" : [");
IncrementIndent();
int i = 0;
foreach (var processIdAndExecutionPerformance in m_processPerformance.OrderByDescending(pip => pip.Value.ProcessExecutionTime.TotalMilliseconds))
{
var process = CachedGraph.PipGraph.GetPipFromPipId(processIdAndExecutionPerformance.Key) as Process;
Contract.Assert(process != null);
var performance = processIdAndExecutionPerformance.Value;
WriteLineIndented("{");
IncrementIndent();
WriteLineIndented(I($"\"id\" : \"{process.PipId.ToString()}\","));
WriteLineIndented(I($"\"semiStableHash\" : \"{Pip.FormatSemiStableHash(process.SemiStableHash)}\","));
WriteLineIndented(I($"\"tool\" : \"{NormalizeString(process.GetToolName(CachedGraph.Context.PathTable).ToString(CachedGraph.Context.StringTable))}\","));
WriteLineIndented(I($"\"description\" : \"{NormalizeString(process.GetDescription(CachedGraph.Context))}\","));
WriteLineIndented(I($"\"numberOfExecutedProcesses\" : {performance.NumberOfProcesses},"));
WriteLineIndented(I($"\"start\" : \"{performance.ExecutionStart}\","));
WriteLineIndented(I($"\"stop\" : \"{performance.ExecutionStop}\","));
WriteLineIndented(I($"\"result\" : \"{performance.ExecutionLevel}\","));
WriteLineIndented(I($"\"executionTimeInMs\" : {performance.ProcessExecutionTime.TotalMilliseconds},"));
WriteLineIndented(I($"\"userExecutionTimeInMs\" : {performance.UserTime.TotalMilliseconds},"));
WriteLineIndented(I($"\"kernelExecutionTimeInMs\" : {performance.KernelTime.TotalMilliseconds},"));
WriteLineIndented(I($"\"peakMemoryUsageInMb\" : {performance.MemoryCounters.PeakWorkingSetMb},"));
WriteIOData(performance.IO);
WriteLineIndented("\"tags\" : [");
IncrementIndent();
int j = 0;
foreach (var tag in process.Tags)
{
WriteLineIndented(I($"\"{NormalizeString(tag.ToString(StringTable))}\"") + (j < process.Tags.Length - 1 ? "," : string.Empty));
j++;
}
DecrementIndent();
if (m_includeProcessTree && m_reportedProcesses.ContainsKey(process.PipId))
{
WriteLineIndented("],");
WriteLineIndented("\"processTree\" : [");
IncrementIndent();
WriteProcessTree(process);
DecrementIndent();
}
WriteLineIndented("]");
DecrementIndent();
WriteLineIndented("}" + (i < m_processPerformance.Count - 1 ? "," : string.Empty));
i++;
}
DecrementIndent();
WriteLineIndented("]");
DecrementIndent();
WriteIndented("}");
}
private void LogAzureDevOpsIssue(EventWrittenEventArgs eventData, string eventType)
{
using (var pooledInstance = Pools.StringBuilderPool.GetInstance())
{
var builder = pooledInstance.Instance;
builder.Append("##vso[task.logIssue type=");
builder.Append(eventType);
var message = eventData.Message;
var args = eventData.Payload == null?CollectionUtilities.EmptyArray <object>() : eventData.Payload.ToArray();
string body;
// see if this event provides provenance info
if (message.StartsWith(EventConstants.ProvenancePrefix, StringComparison.Ordinal))
{
Contract.Assume(args.Length >= 3, "Provenance prefix contains 3 formatting tokens.");
// file
builder.Append(";sourcepath=");
builder.Append(args[0]);
//line
builder.Append(";linenumber=");
builder.Append(args[1]);
//column
builder.Append(";columnnumber=");
builder.Append(args[2]);
//code
builder.Append(";code=DX");
builder.Append(eventData.EventId.ToString("D4"));
}
var newArgs = args;
// construct a short message for ADO console
if ((eventData.EventId == (int)LogEventId.PipProcessError) ||
(eventData.EventId == (int)SharedLogEventId.DistributionWorkerForwardedError && (int)args[1] == (int)LogEventId.PipProcessError))
{
var pipProcessError = new PipProcessErrorEventFields(eventData.Payload, eventData.EventId != (int)LogEventId.PipProcessError);
args[0] = Pip.FormatSemiStableHash(pipProcessError.PipSemiStableHash);
args[1] = pipProcessError.ShortPipDescription;
args[2] = pipProcessError.PipSpecPath;
args[3] = pipProcessError.ExitCode;
args[4] = pipProcessError.OptionalMessage;
args[5] = pipProcessError.OutputToLog;
args[6] = pipProcessError.MessageAboutPathsToLog;
args[7] = pipProcessError.PathsToLog;
message = "[{0}, {1}, {2}] - failed with exit code {3}, {4}\r\n{5}\r\n{6}\r\n{7}";
}
else if (eventData.EventId == (int)SharedLogEventId.DistributionWorkerForwardedError || eventData.EventId == (int)SharedLogEventId.DistributionWorkerForwardedWarning)
{
message = "{0}";
}
body = string.Format(CultureInfo.CurrentCulture, message, args);
builder.Append(";]");
// substitute newlines in the message
var encodedBody = body.Replace("\r\n", $"%0D%0A##[{eventType}]")
.Replace("\r", $"%0D##[{eventType}]")
.Replace("\n", $"%0A##[{eventType}]");
builder.Append(encodedBody);
m_console.WriteOutputLine(MessageLevel.Info, builder.ToString());
}
}
public void FormatTests(long id)
{
Assert.True(Pip.TryParseSemiStableHash(Pip.FormatSemiStableHash(id), out var parsed));
Assert.Equal(id, parsed);
}