/// <nodoc />
public LoggingConfiguration()
{
CustomLog = new Dictionary <AbsolutePath, IReadOnlyList <int> >();
CustomLogEtwKinds = new Dictionary <AbsolutePath, string>();
NoLog = new List <int>();
NoExecutionLog = new List <int>();
ConsoleVerbosity = VerbosityLevel.Informational;
FileVerbosity = VerbosityLevel.Verbose;
LogCounters = true;
TraceInfo = new Dictionary <string, string>();
Color = true;
AnimateTaskbar = true;
LogStats = true;
LogExecution = true;
FingerprintStoreMode = FingerprintStoreMode.Default;
FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days
EngineCacheLogDirectory = AbsolutePath.Invalid;
EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid;
FingerprintsLogDirectory = AbsolutePath.Invalid;
ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid;
CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid;
HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid;
ReplayWarnings = true;
SubstSource = AbsolutePath.Invalid;
SubstTarget = AbsolutePath.Invalid;
FancyConsole = true;
FancyConsoleMaxStatusPips = 5;
LogStatus = true;
FailPipOnFileAccessError = true;
UseCustomPipDescriptionOnConsole = true;
CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled();
RedirectedLogsDirectory = AbsolutePath.Invalid;
}
public void AssertCacheMissEventLogged(params string[] requiredMessages)
{
if (Configuration.Logging.CacheMissAnalysisOption == CacheMissAnalysisOption.LocalMode())
{
AssertLogContains(caseSensitive: false, requiredLogMessages: requiredMessages);
}
}
public void AssertCacheMissEventLogged(params string[] requiredMessages)
{
if (Configuration.Logging.CacheMissAnalysisOption.Mode == CacheMissAnalysisOption.LocalMode().Mode)
{
var messages = requiredMessages.Select((s) => ObservedInputConstants.ToExpandedString(s));
AssertLogContains(caseSensitive: false, requiredLogMessages: messages.ToArray());
}
}
public void FingerprintIsStableForCreatedDirectories()
{
string dir = Path.Combine(SourceRoot, "dir");
AbsolutePath dirPath = AbsolutePath.Create(Context.PathTable, dir);
DirectoryArtifact dirToEnumerate = DirectoryArtifact.CreateWithZeroPartialSealId(dirPath);
Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.LocalMode();
AbsolutePath nestedDirPath = dirPath.Combine(Context.PathTable, "nested");
var nestedDir = DirectoryArtifact.CreateWithZeroPartialSealId(nestedDirPath);
var outputInNestedDor = CreateOutputFileArtifact(nestedDirPath.ToString(Context.PathTable));
// Create a pip that creates the directory to be enumerated
var dirCreatorBuilder = CreatePipBuilder(new List <Operation>
{
// Create a directory nested into the one that is going to be enumerated
Operation.CreateDir(nestedDir, doNotInfer: true),
// Create a file underneath
Operation.WriteFile(outputInNestedDor, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()) // dummy output
});
// This makes sure we use the right file system, which is aware of alien files
dirCreatorBuilder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads;
// Define the shared opaque
dirCreatorBuilder.AddOutputDirectory(dirPath, global::BuildXL.Pips.Operations.SealDirectoryKind.SharedOpaque);
var dirCreator = SchedulePipBuilder(dirCreatorBuilder);
// Create a pip that enumerates the directory
var dirEnumeratorBuilder = CreatePipBuilder(new List <Operation>
{
Operation.EnumerateDir(dirToEnumerate, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()) // dummy output
});
// This makes sure we use the right file system, which is aware of alien files
dirEnumeratorBuilder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads;
// Define the shared opaque
dirEnumeratorBuilder.AddOutputDirectory(dirPath, global::BuildXL.Pips.Operations.SealDirectoryKind.SharedOpaque);
// Make the enumerator depend on the creator
dirEnumeratorBuilder.AddInputFile(dirCreator.ProcessOutputs.GetRequiredOutputFiles().Single());
var dirEnumerator = SchedulePipBuilder(dirEnumeratorBuilder);
// Run once. The created directory should be ignored for the enumeration fingerprint computation.
RunScheduler().AssertSuccess();
// Simulate shared opaque scrubbing
FileUtilities.DeleteDirectoryContents(nestedDirPath.ToString(Context.PathTable), deleteRootDirectory: true);
// This should be a cache hit. By replaying the dirCreator pip, the enumerated directory is re-created. However,
// it should still be ignored by the fingerprint computation and we should get a cache hit.
RunScheduler().AssertCacheHit(dirEnumerator.Process.PipId);
}
/// <nodoc />
public LoggingConfiguration()
{
CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>();
CustomLogEtwKinds = new Dictionary <AbsolutePath, string>();
NoLog = new List <int>();
NoExecutionLog = new List <int>();
ConsoleVerbosity = VerbosityLevel.Informational;
FileVerbosity = VerbosityLevel.Verbose;
LogCounters = true;
TraceInfo = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
Color = true;
AnimateTaskbar = true;
LogStats = true;
LogExecution = true;
FingerprintStoreMode = FingerprintStoreMode.Default;
FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days
FingerprintStoreBulkLoad = false;
EngineCacheLogDirectory = AbsolutePath.Invalid;
EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid;
FingerprintsLogDirectory = AbsolutePath.Invalid;
ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid;
CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid;
HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid;
SubstSource = AbsolutePath.Invalid;
SubstTarget = AbsolutePath.Invalid;
FancyConsole = true;
FancyConsoleMaxStatusPips = 5;
LogStatus = true;
LogTracer = true;
FailPipOnFileAccessError = true;
UseCustomPipDescriptionOnConsole = true;
CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled();
CacheMissDiffFormat = CacheMissDiffFormat.CustomJsonDiff;
AriaIndividualMessageSizeLimitBytes = (int)(0.8 * 1024 * 1024); // 0.8Mb out of Aria's current 1Mb max limit
MaxNumPipTelemetryBatches = 1;
CacheMissBatch = true;
RedirectedLogsDirectory = AbsolutePath.Invalid;
DumpFailedPips = true;
DumpFailedPipsLogLimit = 50;
DumpFailedPipsWithDynamicData = false;
LogCachedPipOutputs = false;
// Telemetry is enabled by default when the build has access to the Aria telemetry client. This is a Microsoft
// internal only package so this corresponds to versions of BuildXL used internally within Microsoft. This define
// controls whether we attempt to reference the telemetry utilities. It will fail at runtime if that client library
// package is not available.
#if FEATURE_ARIA_TELEMETRY
RemoteTelemetry = Configuration.RemoteTelemetry.EnabledAndNotify;
#endif
}
/// <nodoc />
public LoggingConfiguration()
{
CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>();
CustomLogEtwKinds = new Dictionary <AbsolutePath, string>();
NoLog = new List <int>();
NoExecutionLog = new List <int>();
ConsoleVerbosity = VerbosityLevel.Informational;
FileVerbosity = VerbosityLevel.Verbose;
LogCounters = true;
TraceInfo = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase);
Color = true;
AnimateTaskbar = true;
LogStats = true;
LogExecution = true;
FingerprintStoreMode = FingerprintStoreMode.Default;
FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days
FingerprintStoreBulkLoad = false;
EngineCacheLogDirectory = AbsolutePath.Invalid;
EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid;
FingerprintsLogDirectory = AbsolutePath.Invalid;
ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid;
CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid;
HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid;
ReplayWarnings = true;
SubstSource = AbsolutePath.Invalid;
SubstTarget = AbsolutePath.Invalid;
FancyConsole = true;
FancyConsoleMaxStatusPips = 5;
LogStatus = true;
LogTracer = true;
FailPipOnFileAccessError = true;
UseCustomPipDescriptionOnConsole = true;
CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled();
CacheMissDiffFormat = CacheMissDiffFormat.CustomJsonDiff;
AriaIndividualMessageSizeLimitBytes = (int)(0.8 * 1024 * 1024); // 0.8Mb out of Aria's current 1Mb max limit
MaxNumPipTelemetryBatches = 1;
CacheMissBatch = true;
RedirectedLogsDirectory = AbsolutePath.Invalid;
DumpFailedPips = true;
DumpFailedPipsLogLimit = 50;
DumpFailedPipsWithDynamicData = false;
LogCachedPipOutputs = false;
}
/// <nodoc />
public LoggingConfiguration()
{
CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>();
CustomLogEtwKinds = new Dictionary <AbsolutePath, string>();
NoLog = new List <int>();
NoExecutionLog = new List <int>();
ConsoleVerbosity = VerbosityLevel.Informational;
FileVerbosity = VerbosityLevel.Verbose;
LogCounters = true;
TraceInfo = new Dictionary <string, string>();
Color = true;
AnimateTaskbar = true;
LogStats = true;
LogExecution = true;
FingerprintStoreMode = FingerprintStoreMode.Default;
FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days
FingerprintStoreBulkLoad = false;
EngineCacheLogDirectory = AbsolutePath.Invalid;
EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid;
FingerprintsLogDirectory = AbsolutePath.Invalid;
ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid;
CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid;
HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid;
ReplayWarnings = true;
SubstSource = AbsolutePath.Invalid;
SubstTarget = AbsolutePath.Invalid;
FancyConsole = true;
FancyConsoleMaxStatusPips = 5;
LogStatus = true;
FailPipOnFileAccessError = true;
UseCustomPipDescriptionOnConsole = true;
CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled();
CacheMissDiffFormat = CacheMissDiffFormat.CustomJsonDiff;
AriaIndividualMessageSizeLimitBytes = (int)(1.5 * 1024 * 1024); // 1.5Mb out of Aria's current 2.5Mb max limit
MaxNumPipTelemetryBatches = 10;
// Temporarily disable it by default due to crash.
// TODO: Enable it by default once crash is fixed.
CacheMissBatch = false;
RedirectedLogsDirectory = AbsolutePath.Invalid;
}
public void CreatedDirectoriesUnderSharedOpaquesAreNotPartOfTheFingerprint()
{
string dir = Path.Combine(SourceRoot, "dir");
AbsolutePath dirPath = AbsolutePath.Create(Context.PathTable, dir);
DirectoryArtifact dirToEnumerate = DirectoryArtifact.CreateWithZeroPartialSealId(dirPath);
Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.LocalMode();
AbsolutePath nestedDirPath = dirPath.Combine(Context.PathTable, "nested");
var nestedDir = DirectoryArtifact.CreateWithZeroPartialSealId(nestedDirPath);
var operations = new List <Operation>
{
// Create a directory nested into the one that is going to be enumerated
Operation.CreateDir(nestedDir, doNotInfer: true),
Operation.EnumerateDir(dirToEnumerate, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()) // dummy output
};
var builder = CreatePipBuilder(operations);
// This makes sure we use the right file system, which is aware of alien files
builder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads;
// Define the shared opaque
builder.AddOutputDirectory(dirPath, global::BuildXL.Pips.Operations.SealDirectoryKind.SharedOpaque);
var pip = SchedulePipBuilder(builder);
// Run once
RunScheduler().AssertSuccess();
// Simulate shared opaque scrubbing
FileUtilities.DeleteDirectoryContents(nestedDirPath.ToString(Context.PathTable), deleteRootDirectory: true);
// This should be a cache hit. Directories created by a pip are not part of the fingerprint
RunScheduler().AssertCacheHit(pip.Process.PipId);
}
/// <nodoc />
public LoggingConfiguration(ILoggingConfiguration template, PathRemapper pathRemapper)
: base(template)
{
Contract.Assume(template != null);
Contract.Assume(pathRemapper != null);
LogsDirectory = pathRemapper.Remap(template.LogsDirectory);
RedirectedLogsDirectory = pathRemapper.Remap(template.RedirectedLogsDirectory);
LogPrefix = template.LogPrefix;
Log = pathRemapper.Remap(template.Log);
ErrorLog = pathRemapper.Remap(template.ErrorLog);
WarningLog = pathRemapper.Remap(template.WarningLog);
LogExecution = template.LogExecution;
ExecutionLog = pathRemapper.Remap(template.ExecutionLog);
StoreFingerprints = template.StoreFingerprints;
FingerprintStoreMode = template.FingerprintStoreMode;
FingerprintStoreMaxEntryAgeMinutes = template.FingerprintStoreMaxEntryAgeMinutes;
FingerprintsLogDirectory = pathRemapper.Remap(template.FingerprintsLogDirectory);
ExecutionFingerprintStoreLogDirectory = pathRemapper.Remap(template.ExecutionFingerprintStoreLogDirectory);
CacheLookupFingerprintStoreLogDirectory = pathRemapper.Remap(template.CacheLookupFingerprintStoreLogDirectory);
HistoricMetadataCacheLogDirectory = pathRemapper.Remap(template.HistoricMetadataCacheLogDirectory);
EngineCacheLogDirectory = pathRemapper.Remap(template.EngineCacheLogDirectory);
EngineCacheCorruptFilesLogDirectory = pathRemapper.Remap(template.EngineCacheCorruptFilesLogDirectory);
CustomLog = new Dictionary <AbsolutePath, IReadOnlyList <int> >();
foreach (var kv in template.CustomLog)
{
CustomLog.Add(pathRemapper.Remap(kv.Key), kv.Value);
}
CustomLogEtwKinds = new Dictionary <AbsolutePath, string>();
foreach (var kv in template.CustomLogEtwKinds)
{
CustomLogEtwKinds.Add(pathRemapper.Remap(kv.Key), kv.Value);
}
NoLog = new List <int>(template.NoLog);
NoExecutionLog = new List <int>(template.NoExecutionLog);
Diagnostic = template.Diagnostic;
ConsoleVerbosity = template.ConsoleVerbosity;
FileVerbosity = template.FileVerbosity;
LogCounters = template.LogCounters;
LogStats = template.LogStats;
EnableAsyncLogging = template.EnableAsyncLogging;
StatsLog = pathRemapper.Remap(template.StatsLog);
EventSummaryLog = pathRemapper.Remap(template.EventSummaryLog);
Environment = template.Environment;
RemoteTelemetry = template.RemoteTelemetry;
TraceInfo = new Dictionary <string, string>();
foreach (var kv in template.TraceInfo)
{
TraceInfo.Add(kv.Key, kv.Value);
}
Color = template.Color;
AnimateTaskbar = template.AnimateTaskbar;
RelatedActivityId = template.RelatedActivityId;
LogsToRetain = template.LogsToRetain;
FancyConsole = template.FancyConsole;
FancyConsoleMaxStatusPips = template.FancyConsoleMaxStatusPips;
SubstSource = pathRemapper.Remap(template.SubstSource);
SubstTarget = pathRemapper.Remap(template.SubstTarget);
DisableLoggedPathTranslation = template.DisableLoggedPathTranslation;
LogStatus = template.LogStatus;
StatusFrequencyMs = template.StatusFrequencyMs;
StatusLog = pathRemapper.Remap(template.StatusLog);
CacheMissLog = pathRemapper.Remap(template.CacheMissLog);
DevLog = pathRemapper.Remap(template.DevLog);
RpcLog = pathRemapper.Remap(template.RpcLog);
PipOutputLog = pathRemapper.Remap(template.PipOutputLog);
FailPipOnFileAccessError = template.FailPipOnFileAccessError;
LogMemory = template.LogMemory;
ReplayWarnings = template.ReplayWarnings;
UseCustomPipDescriptionOnConsole = template.UseCustomPipDescriptionOnConsole;
CacheMissAnalysisOption = new CacheMissAnalysisOption(
template.CacheMissAnalysisOption.Mode,
new List <string>(template.CacheMissAnalysisOption.Keys),
pathRemapper.Remap(template.CacheMissAnalysisOption.CustomPath));
}
private void DisableFingerprintStore()
{
Configuration.Logging.StoreFingerprints = false;
Configuration.Logging.FingerprintStoreMode = FingerprintStoreMode.Invalid;
Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled();
}
private void EnableFingerprintStore()
{
Configuration.Logging.StoreFingerprints = true;
Configuration.Logging.FingerprintStoreMode = FingerprintStoreMode.ExecutionFingerprintsOnly;
Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.LocalMode();
}
/// <nodoc />
public LoggingConfiguration(ILoggingConfiguration template, PathRemapper pathRemapper)
: base(template)
{
Contract.Assume(template != null);
Contract.Assume(pathRemapper != null);
LogsDirectory = pathRemapper.Remap(template.LogsDirectory);
RedirectedLogsDirectory = pathRemapper.Remap(template.RedirectedLogsDirectory);
LogPrefix = template.LogPrefix;
Log = pathRemapper.Remap(template.Log);
ErrorLog = pathRemapper.Remap(template.ErrorLog);
WarningLog = pathRemapper.Remap(template.WarningLog);
LogExecution = template.LogExecution;
LogPackedExecution = template.LogPackedExecution;
ExecutionLog = pathRemapper.Remap(template.ExecutionLog);
StoreFingerprints = template.StoreFingerprints;
SaveFingerprintStoreToLogs = template.SaveFingerprintStoreToLogs;
FingerprintStoreMode = template.FingerprintStoreMode;
FingerprintStoreMaxEntryAgeMinutes = template.FingerprintStoreMaxEntryAgeMinutes;
FingerprintStoreBulkLoad = template.FingerprintStoreBulkLoad;
FingerprintsLogDirectory = pathRemapper.Remap(template.FingerprintsLogDirectory);
ExecutionFingerprintStoreLogDirectory = pathRemapper.Remap(template.ExecutionFingerprintStoreLogDirectory);
CacheLookupFingerprintStoreLogDirectory = pathRemapper.Remap(template.CacheLookupFingerprintStoreLogDirectory);
HistoricMetadataCacheLogDirectory = pathRemapper.Remap(template.HistoricMetadataCacheLogDirectory);
EngineCacheLogDirectory = pathRemapper.Remap(template.EngineCacheLogDirectory);
EngineCacheCorruptFilesLogDirectory = pathRemapper.Remap(template.EngineCacheCorruptFilesLogDirectory);
CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>();
foreach (var kv in template.CustomLog)
{
CustomLog.Add(pathRemapper.Remap(kv.Key), kv.Value);
}
CustomLogEtwKinds = new Dictionary <AbsolutePath, string>();
foreach (var kv in template.CustomLogEtwKinds)
{
CustomLogEtwKinds.Add(pathRemapper.Remap(kv.Key), kv.Value);
}
NoLog = new List <int>(template.NoLog);
NoExecutionLog = new List <int>(template.NoExecutionLog);
Diagnostic = template.Diagnostic;
ConsoleVerbosity = template.ConsoleVerbosity;
FileVerbosity = template.FileVerbosity;
LogCounters = template.LogCounters;
LogStats = template.LogStats;
EnableAsyncLogging = template.EnableAsyncLogging;
StatsLog = pathRemapper.Remap(template.StatsLog);
StatsPrfLog = pathRemapper.Remap(template.StatsPrfLog);
EventSummaryLog = pathRemapper.Remap(template.EventSummaryLog);
Environment = template.Environment;
RemoteTelemetry = template.RemoteTelemetry;
TraceInfo = new Dictionary <string, string>();
foreach (var kv in template.TraceInfo)
{
TraceInfo.Add(kv.Key, kv.Value);
}
Color = template.Color;
AnimateTaskbar = template.AnimateTaskbar;
RelatedActivityId = template.RelatedActivityId;
LogsToRetain = template.LogsToRetain;
FancyConsole = template.FancyConsole;
FancyConsoleMaxStatusPips = template.FancyConsoleMaxStatusPips;
SubstSource = pathRemapper.Remap(template.SubstSource);
SubstTarget = pathRemapper.Remap(template.SubstTarget);
DisableLoggedPathTranslation = template.DisableLoggedPathTranslation;
LogStatus = template.LogStatus;
LogTracer = template.LogTracer;
StatusFrequencyMs = template.StatusFrequencyMs;
StatusLog = pathRemapper.Remap(template.StatusLog);
TraceLog = pathRemapper.Remap(template.TraceLog);
CacheMissLog = pathRemapper.Remap(template.CacheMissLog);
DevLog = pathRemapper.Remap(template.DevLog);
RpcLog = pathRemapper.Remap(template.RpcLog);
PipOutputLog = pathRemapper.Remap(template.PipOutputLog);
FailPipOnFileAccessError = template.FailPipOnFileAccessError;
LogMemory = template.LogMemory;
ReplayWarnings = template.ReplayWarnings;
UseCustomPipDescriptionOnConsole = template.UseCustomPipDescriptionOnConsole;
CacheMissAnalysisOption = new CacheMissAnalysisOption(
template.CacheMissAnalysisOption.Mode,
new List <string>(template.CacheMissAnalysisOption.Keys),
pathRemapper.Remap(template.CacheMissAnalysisOption.CustomPath));
CacheMissDiffFormat = template.CacheMissDiffFormat;
CacheMissBatch = template.CacheMissBatch;
OptimizeConsoleOutputForAzureDevOps = template.OptimizeConsoleOutputForAzureDevOps;
InvocationExpandedCommandLineArguments = template.InvocationExpandedCommandLineArguments;
OptimizeProgressUpdatingForAzureDevOps = template.OptimizeProgressUpdatingForAzureDevOps;
OptimizeVsoAnnotationsForAzureDevOps = template.OptimizeVsoAnnotationsForAzureDevOps;
AriaIndividualMessageSizeLimitBytes = template.AriaIndividualMessageSizeLimitBytes;
MaxNumPipTelemetryBatches = template.MaxNumPipTelemetryBatches;
DumpFailedPips = template.DumpFailedPips;
DumpFailedPipsLogLimit = template.DumpFailedPipsLogLimit;
DumpFailedPipsWithDynamicData = template.DumpFailedPipsWithDynamicData;
LogCachedPipOutputs = template.LogCachedPipOutputs;
}
