/// <nodoc /> public LoggingConfiguration() { CustomLog = new Dictionary <AbsolutePath, IReadOnlyList <int> >(); CustomLogEtwKinds = new Dictionary <AbsolutePath, string>(); NoLog = new List <int>(); NoExecutionLog = new List <int>(); ConsoleVerbosity = VerbosityLevel.Informational; FileVerbosity = VerbosityLevel.Verbose; LogCounters = true; TraceInfo = new Dictionary <string, string>(); Color = true; AnimateTaskbar = true; LogStats = true; LogExecution = true; FingerprintStoreMode = FingerprintStoreMode.Default; FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days EngineCacheLogDirectory = AbsolutePath.Invalid; EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid; FingerprintsLogDirectory = AbsolutePath.Invalid; ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid; CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid; HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid; ReplayWarnings = true; SubstSource = AbsolutePath.Invalid; SubstTarget = AbsolutePath.Invalid; FancyConsole = true; FancyConsoleMaxStatusPips = 5; LogStatus = true; FailPipOnFileAccessError = true; UseCustomPipDescriptionOnConsole = true; CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled(); RedirectedLogsDirectory = AbsolutePath.Invalid; }
public void AssertCacheMissEventLogged(params string[] requiredMessages) { if (Configuration.Logging.CacheMissAnalysisOption == CacheMissAnalysisOption.LocalMode()) { AssertLogContains(caseSensitive: false, requiredLogMessages: requiredMessages); } }
public void AssertCacheMissEventLogged(params string[] requiredMessages) { if (Configuration.Logging.CacheMissAnalysisOption.Mode == CacheMissAnalysisOption.LocalMode().Mode) { var messages = requiredMessages.Select((s) => ObservedInputConstants.ToExpandedString(s)); AssertLogContains(caseSensitive: false, requiredLogMessages: messages.ToArray()); } }
public void FingerprintIsStableForCreatedDirectories() { string dir = Path.Combine(SourceRoot, "dir"); AbsolutePath dirPath = AbsolutePath.Create(Context.PathTable, dir); DirectoryArtifact dirToEnumerate = DirectoryArtifact.CreateWithZeroPartialSealId(dirPath); Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.LocalMode(); AbsolutePath nestedDirPath = dirPath.Combine(Context.PathTable, "nested"); var nestedDir = DirectoryArtifact.CreateWithZeroPartialSealId(nestedDirPath); var outputInNestedDor = CreateOutputFileArtifact(nestedDirPath.ToString(Context.PathTable)); // Create a pip that creates the directory to be enumerated var dirCreatorBuilder = CreatePipBuilder(new List <Operation> { // Create a directory nested into the one that is going to be enumerated Operation.CreateDir(nestedDir, doNotInfer: true), // Create a file underneath Operation.WriteFile(outputInNestedDor, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) // dummy output }); // This makes sure we use the right file system, which is aware of alien files dirCreatorBuilder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads; // Define the shared opaque dirCreatorBuilder.AddOutputDirectory(dirPath, global::BuildXL.Pips.Operations.SealDirectoryKind.SharedOpaque); var dirCreator = SchedulePipBuilder(dirCreatorBuilder); // Create a pip that enumerates the directory var dirEnumeratorBuilder = CreatePipBuilder(new List <Operation> { Operation.EnumerateDir(dirToEnumerate, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) // dummy output }); // This makes sure we use the right file system, which is aware of alien files dirEnumeratorBuilder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads; // Define the shared opaque dirEnumeratorBuilder.AddOutputDirectory(dirPath, global::BuildXL.Pips.Operations.SealDirectoryKind.SharedOpaque); // Make the enumerator depend on the creator dirEnumeratorBuilder.AddInputFile(dirCreator.ProcessOutputs.GetRequiredOutputFiles().Single()); var dirEnumerator = SchedulePipBuilder(dirEnumeratorBuilder); // Run once. The created directory should be ignored for the enumeration fingerprint computation. RunScheduler().AssertSuccess(); // Simulate shared opaque scrubbing FileUtilities.DeleteDirectoryContents(nestedDirPath.ToString(Context.PathTable), deleteRootDirectory: true); // This should be a cache hit. By replaying the dirCreator pip, the enumerated directory is re-created. However, // it should still be ignored by the fingerprint computation and we should get a cache hit. RunScheduler().AssertCacheHit(dirEnumerator.Process.PipId); }
/// <nodoc /> public LoggingConfiguration() { CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>(); CustomLogEtwKinds = new Dictionary <AbsolutePath, string>(); NoLog = new List <int>(); NoExecutionLog = new List <int>(); ConsoleVerbosity = VerbosityLevel.Informational; FileVerbosity = VerbosityLevel.Verbose; LogCounters = true; TraceInfo = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); Color = true; AnimateTaskbar = true; LogStats = true; LogExecution = true; FingerprintStoreMode = FingerprintStoreMode.Default; FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days FingerprintStoreBulkLoad = false; EngineCacheLogDirectory = AbsolutePath.Invalid; EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid; FingerprintsLogDirectory = AbsolutePath.Invalid; ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid; CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid; HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid; SubstSource = AbsolutePath.Invalid; SubstTarget = AbsolutePath.Invalid; FancyConsole = true; FancyConsoleMaxStatusPips = 5; LogStatus = true; LogTracer = true; FailPipOnFileAccessError = true; UseCustomPipDescriptionOnConsole = true; CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled(); CacheMissDiffFormat = CacheMissDiffFormat.CustomJsonDiff; AriaIndividualMessageSizeLimitBytes = (int)(0.8 * 1024 * 1024); // 0.8Mb out of Aria's current 1Mb max limit MaxNumPipTelemetryBatches = 1; CacheMissBatch = true; RedirectedLogsDirectory = AbsolutePath.Invalid; DumpFailedPips = true; DumpFailedPipsLogLimit = 50; DumpFailedPipsWithDynamicData = false; LogCachedPipOutputs = false; // Telemetry is enabled by default when the build has access to the Aria telemetry client. This is a Microsoft // internal only package so this corresponds to versions of BuildXL used internally within Microsoft. This define // controls whether we attempt to reference the telemetry utilities. It will fail at runtime if that client library // package is not available. #if FEATURE_ARIA_TELEMETRY RemoteTelemetry = Configuration.RemoteTelemetry.EnabledAndNotify; #endif }
/// <nodoc /> public LoggingConfiguration() { CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>(); CustomLogEtwKinds = new Dictionary <AbsolutePath, string>(); NoLog = new List <int>(); NoExecutionLog = new List <int>(); ConsoleVerbosity = VerbosityLevel.Informational; FileVerbosity = VerbosityLevel.Verbose; LogCounters = true; TraceInfo = new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase); Color = true; AnimateTaskbar = true; LogStats = true; LogExecution = true; FingerprintStoreMode = FingerprintStoreMode.Default; FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days FingerprintStoreBulkLoad = false; EngineCacheLogDirectory = AbsolutePath.Invalid; EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid; FingerprintsLogDirectory = AbsolutePath.Invalid; ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid; CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid; HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid; ReplayWarnings = true; SubstSource = AbsolutePath.Invalid; SubstTarget = AbsolutePath.Invalid; FancyConsole = true; FancyConsoleMaxStatusPips = 5; LogStatus = true; LogTracer = true; FailPipOnFileAccessError = true; UseCustomPipDescriptionOnConsole = true; CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled(); CacheMissDiffFormat = CacheMissDiffFormat.CustomJsonDiff; AriaIndividualMessageSizeLimitBytes = (int)(0.8 * 1024 * 1024); // 0.8Mb out of Aria's current 1Mb max limit MaxNumPipTelemetryBatches = 1; CacheMissBatch = true; RedirectedLogsDirectory = AbsolutePath.Invalid; DumpFailedPips = true; DumpFailedPipsLogLimit = 50; DumpFailedPipsWithDynamicData = false; LogCachedPipOutputs = false; }
/// <nodoc /> public LoggingConfiguration() { CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>(); CustomLogEtwKinds = new Dictionary <AbsolutePath, string>(); NoLog = new List <int>(); NoExecutionLog = new List <int>(); ConsoleVerbosity = VerbosityLevel.Informational; FileVerbosity = VerbosityLevel.Verbose; LogCounters = true; TraceInfo = new Dictionary <string, string>(); Color = true; AnimateTaskbar = true; LogStats = true; LogExecution = true; FingerprintStoreMode = FingerprintStoreMode.Default; FingerprintStoreMaxEntryAgeMinutes = 4320; // 3 days FingerprintStoreBulkLoad = false; EngineCacheLogDirectory = AbsolutePath.Invalid; EngineCacheCorruptFilesLogDirectory = AbsolutePath.Invalid; FingerprintsLogDirectory = AbsolutePath.Invalid; ExecutionFingerprintStoreLogDirectory = AbsolutePath.Invalid; CacheLookupFingerprintStoreLogDirectory = AbsolutePath.Invalid; HistoricMetadataCacheLogDirectory = AbsolutePath.Invalid; ReplayWarnings = true; SubstSource = AbsolutePath.Invalid; SubstTarget = AbsolutePath.Invalid; FancyConsole = true; FancyConsoleMaxStatusPips = 5; LogStatus = true; FailPipOnFileAccessError = true; UseCustomPipDescriptionOnConsole = true; CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled(); CacheMissDiffFormat = CacheMissDiffFormat.CustomJsonDiff; AriaIndividualMessageSizeLimitBytes = (int)(1.5 * 1024 * 1024); // 1.5Mb out of Aria's current 2.5Mb max limit MaxNumPipTelemetryBatches = 10; // Temporarily disable it by default due to crash. // TODO: Enable it by default once crash is fixed. CacheMissBatch = false; RedirectedLogsDirectory = AbsolutePath.Invalid; }
public void CreatedDirectoriesUnderSharedOpaquesAreNotPartOfTheFingerprint() { string dir = Path.Combine(SourceRoot, "dir"); AbsolutePath dirPath = AbsolutePath.Create(Context.PathTable, dir); DirectoryArtifact dirToEnumerate = DirectoryArtifact.CreateWithZeroPartialSealId(dirPath); Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.LocalMode(); AbsolutePath nestedDirPath = dirPath.Combine(Context.PathTable, "nested"); var nestedDir = DirectoryArtifact.CreateWithZeroPartialSealId(nestedDirPath); var operations = new List <Operation> { // Create a directory nested into the one that is going to be enumerated Operation.CreateDir(nestedDir, doNotInfer: true), Operation.EnumerateDir(dirToEnumerate, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) // dummy output }; var builder = CreatePipBuilder(operations); // This makes sure we use the right file system, which is aware of alien files builder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads; // Define the shared opaque builder.AddOutputDirectory(dirPath, global::BuildXL.Pips.Operations.SealDirectoryKind.SharedOpaque); var pip = SchedulePipBuilder(builder); // Run once RunScheduler().AssertSuccess(); // Simulate shared opaque scrubbing FileUtilities.DeleteDirectoryContents(nestedDirPath.ToString(Context.PathTable), deleteRootDirectory: true); // This should be a cache hit. Directories created by a pip are not part of the fingerprint RunScheduler().AssertCacheHit(pip.Process.PipId); }
/// <nodoc /> public LoggingConfiguration(ILoggingConfiguration template, PathRemapper pathRemapper) : base(template) { Contract.Assume(template != null); Contract.Assume(pathRemapper != null); LogsDirectory = pathRemapper.Remap(template.LogsDirectory); RedirectedLogsDirectory = pathRemapper.Remap(template.RedirectedLogsDirectory); LogPrefix = template.LogPrefix; Log = pathRemapper.Remap(template.Log); ErrorLog = pathRemapper.Remap(template.ErrorLog); WarningLog = pathRemapper.Remap(template.WarningLog); LogExecution = template.LogExecution; ExecutionLog = pathRemapper.Remap(template.ExecutionLog); StoreFingerprints = template.StoreFingerprints; FingerprintStoreMode = template.FingerprintStoreMode; FingerprintStoreMaxEntryAgeMinutes = template.FingerprintStoreMaxEntryAgeMinutes; FingerprintsLogDirectory = pathRemapper.Remap(template.FingerprintsLogDirectory); ExecutionFingerprintStoreLogDirectory = pathRemapper.Remap(template.ExecutionFingerprintStoreLogDirectory); CacheLookupFingerprintStoreLogDirectory = pathRemapper.Remap(template.CacheLookupFingerprintStoreLogDirectory); HistoricMetadataCacheLogDirectory = pathRemapper.Remap(template.HistoricMetadataCacheLogDirectory); EngineCacheLogDirectory = pathRemapper.Remap(template.EngineCacheLogDirectory); EngineCacheCorruptFilesLogDirectory = pathRemapper.Remap(template.EngineCacheCorruptFilesLogDirectory); CustomLog = new Dictionary <AbsolutePath, IReadOnlyList <int> >(); foreach (var kv in template.CustomLog) { CustomLog.Add(pathRemapper.Remap(kv.Key), kv.Value); } CustomLogEtwKinds = new Dictionary <AbsolutePath, string>(); foreach (var kv in template.CustomLogEtwKinds) { CustomLogEtwKinds.Add(pathRemapper.Remap(kv.Key), kv.Value); } NoLog = new List <int>(template.NoLog); NoExecutionLog = new List <int>(template.NoExecutionLog); Diagnostic = template.Diagnostic; ConsoleVerbosity = template.ConsoleVerbosity; FileVerbosity = template.FileVerbosity; LogCounters = template.LogCounters; LogStats = template.LogStats; EnableAsyncLogging = template.EnableAsyncLogging; StatsLog = pathRemapper.Remap(template.StatsLog); EventSummaryLog = pathRemapper.Remap(template.EventSummaryLog); Environment = template.Environment; RemoteTelemetry = template.RemoteTelemetry; TraceInfo = new Dictionary <string, string>(); foreach (var kv in template.TraceInfo) { TraceInfo.Add(kv.Key, kv.Value); } Color = template.Color; AnimateTaskbar = template.AnimateTaskbar; RelatedActivityId = template.RelatedActivityId; LogsToRetain = template.LogsToRetain; FancyConsole = template.FancyConsole; FancyConsoleMaxStatusPips = template.FancyConsoleMaxStatusPips; SubstSource = pathRemapper.Remap(template.SubstSource); SubstTarget = pathRemapper.Remap(template.SubstTarget); DisableLoggedPathTranslation = template.DisableLoggedPathTranslation; LogStatus = template.LogStatus; StatusFrequencyMs = template.StatusFrequencyMs; StatusLog = pathRemapper.Remap(template.StatusLog); CacheMissLog = pathRemapper.Remap(template.CacheMissLog); DevLog = pathRemapper.Remap(template.DevLog); RpcLog = pathRemapper.Remap(template.RpcLog); PipOutputLog = pathRemapper.Remap(template.PipOutputLog); FailPipOnFileAccessError = template.FailPipOnFileAccessError; LogMemory = template.LogMemory; ReplayWarnings = template.ReplayWarnings; UseCustomPipDescriptionOnConsole = template.UseCustomPipDescriptionOnConsole; CacheMissAnalysisOption = new CacheMissAnalysisOption( template.CacheMissAnalysisOption.Mode, new List <string>(template.CacheMissAnalysisOption.Keys), pathRemapper.Remap(template.CacheMissAnalysisOption.CustomPath)); }
private void DisableFingerprintStore() { Configuration.Logging.StoreFingerprints = false; Configuration.Logging.FingerprintStoreMode = FingerprintStoreMode.Invalid; Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.Disabled(); }
private void EnableFingerprintStore() { Configuration.Logging.StoreFingerprints = true; Configuration.Logging.FingerprintStoreMode = FingerprintStoreMode.ExecutionFingerprintsOnly; Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.LocalMode(); }
/// <nodoc /> public LoggingConfiguration(ILoggingConfiguration template, PathRemapper pathRemapper) : base(template) { Contract.Assume(template != null); Contract.Assume(pathRemapper != null); LogsDirectory = pathRemapper.Remap(template.LogsDirectory); RedirectedLogsDirectory = pathRemapper.Remap(template.RedirectedLogsDirectory); LogPrefix = template.LogPrefix; Log = pathRemapper.Remap(template.Log); ErrorLog = pathRemapper.Remap(template.ErrorLog); WarningLog = pathRemapper.Remap(template.WarningLog); LogExecution = template.LogExecution; LogPackedExecution = template.LogPackedExecution; ExecutionLog = pathRemapper.Remap(template.ExecutionLog); StoreFingerprints = template.StoreFingerprints; SaveFingerprintStoreToLogs = template.SaveFingerprintStoreToLogs; FingerprintStoreMode = template.FingerprintStoreMode; FingerprintStoreMaxEntryAgeMinutes = template.FingerprintStoreMaxEntryAgeMinutes; FingerprintStoreBulkLoad = template.FingerprintStoreBulkLoad; FingerprintsLogDirectory = pathRemapper.Remap(template.FingerprintsLogDirectory); ExecutionFingerprintStoreLogDirectory = pathRemapper.Remap(template.ExecutionFingerprintStoreLogDirectory); CacheLookupFingerprintStoreLogDirectory = pathRemapper.Remap(template.CacheLookupFingerprintStoreLogDirectory); HistoricMetadataCacheLogDirectory = pathRemapper.Remap(template.HistoricMetadataCacheLogDirectory); EngineCacheLogDirectory = pathRemapper.Remap(template.EngineCacheLogDirectory); EngineCacheCorruptFilesLogDirectory = pathRemapper.Remap(template.EngineCacheCorruptFilesLogDirectory); CustomLog = new Dictionary <AbsolutePath, (IReadOnlyList <int>, EventLevel?)>(); foreach (var kv in template.CustomLog) { CustomLog.Add(pathRemapper.Remap(kv.Key), kv.Value); } CustomLogEtwKinds = new Dictionary <AbsolutePath, string>(); foreach (var kv in template.CustomLogEtwKinds) { CustomLogEtwKinds.Add(pathRemapper.Remap(kv.Key), kv.Value); } NoLog = new List <int>(template.NoLog); NoExecutionLog = new List <int>(template.NoExecutionLog); Diagnostic = template.Diagnostic; ConsoleVerbosity = template.ConsoleVerbosity; FileVerbosity = template.FileVerbosity; LogCounters = template.LogCounters; LogStats = template.LogStats; EnableAsyncLogging = template.EnableAsyncLogging; StatsLog = pathRemapper.Remap(template.StatsLog); StatsPrfLog = pathRemapper.Remap(template.StatsPrfLog); EventSummaryLog = pathRemapper.Remap(template.EventSummaryLog); Environment = template.Environment; RemoteTelemetry = template.RemoteTelemetry; TraceInfo = new Dictionary <string, string>(); foreach (var kv in template.TraceInfo) { TraceInfo.Add(kv.Key, kv.Value); } Color = template.Color; AnimateTaskbar = template.AnimateTaskbar; RelatedActivityId = template.RelatedActivityId; LogsToRetain = template.LogsToRetain; FancyConsole = template.FancyConsole; FancyConsoleMaxStatusPips = template.FancyConsoleMaxStatusPips; SubstSource = pathRemapper.Remap(template.SubstSource); SubstTarget = pathRemapper.Remap(template.SubstTarget); DisableLoggedPathTranslation = template.DisableLoggedPathTranslation; LogStatus = template.LogStatus; LogTracer = template.LogTracer; StatusFrequencyMs = template.StatusFrequencyMs; StatusLog = pathRemapper.Remap(template.StatusLog); TraceLog = pathRemapper.Remap(template.TraceLog); CacheMissLog = pathRemapper.Remap(template.CacheMissLog); DevLog = pathRemapper.Remap(template.DevLog); RpcLog = pathRemapper.Remap(template.RpcLog); PipOutputLog = pathRemapper.Remap(template.PipOutputLog); FailPipOnFileAccessError = template.FailPipOnFileAccessError; LogMemory = template.LogMemory; ReplayWarnings = template.ReplayWarnings; UseCustomPipDescriptionOnConsole = template.UseCustomPipDescriptionOnConsole; CacheMissAnalysisOption = new CacheMissAnalysisOption( template.CacheMissAnalysisOption.Mode, new List <string>(template.CacheMissAnalysisOption.Keys), pathRemapper.Remap(template.CacheMissAnalysisOption.CustomPath)); CacheMissDiffFormat = template.CacheMissDiffFormat; CacheMissBatch = template.CacheMissBatch; OptimizeConsoleOutputForAzureDevOps = template.OptimizeConsoleOutputForAzureDevOps; InvocationExpandedCommandLineArguments = template.InvocationExpandedCommandLineArguments; OptimizeProgressUpdatingForAzureDevOps = template.OptimizeProgressUpdatingForAzureDevOps; OptimizeVsoAnnotationsForAzureDevOps = template.OptimizeVsoAnnotationsForAzureDevOps; AriaIndividualMessageSizeLimitBytes = template.AriaIndividualMessageSizeLimitBytes; MaxNumPipTelemetryBatches = template.MaxNumPipTelemetryBatches; DumpFailedPips = template.DumpFailedPips; DumpFailedPipsLogLimit = template.DumpFailedPipsLogLimit; DumpFailedPipsWithDynamicData = template.DumpFailedPipsWithDynamicData; LogCachedPipOutputs = template.LogCachedPipOutputs; }