/// <summary> /// Calls the implementation of UnsynchronizedFlush within the lock used for writing events to ensure no other /// writes occur in conjunction with flush /// </summary> protected void SynchronizedFlush() { lock (m_lock) { if (m_disabledDueToDiskWriteFailure) { return; } try { UnsynchronizedFlush(); } catch (Exception ex) { // See identical handling in OnEventWritten. if (m_disabledDueToDiskWriteFailureEventHandler != null && ExceptionUtilities.AnalyzeExceptionRootCause(ex) == ExceptionRootCause.OutOfDiskSpace) { m_disabledDueToDiskWriteFailure = true; OnDisabledDueToDiskWriteFailure(); } else { throw; } } } }
public void ClassifyMissingRuntimeDependencyTest() { Assert.Equal(ExceptionRootCause.MissingRuntimeDependency, ExceptionUtilities.AnalyzeExceptionRootCause(new FileLoadException())); Assert.Equal(ExceptionRootCause.MissingRuntimeDependency, ExceptionUtilities.AnalyzeExceptionRootCause(new FileNotFoundException("Could not load file or assembly"))); Assert.Equal(ExceptionRootCause.MissingRuntimeDependency, ExceptionUtilities.AnalyzeExceptionRootCause(new DllNotFoundException())); Assert.Equal(ExceptionRootCause.MissingRuntimeDependency, ExceptionUtilities.AnalyzeExceptionRootCause(new TypeLoadException())); }
private void HandleUnhandledFailure(LoggingContext loggingContext, Exception exception) { if (Interlocked.CompareExchange(ref m_handlingUnhandledFailureInProgress, 1, comparand: 0) != 0) { Thread.Sleep(TimeSpan.FromSeconds(3)); ExceptionUtilities.FailFast("Second-chance exception handler has not completed in the allowed time.", new InvalidOperationException()); return; } try { GeneratorExitCode effectiveExitCode = GeneratorExitCode.InternalError; ExceptionRootCause rootCause = exception is NullReferenceException ? ExceptionRootCause.FailFast : ExceptionUtilities.AnalyzeExceptionRootCause(exception); switch (rootCause) { case ExceptionRootCause.OutOfDiskSpace: case ExceptionRootCause.DataErrorDriveFailure: case ExceptionRootCause.DeviceAccessError: effectiveExitCode = GeneratorExitCode.InfrastructureError; break; case ExceptionRootCause.MissingRuntimeDependency: effectiveExitCode = GeneratorExitCode.MissingRuntimeDependency; break; } string failureMessage = exception.ToStringDemystified(); if (effectiveExitCode == GeneratorExitCode.InfrastructureError) { Logger.Log.UnhandledInfrastructureError(loggingContext, failureMessage); } else { Logger.Log.UnhandledFailure(loggingContext, failureMessage); } if (rootCause == ExceptionRootCause.FailFast) { ExceptionUtilities.FailFast("Exception is configured to fail fast", exception); } Environment.Exit((int)effectiveExitCode); } catch (Exception e) { PrintErrorToConsole("Unhandled exception in exception handler"); PrintErrorToConsole(e.ToStringDemystified()); } finally { Environment.Exit((int)GeneratorExitCode.InternalError); } }
/// <summary> /// Constructor /// </summary> public EngineSerializer( LoggingContext loggingContext, string engineCacheLocation, FileEnvelopeId?correlationId = null, bool useCompression = false, bool debug = false, bool readOnly = false, FileSystemStreamProvider readStreamProvider = null, ITempCleaner tempDirectoryCleaner = null) { Contract.Requires(loggingContext != null); Contract.Requires(engineCacheLocation != null); Contract.Requires(Path.IsPathRooted(engineCacheLocation)); Contract.Requires(!string.IsNullOrWhiteSpace(engineCacheLocation)); LoggingContext = loggingContext; m_engineCacheLocation = engineCacheLocation; m_debug = debug; m_correlationId = correlationId; m_useCompression = useCompression; m_readStreamProvider = readStreamProvider ?? FileSystemStreamProvider.Default; m_tempDirectoryCleaner = tempDirectoryCleaner; if (!readOnly) { try { FileUtilities.CreateDirectoryWithRetry(engineCacheLocation); } catch (Exception ex) { ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex); BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause); throw new BuildXLException("Unable to create engine serializer cache directory: ", ex); } } }
/// <summary> /// Creates and starts a task to deserialize an object /// </summary> /// <param name="file">This will become the filename</param> /// <param name="deserializer">Deserialization function; its get a reader for the file stream, and a function that allows obtaining additional streams if needed</param> /// <param name="skipHeader">If enabled, the correlation id is not checked for consistency</param> /// <returns>task for deserialized value</returns> internal Task <TObject> DeserializeFromFileAsync <TObject>( GraphCacheFile file, Func <BuildXLReader, Task <TObject> > deserializer, bool skipHeader = false) { var task = Task.Run( async() => { var objectLabel = GetFileName(file); string path = GetFullPath(objectLabel); FileEnvelope fileEnvelope = GetFileEnvelope(file); var result = default(TObject); try { Stopwatch sw = Stopwatch.StartNew(); using (var fileStreamWrapper = m_readStreamProvider.OpenReadStream(path)) { var fileStream = fileStreamWrapper.Value; FileEnvelopeId persistedCorrelationId = fileEnvelope.ReadHeader(fileStream); if (!skipHeader) { // We are going to check if all files that are going to be (concurrently) deserialized have matching correlation ids. // The first discovered correlation id is going to be used to check all others. if (m_correlationId == null) { Interlocked.CompareExchange(ref m_correlationId, persistedCorrelationId, null); } FileEnvelope.CheckCorrelationIds(persistedCorrelationId, (FileEnvelopeId)m_correlationId); } var isCompressed = fileStream.ReadByte() == 1; using (Stream readStream = isCompressed ? new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionMode.Decompress), 64 << 10)) : fileStream) using (BuildXLReader reader = new BuildXLReader(m_debug, readStream, leaveOpen: false)) { result = await deserializer(reader); } } Tracing.Logger.Log.DeserializedFile(LoggingContext, path, sw.ElapsedMilliseconds); return(result); } catch (BuildXLException ex) { if (ex.InnerException is FileNotFoundException) { // Files might be deleted manually in the EngineCache directory. Log it as verbose. Tracing.Logger.Log.FailedToDeserializeDueToFileNotFound(LoggingContext, path); return(result); } Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.LogEventMessage); return(result); } catch (IOException ex) { Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } catch (TaskCanceledException) { throw; } catch (Exception ex) { // There are 2 reasons to be here. // 1. A malformed file can cause ContractException, IndexOutOfRangeException, MemoryException or something else. // 2. We may have a bug. // Since the malformed file will always cause a crash until someone removes the file from the cache, allow BuildXL to recover // by eating the exception. However remember to log it in order to keep track of bugs. ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex); BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause); Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } }); lock (m_deserializationSyncObject) { m_deserializationTasks.Add(task); } return(task); }
protected override void OnEventWritten(EventWrittenEventArgs eventData) { // Ensure this event which routes text log messages to ETW is not handled by // event listeners if (eventData.EventId == (int)EventId.TextLogEtwOnly) { return; } // If diagnostic events are enabled, we may see events not matching the NormalKeywords. // In that case we further filter based on the event's Task. See EnableTaskDiagnostics. // Keywords == 0 acts like a wildcard (all keywords); see EventSource sources. For some reason EventSource sets some top bits. if (unchecked ((uint)eventData.Keywords) != 0) { if ((eventData.Keywords & NormalKeywords) == 0 && (eventData.Keywords & DiagnosticsKeywords) != 0 && (eventData.Task > BuildXL.Utilities.Instrumentation.Common.Tasks.Max || !m_enableTaskDiagnostics[unchecked ((int)eventData.Task)])) { return; } if ((eventData.Keywords & Keywords.SelectivelyEnabled) != 0 && (m_eventMask?.IsSelectivelyEnabled != true)) { // Exclude any event which is selectively enabled if this event listener does // not selectively enable events (i.e. specify specific event ids which should not be // excluded) return; } } // Out-of-band messages from EventListener.ReportOutOfBandMessage end up with event ID 0. // We don't allow these messages to be upgraded to errors by the WarningMapper since they typically // are not actionable. bool isOutOfBand = eventData.EventId == 0; EventLevel level = isOutOfBand ? EventLevel.Warning : eventData.Level; bool suppressedWarning = false; if (level == EventLevel.Warning && !isOutOfBand) { if (m_warningMapper != null) { switch (m_warningMapper(eventData.EventId)) { case WarningState.AsError: { level = EventLevel.Error; break; } case WarningState.AsWarning: { // nothing to do break; } case WarningState.Suppressed: { suppressedWarning = true; break; } } } } // Bail out if the event is not enabled based on the event mask if (m_eventMask != null && !m_eventMask.IsEnabled(level, eventData.EventId)) { return; } // Derived listeners don't need to worry about locking, we do it here... lock (m_lock) { try { if (m_disabledDueToDiskWriteFailure) { return; } if (m_limitToCriticalLevelOnly && level != EventLevel.Critical) { return; } // dispatch to the appropriate handler method switch (level) { case EventLevel.Critical: { OnCritical(eventData); break; } case EventLevel.Error: { OnError(eventData); break; } case EventLevel.Informational: { OnInformational(eventData); break; } case EventLevel.LogAlways: { OnAlways(eventData); break; } case EventLevel.Verbose: { OnVerbose(eventData); break; } default: { Contract.Assert(level == EventLevel.Warning); if (isOutOfBand) { OnEventSourceInternalWarning(eventData); } else { if (!suppressedWarning) { OnWarning(eventData); } else { OnSuppressedWarning(eventData); } } break; } } } catch (Exception ex) { // Event listeners have an unfortunate distinction of being very useful for reporting terminal failures (i.e., in an unhandled exception handler). // So, we need graceful failure in the event of systemic inability to write to some listeners (e.g. out of disk space): The unhandled exception handler // should, in that circumstance, be able to write to any not-yet-broken listeners. if (m_disabledDueToDiskWriteFailureEventHandler != null && ExceptionUtilities.AnalyzeExceptionRootCause(ex) == ExceptionRootCause.OutOfDiskSpace) { m_disabledDueToDiskWriteFailure = true; OnDisabledDueToDiskWriteFailure(); } else { throw; } } } }
public void ClassifyOutOfDiskSpace() { Assert.Equal(ExceptionRootCause.OutOfDiskSpace, ExceptionUtilities.AnalyzeExceptionRootCause(new IOException("No space left on device"))); }