/// <summary> /// Marks for failure. /// </summary> /// <param name="exception">Exception that causes the failure.</param> /// <param name="rootCause"><see cref="ExceptionRootCause"/></param> /// <returns>True if all markings were successful.</returns> public bool TryMarkFailure(Exception exception, ExceptionRootCause rootCause) { bool success = true; foreach (var action in m_actions) { if (action.ShouldMarkFailure(exception, rootCause)) { var possibleResult = action.TryMarkFailure(exception, rootCause); if (!possibleResult.Succeeded) { Logger.Log.FailedToMarkFailure( m_loggingContext, action.Name, possibleResult.Failure.DescribeIncludingInnerFailures()); success = false; } else { Logger.Log.SuccessfulMarkFailure(m_loggingContext, action.Name); } } } return(success); }
private void HandleUnhandledFailure(LoggingContext loggingContext, Exception exception) { if (Interlocked.CompareExchange(ref m_handlingUnhandledFailureInProgress, 1, comparand: 0) != 0) { Thread.Sleep(TimeSpan.FromSeconds(3)); ExceptionUtilities.FailFast("Second-chance exception handler has not completed in the allowed time.", new InvalidOperationException()); return; } try { GeneratorExitCode effectiveExitCode = GeneratorExitCode.InternalError; ExceptionRootCause rootCause = exception is NullReferenceException ? ExceptionRootCause.FailFast : ExceptionUtilities.AnalyzeExceptionRootCause(exception); switch (rootCause) { case ExceptionRootCause.OutOfDiskSpace: case ExceptionRootCause.DataErrorDriveFailure: case ExceptionRootCause.DeviceAccessError: effectiveExitCode = GeneratorExitCode.InfrastructureError; break; case ExceptionRootCause.MissingRuntimeDependency: effectiveExitCode = GeneratorExitCode.MissingRuntimeDependency; break; } string failureMessage = exception.ToStringDemystified(); if (effectiveExitCode == GeneratorExitCode.InfrastructureError) { Logger.Log.UnhandledInfrastructureError(loggingContext, failureMessage); } else { Logger.Log.UnhandledFailure(loggingContext, failureMessage); } if (rootCause == ExceptionRootCause.FailFast) { ExceptionUtilities.FailFast("Exception is configured to fail fast", exception); } Environment.Exit((int)effectiveExitCode); } catch (Exception e) { PrintErrorToConsole("Unhandled exception in exception handler"); PrintErrorToConsole(e.ToStringDemystified()); } finally { Environment.Exit((int)GeneratorExitCode.InternalError); } }
/// <summary> /// Tries to mark failure if the condition for marking failure (<see cref="ShouldMarkFailure(Exception, ExceptionRootCause)"/>) is satisfied. /// </summary> /// <param name="exception">Exception that causes the failure.</param> /// <param name="rootCause"><see cref="ExceptionRootCause"/></param> /// <returns>Succeeds if marking was successful.</returns> public Possible <Unit> TryMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause) { if (!ShouldMarkFailure(exception, rootCause)) { return(Unit.Void); } return(MarkFailure(exception)); }
public abstract void DominoCatastrophicFailure(LoggingContext context, string exception, BuildInfo buildInfo, ExceptionRootCause rootCause, bool wasServer, string firstUserError, string lastUserError, string firstInsfrastructureError, string lastInfrastructureError, string firstInternalError, string lastInternalError);
/// <summary> /// Creates a BuildXLException with the given diagnostic message that was caused by the given /// <paramref name="innerException" />. If a <paramref name="rootCause"/> is not provided, the root cause /// is derived from the inner exception. /// </summary> public BuildXLException([Localizable(false)] string message, Exception innerException, ExceptionRootCause rootCause = ExceptionRootCause.Unknown) : base(message, innerException) { if (rootCause != ExceptionRootCause.Unknown) { m_rootCause = rootCause; } else if (innerException != null) { m_rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(innerException); } else { m_rootCause = ExceptionRootCause.Unknown; } }
/// <summary> /// Constructor /// </summary> public EngineSerializer( LoggingContext loggingContext, string engineCacheLocation, FileEnvelopeId?correlationId = null, bool useCompression = false, bool debug = false, bool readOnly = false, FileSystemStreamProvider readStreamProvider = null, ITempCleaner tempDirectoryCleaner = null) { Contract.Requires(loggingContext != null); Contract.Requires(engineCacheLocation != null); Contract.Requires(Path.IsPathRooted(engineCacheLocation)); Contract.Requires(!string.IsNullOrWhiteSpace(engineCacheLocation)); LoggingContext = loggingContext; m_engineCacheLocation = engineCacheLocation; m_debug = debug; m_correlationId = correlationId; m_useCompression = useCompression; m_readStreamProvider = readStreamProvider ?? FileSystemStreamProvider.Default; m_tempDirectoryCleaner = tempDirectoryCleaner; if (!readOnly) { try { FileUtilities.CreateDirectoryWithRetry(engineCacheLocation); } catch (Exception ex) { ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex); BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause); throw new BuildXLException("Unable to create engine serializer cache directory: ", ex); } } }
/// <inheritdoc /> public override bool ShouldMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause) { return(rootCause == ExceptionRootCause.CorruptedCache); }
/// <summary> /// Creates and starts a task to deserialize an object /// </summary> /// <param name="file">This will become the filename</param> /// <param name="deserializer">Deserialization function; its get a reader for the file stream, and a function that allows obtaining additional streams if needed</param> /// <param name="skipHeader">If enabled, the correlation id is not checked for consistency</param> /// <returns>task for deserialized value</returns> internal Task <TObject> DeserializeFromFileAsync <TObject>( GraphCacheFile file, Func <BuildXLReader, Task <TObject> > deserializer, bool skipHeader = false) { var task = Task.Run( async() => { var objectLabel = GetFileName(file); string path = GetFullPath(objectLabel); FileEnvelope fileEnvelope = GetFileEnvelope(file); var result = default(TObject); try { Stopwatch sw = Stopwatch.StartNew(); using (var fileStreamWrapper = m_readStreamProvider.OpenReadStream(path)) { var fileStream = fileStreamWrapper.Value; FileEnvelopeId persistedCorrelationId = fileEnvelope.ReadHeader(fileStream); if (!skipHeader) { // We are going to check if all files that are going to be (concurrently) deserialized have matching correlation ids. // The first discovered correlation id is going to be used to check all others. if (m_correlationId == null) { Interlocked.CompareExchange(ref m_correlationId, persistedCorrelationId, null); } FileEnvelope.CheckCorrelationIds(persistedCorrelationId, (FileEnvelopeId)m_correlationId); } var isCompressed = fileStream.ReadByte() == 1; using (Stream readStream = isCompressed ? new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionMode.Decompress), 64 << 10)) : fileStream) using (BuildXLReader reader = new BuildXLReader(m_debug, readStream, leaveOpen: false)) { result = await deserializer(reader); } } Tracing.Logger.Log.DeserializedFile(LoggingContext, path, sw.ElapsedMilliseconds); return(result); } catch (BuildXLException ex) { if (ex.InnerException is FileNotFoundException) { // Files might be deleted manually in the EngineCache directory. Log it as verbose. Tracing.Logger.Log.FailedToDeserializeDueToFileNotFound(LoggingContext, path); return(result); } Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.LogEventMessage); return(result); } catch (IOException ex) { Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } catch (TaskCanceledException) { throw; } catch (Exception ex) { // There are 2 reasons to be here. // 1. A malformed file can cause ContractException, IndexOutOfRangeException, MemoryException or something else. // 2. We may have a bug. // Since the malformed file will always cause a crash until someone removes the file from the cache, allow BuildXL to recover // by eating the exception. However remember to log it in order to keep track of bugs. ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex); BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause); Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message); return(result); } }); lock (m_deserializationSyncObject) { m_deserializationTasks.Add(task); } return(task); }
/// <summary> /// Attempts to assign a root cause to an arbitrary exception. /// </summary> /// <remarks> /// The analysis strategy is so: /// - Walking from the outermost to innermost exception, find the first root cause /// (i.e., first that is not <see cref="ExceptionRootCause.Unknown"/>) /// - Respect <see cref="BuildXLException.RootCause"/> (we allow the enlightened creator /// of the exception to provide a root cause). /// - For everything else, try to match patterns of exception type, HRESULT, etc. /// </remarks> public static ExceptionRootCause AnalyzeExceptionRootCause(Exception ex) { Contract.RequiresNotNull(ex); const int OutOfDiskSpaceHResult = unchecked ((int)0x80070070); const int DataErrorCRCResult = unchecked ((int)0x80070017); const int DataErrorSeek = unchecked ((int)0x80070019); const int DataErrorSector = unchecked ((int)0x8007001B); const int DataErrorWriteFault = unchecked ((int)0x8007001D); const int DataErrorReadFault = unchecked ((int)0x8007001E); const int DataErrorGeneralFault = unchecked ((int)0x8007001F); const int NoSystemResourcesFault = unchecked ((int)0x800705AA); const int PipeNotConnectedFault = unchecked ((int)0x800700E9); const int NotEnoughStorage = unchecked ((int)0x8); const int IncorrectFunction = unchecked ((int)0x00000001); var buildXLException = ex as BuildXLException; if (buildXLException != null) { return(buildXLException.RootCause); } if (ex is OutOfMemoryException) { return(ExceptionRootCause.OutOfMemory); } if (ex is EventSourceException evSrcException && evSrcException.InnerException is BuildXLException innerBuldXLException) { return(innerBuldXLException.RootCause); } int exHResult = GetHResult(ex); if (ex is IOException && exHResult == IncorrectFunction) { return(ExceptionRootCause.DeviceAccessError); } if (ex is IOException && ex.Message.Contains("No space left on device")) { return(ExceptionRootCause.OutOfDiskSpace); } Win32Exception win32Ex = ex as Win32Exception; if (exHResult == OutOfDiskSpaceHResult || (win32Ex != null && win32Ex.ErrorCode == NotEnoughStorage)) { return(ExceptionRootCause.OutOfDiskSpace); } if (exHResult == DataErrorCRCResult || exHResult == DataErrorSeek || exHResult == DataErrorSector || exHResult == DataErrorWriteFault || exHResult == DataErrorReadFault || exHResult == DataErrorGeneralFault) { return(ExceptionRootCause.DataErrorDriveFailure); } if (exHResult == NoSystemResourcesFault) { return(ExceptionRootCause.NoSystemResources); } if (ex is FileLoadException || (ex is FileNotFoundException && ex.Message.Contains(Strings.ExceptionUtilities_MissingDependencyPattern)) || ex is DllNotFoundException || ex is TypeLoadException) { return(ExceptionRootCause.MissingRuntimeDependency); } if (exHResult == PipeNotConnectedFault) { // Verify that the failure is caused by the console. try { Console.Out.Flush(); Console.Error.Flush(); } catch (IOException) { return(ExceptionRootCause.ConsoleNotConnected); } } var aggregateException = ex as AggregateException; if (aggregateException != null) { // Aggregate exceptions may have multiple inner exceptions. We take the first root cause we find // (rather than trying to faithfully represent multiple root causes). aggregateException = aggregateException.Flatten(); foreach (Exception inner in aggregateException.InnerExceptions) { ExceptionRootCause innerRootCause = AnalyzeExceptionRootCause(inner); if (innerRootCause != ExceptionRootCause.Unknown) { return(innerRootCause); } } } else if (ex.InnerException != null) { return(AnalyzeExceptionRootCause(ex.InnerException)); } return(ExceptionRootCause.Unknown); }
/// <summary> /// Checks if failure should be marked. /// </summary> /// <param name="exception">Exception that causes the failure.</param> /// <param name="rootCause"><see cref="ExceptionRootCause"/></param> /// <returns>True iff failure should be marked.</returns> public abstract bool ShouldMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause);
/// <summary> /// Only run handling for failures of unknown root cause. Many of the known root causes are unrelated to the build /// and cannot be handled during the build. /// </summary> public override bool ShouldMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause) { return(rootCause == ExceptionRootCause.Unknown); }
private BuildXLException(SerializationInfo info, StreamingContext context) : base(info, context) { m_rootCause = (ExceptionRootCause)info.GetInt32(ExceptionRootCauseName); }
#pragma warning restore CA2235 // Mark all non-serializable fields /// <summary> /// Creates a BuildXLException with the given diagnostic message, without any inner exception as the root cause. /// Instead, a well-known <see cref="ExceptionRootCause"/> may be provided. /// </summary> public BuildXLException([Localizable(false)] string message, ExceptionRootCause rootCause = ExceptionRootCause.Unknown) : base(message) { m_rootCause = rootCause; }