/// <summary>
/// Marks for failure.
/// </summary>
/// <param name="exception">Exception that causes the failure.</param>
/// <param name="rootCause"><see cref="ExceptionRootCause"/></param>
/// <returns>True if all markings were successful.</returns>
public bool TryMarkFailure(Exception exception, ExceptionRootCause rootCause)
{
bool success = true;
foreach (var action in m_actions)
{
if (action.ShouldMarkFailure(exception, rootCause))
{
var possibleResult = action.TryMarkFailure(exception, rootCause);
if (!possibleResult.Succeeded)
{
Logger.Log.FailedToMarkFailure(
m_loggingContext,
action.Name,
possibleResult.Failure.DescribeIncludingInnerFailures());
success = false;
}
else
{
Logger.Log.SuccessfulMarkFailure(m_loggingContext, action.Name);
}
}
}
return(success);
}
private void HandleUnhandledFailure(LoggingContext loggingContext, Exception exception)
{
if (Interlocked.CompareExchange(ref m_handlingUnhandledFailureInProgress, 1, comparand: 0) != 0)
{
Thread.Sleep(TimeSpan.FromSeconds(3));
ExceptionUtilities.FailFast("Second-chance exception handler has not completed in the allowed time.", new InvalidOperationException());
return;
}
try
{
GeneratorExitCode effectiveExitCode = GeneratorExitCode.InternalError;
ExceptionRootCause rootCause = exception is NullReferenceException
? ExceptionRootCause.FailFast
: ExceptionUtilities.AnalyzeExceptionRootCause(exception);
switch (rootCause)
{
case ExceptionRootCause.OutOfDiskSpace:
case ExceptionRootCause.DataErrorDriveFailure:
case ExceptionRootCause.DeviceAccessError:
effectiveExitCode = GeneratorExitCode.InfrastructureError;
break;
case ExceptionRootCause.MissingRuntimeDependency:
effectiveExitCode = GeneratorExitCode.MissingRuntimeDependency;
break;
}
string failureMessage = exception.ToStringDemystified();
if (effectiveExitCode == GeneratorExitCode.InfrastructureError)
{
Logger.Log.UnhandledInfrastructureError(loggingContext, failureMessage);
}
else
{
Logger.Log.UnhandledFailure(loggingContext, failureMessage);
}
if (rootCause == ExceptionRootCause.FailFast)
{
ExceptionUtilities.FailFast("Exception is configured to fail fast", exception);
}
Environment.Exit((int)effectiveExitCode);
}
catch (Exception e)
{
PrintErrorToConsole("Unhandled exception in exception handler");
PrintErrorToConsole(e.ToStringDemystified());
}
finally
{
Environment.Exit((int)GeneratorExitCode.InternalError);
}
}
/// <summary>
/// Tries to mark failure if the condition for marking failure (<see cref="ShouldMarkFailure(Exception, ExceptionRootCause)"/>) is satisfied.
/// </summary>
/// <param name="exception">Exception that causes the failure.</param>
/// <param name="rootCause"><see cref="ExceptionRootCause"/></param>
/// <returns>Succeeds if marking was successful.</returns>
public Possible <Unit> TryMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause)
{
if (!ShouldMarkFailure(exception, rootCause))
{
return(Unit.Void);
}
return(MarkFailure(exception));
}
public abstract void DominoCatastrophicFailure(LoggingContext context,
string exception,
BuildInfo buildInfo,
ExceptionRootCause rootCause,
bool wasServer,
string firstUserError,
string lastUserError,
string firstInsfrastructureError,
string lastInfrastructureError,
string firstInternalError,
string lastInternalError);
/// <summary>
/// Creates a BuildXLException with the given diagnostic message that was caused by the given
/// <paramref name="innerException" />. If a <paramref name="rootCause"/> is not provided, the root cause
/// is derived from the inner exception.
/// </summary>
public BuildXLException([Localizable(false)] string message, Exception innerException, ExceptionRootCause rootCause = ExceptionRootCause.Unknown)
: base(message, innerException)
{
if (rootCause != ExceptionRootCause.Unknown)
{
m_rootCause = rootCause;
}
else if (innerException != null)
{
m_rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(innerException);
}
else
{
m_rootCause = ExceptionRootCause.Unknown;
}
}
/// <summary>
/// Constructor
/// </summary>
public EngineSerializer(
LoggingContext loggingContext,
string engineCacheLocation,
FileEnvelopeId?correlationId = null,
bool useCompression = false,
bool debug = false,
bool readOnly = false,
FileSystemStreamProvider readStreamProvider = null,
ITempCleaner tempDirectoryCleaner = null)
{
Contract.Requires(loggingContext != null);
Contract.Requires(engineCacheLocation != null);
Contract.Requires(Path.IsPathRooted(engineCacheLocation));
Contract.Requires(!string.IsNullOrWhiteSpace(engineCacheLocation));
LoggingContext = loggingContext;
m_engineCacheLocation = engineCacheLocation;
m_debug = debug;
m_correlationId = correlationId;
m_useCompression = useCompression;
m_readStreamProvider = readStreamProvider ?? FileSystemStreamProvider.Default;
m_tempDirectoryCleaner = tempDirectoryCleaner;
if (!readOnly)
{
try
{
FileUtilities.CreateDirectoryWithRetry(engineCacheLocation);
}
catch (Exception ex)
{
ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex);
BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause);
throw new BuildXLException("Unable to create engine serializer cache directory: ", ex);
}
}
}
/// <inheritdoc />
public override bool ShouldMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause)
{
return(rootCause == ExceptionRootCause.CorruptedCache);
}
/// <summary>
/// Creates and starts a task to deserialize an object
/// </summary>
/// <param name="file">This will become the filename</param>
/// <param name="deserializer">Deserialization function; its get a reader for the file stream, and a function that allows obtaining additional streams if needed</param>
/// <param name="skipHeader">If enabled, the correlation id is not checked for consistency</param>
/// <returns>task for deserialized value</returns>
internal Task <TObject> DeserializeFromFileAsync <TObject>(
GraphCacheFile file,
Func <BuildXLReader, Task <TObject> > deserializer,
bool skipHeader = false)
{
var task = Task.Run(
async() =>
{
var objectLabel = GetFileName(file);
string path = GetFullPath(objectLabel);
FileEnvelope fileEnvelope = GetFileEnvelope(file);
var result = default(TObject);
try
{
Stopwatch sw = Stopwatch.StartNew();
using (var fileStreamWrapper = m_readStreamProvider.OpenReadStream(path))
{
var fileStream = fileStreamWrapper.Value;
FileEnvelopeId persistedCorrelationId = fileEnvelope.ReadHeader(fileStream);
if (!skipHeader)
{
// We are going to check if all files that are going to be (concurrently) deserialized have matching correlation ids.
// The first discovered correlation id is going to be used to check all others.
if (m_correlationId == null)
{
Interlocked.CompareExchange(ref m_correlationId, persistedCorrelationId, null);
}
FileEnvelope.CheckCorrelationIds(persistedCorrelationId, (FileEnvelopeId)m_correlationId);
}
var isCompressed = fileStream.ReadByte() == 1;
using (Stream readStream = isCompressed ? new TrackedStream(new BufferedStream(new DeflateStream(fileStream, CompressionMode.Decompress), 64 << 10)) : fileStream)
using (BuildXLReader reader = new BuildXLReader(m_debug, readStream, leaveOpen: false))
{
result = await deserializer(reader);
}
}
Tracing.Logger.Log.DeserializedFile(LoggingContext, path, sw.ElapsedMilliseconds);
return(result);
}
catch (BuildXLException ex)
{
if (ex.InnerException is FileNotFoundException)
{
// Files might be deleted manually in the EngineCache directory. Log it as verbose.
Tracing.Logger.Log.FailedToDeserializeDueToFileNotFound(LoggingContext, path);
return(result);
}
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.LogEventMessage);
return(result);
}
catch (IOException ex)
{
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message);
return(result);
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
// There are 2 reasons to be here.
// 1. A malformed file can cause ContractException, IndexOutOfRangeException, MemoryException or something else.
// 2. We may have a bug.
// Since the malformed file will always cause a crash until someone removes the file from the cache, allow BuildXL to recover
// by eating the exception. However remember to log it in order to keep track of bugs.
ExceptionRootCause rootCause = ExceptionUtilities.AnalyzeExceptionRootCause(ex);
BuildXL.Tracing.UnexpectedCondition.Log(LoggingContext, ex.ToStringDemystified() + Environment.NewLine + rootCause);
Tracing.Logger.Log.FailedToDeserializePipGraph(LoggingContext, path, ex.Message);
return(result);
}
});
lock (m_deserializationSyncObject)
{
m_deserializationTasks.Add(task);
}
return(task);
}
/// <summary>
/// Attempts to assign a root cause to an arbitrary exception.
/// </summary>
/// <remarks>
/// The analysis strategy is so:
/// - Walking from the outermost to innermost exception, find the first root cause
/// (i.e., first that is not <see cref="ExceptionRootCause.Unknown"/>)
/// - Respect <see cref="BuildXLException.RootCause"/> (we allow the enlightened creator
/// of the exception to provide a root cause).
/// - For everything else, try to match patterns of exception type, HRESULT, etc.
/// </remarks>
public static ExceptionRootCause AnalyzeExceptionRootCause(Exception ex)
{
Contract.RequiresNotNull(ex);
const int OutOfDiskSpaceHResult = unchecked ((int)0x80070070);
const int DataErrorCRCResult = unchecked ((int)0x80070017);
const int DataErrorSeek = unchecked ((int)0x80070019);
const int DataErrorSector = unchecked ((int)0x8007001B);
const int DataErrorWriteFault = unchecked ((int)0x8007001D);
const int DataErrorReadFault = unchecked ((int)0x8007001E);
const int DataErrorGeneralFault = unchecked ((int)0x8007001F);
const int NoSystemResourcesFault = unchecked ((int)0x800705AA);
const int PipeNotConnectedFault = unchecked ((int)0x800700E9);
const int NotEnoughStorage = unchecked ((int)0x8);
const int IncorrectFunction = unchecked ((int)0x00000001);
var buildXLException = ex as BuildXLException;
if (buildXLException != null)
{
return(buildXLException.RootCause);
}
if (ex is OutOfMemoryException)
{
return(ExceptionRootCause.OutOfMemory);
}
if (ex is EventSourceException evSrcException && evSrcException.InnerException is BuildXLException innerBuldXLException)
{
return(innerBuldXLException.RootCause);
}
int exHResult = GetHResult(ex);
if (ex is IOException && exHResult == IncorrectFunction)
{
return(ExceptionRootCause.DeviceAccessError);
}
if (ex is IOException && ex.Message.Contains("No space left on device"))
{
return(ExceptionRootCause.OutOfDiskSpace);
}
Win32Exception win32Ex = ex as Win32Exception;
if (exHResult == OutOfDiskSpaceHResult ||
(win32Ex != null && win32Ex.ErrorCode == NotEnoughStorage))
{
return(ExceptionRootCause.OutOfDiskSpace);
}
if (exHResult == DataErrorCRCResult ||
exHResult == DataErrorSeek ||
exHResult == DataErrorSector ||
exHResult == DataErrorWriteFault ||
exHResult == DataErrorReadFault ||
exHResult == DataErrorGeneralFault)
{
return(ExceptionRootCause.DataErrorDriveFailure);
}
if (exHResult == NoSystemResourcesFault)
{
return(ExceptionRootCause.NoSystemResources);
}
if (ex is FileLoadException ||
(ex is FileNotFoundException && ex.Message.Contains(Strings.ExceptionUtilities_MissingDependencyPattern)) ||
ex is DllNotFoundException ||
ex is TypeLoadException)
{
return(ExceptionRootCause.MissingRuntimeDependency);
}
if (exHResult == PipeNotConnectedFault)
{
// Verify that the failure is caused by the console.
try
{
Console.Out.Flush();
Console.Error.Flush();
}
catch (IOException)
{
return(ExceptionRootCause.ConsoleNotConnected);
}
}
var aggregateException = ex as AggregateException;
if (aggregateException != null)
{
// Aggregate exceptions may have multiple inner exceptions. We take the first root cause we find
// (rather than trying to faithfully represent multiple root causes).
aggregateException = aggregateException.Flatten();
foreach (Exception inner in aggregateException.InnerExceptions)
{
ExceptionRootCause innerRootCause = AnalyzeExceptionRootCause(inner);
if (innerRootCause != ExceptionRootCause.Unknown)
{
return(innerRootCause);
}
}
}
else if (ex.InnerException != null)
{
return(AnalyzeExceptionRootCause(ex.InnerException));
}
return(ExceptionRootCause.Unknown);
}
/// <summary> /// Checks if failure should be marked. /// </summary> /// <param name="exception">Exception that causes the failure.</param> /// <param name="rootCause"><see cref="ExceptionRootCause"/></param> /// <returns>True iff failure should be marked.</returns> public abstract bool ShouldMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause);
/// <summary>
/// Only run handling for failures of unknown root cause. Many of the known root causes are unrelated to the build
/// and cannot be handled during the build.
/// </summary>
public override bool ShouldMarkFailure([NotNull] Exception exception, ExceptionRootCause rootCause)
{
return(rootCause == ExceptionRootCause.Unknown);
}
private BuildXLException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
m_rootCause = (ExceptionRootCause)info.GetInt32(ExceptionRootCauseName);
}
#pragma warning restore CA2235 // Mark all non-serializable fields
/// <summary>
/// Creates a BuildXLException with the given diagnostic message, without any inner exception as the root cause.
/// Instead, a well-known <see cref="ExceptionRootCause"/> may be provided.
/// </summary>
public BuildXLException([Localizable(false)] string message, ExceptionRootCause rootCause = ExceptionRootCause.Unknown)
: base(message)
{
m_rootCause = rootCause;
}
