/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
StartupStarted = true;
return(StartupCall <MemoizationStoreTracer> .RunAsync(_tracer, context, async() =>
{
var innerCacheStarted = await _innerICache.StartupAsync(context);
if (!innerCacheStarted)
{
context.Error("StartUp call on inner cache failed.");
return new BoolResult(innerCacheStarted);
}
var metadataCacheStarted = await _metadataCache.StartupAsync(context);
if (!metadataCacheStarted)
{
context.Error("StartUp call on metadata cache failed. Shutting down inner cache.");
await _innerICache.ShutdownAsync(context).ThrowIfFailure();
return new BoolResult(metadataCacheStarted);
}
StartupCompleted = true;
return BoolResult.Success;
}));
}
/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
return(StartupCall <ContentStoreTracer> .RunAsync(Tracer, context, async() =>
{
StartupStarted = true;
var finalResult = BoolResult.Success;
var stores = _drivesWithContentStore.Values.ToArray();
for (var i = 0; i < stores.Length; i++)
{
var startupResult = await stores[i].StartupAsync(context).ConfigureAwait(false);
if (!startupResult.Succeeded)
{
finalResult = startupResult;
for (var j = 0; j < i; j++)
{
var shutdownResult = await stores[j].ShutdownAsync(context).ConfigureAwait(false);
if (!shutdownResult.Succeeded)
{
finalResult = new BoolResult(finalResult, shutdownResult.ErrorMessage);
}
}
}
}
StartupCompleted = true;
return finalResult;
}));
}
/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
StartupStarted = true;
return(StartupCall <MemoizationStoreTracer> .RunAsync(Tracer.MemoizationStoreTracer, context, async() =>
{
BoolResult result;
var backingContentSessionTask = Task.Run(async() => await BackingContentSession.StartupAsync(context).ConfigureAwait(false));
var writeThroughContentSessionResult = WriteThroughContentSession != null
? await WriteThroughContentSession.StartupAsync(context).ConfigureAwait(false)
: BoolResult.Success;
var backingContentSessionResult = await backingContentSessionTask.ConfigureAwait(false);
if (backingContentSessionResult.Succeeded && writeThroughContentSessionResult.Succeeded)
{
_taskTracker = new BackgroundTaskTracker(Component, new Context(context));
result = BoolResult.Success;
}
else
{
var sb = new StringBuilder();
if (backingContentSessionResult.Succeeded)
{
var r = await BackingContentSession.ShutdownAsync(context).ConfigureAwait(false);
if (!r.Succeeded)
{
sb.Append($"Backing content session shutdown failed, error=[{r}]");
}
}
else
{
sb.Append($"Backing content session startup failed, error=[{backingContentSessionResult}]");
}
if (writeThroughContentSessionResult.Succeeded)
{
var r = WriteThroughContentSession != null
? await WriteThroughContentSession.ShutdownAsync(context).ConfigureAwait(false)
: BoolResult.Success;
if (!r.Succeeded)
{
sb.Append(sb.Length > 0 ? ", " : string.Empty);
sb.Append($"Write-through content session shutdown failed, error=[{r}]");
}
}
else
{
sb.Append(sb.Length > 0 ? ", " : string.Empty);
sb.Append($"Write-through content session startup failed, error=[{writeThroughContentSessionResult}]");
}
result = new BoolResult(sb.ToString());
}
StartupCompleted = true;
return result;
}));
}
/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
StartupStarted = true;
return(StartupCall <ContentSessionTracer> .RunAsync(_tracer, context, async() =>
{
var startupResults =
await
Task.WhenAll(_sessionForStream.StartupAsync(context), _sessionForPath.StartupAsync(context));
Contract.Assert(startupResults.Length == 2);
var startupResultForStream = startupResults[0];
var startupResultForPath = startupResults[1];
var result = startupResultForStream & startupResultForPath;
if (!result.Succeeded)
{
var sb = new StringBuilder();
if (!startupResultForStream.Succeeded)
{
sb.Concat($"{SessionForStreamText} startup failed, error=[{startupResultForStream}]", "; ");
}
if (!startupResultForPath.Succeeded)
{
sb.Concat($"{SessionForPathText} startup failed, error=[{startupResultForPath}]", "; ");
}
if (startupResultForStream.Succeeded)
{
var shutdownResult = await _sessionForStream.ShutdownAsync(context);
if (!shutdownResult.Succeeded)
{
sb.Concat($"{SessionForStreamText} shutdown failed, error=[{shutdownResult}]", "; ");
}
}
if (startupResultForPath.Succeeded)
{
var shutdownResult = await _sessionForPath.ShutdownAsync(context);
if (!shutdownResult.Succeeded)
{
sb.Concat($"{SessionForPathText} shutdown failed, error=[{shutdownResult}]", "; ");
}
}
result = new BoolResult(sb.ToString());
}
StartupCompleted = true;
return result;
}));
}
/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
StartupStarted = true;
return(StartupCall <ContentSessionTracer> .RunAsync(
_tracer,
context,
async() =>
{
_tracer.TraceStartupConfiguration(context, Configuration);
if (Configuration.RedisGlobalStoreConnectionString != null)
{
RedisDatabaseFactoryForRedisGlobalStore = await RedisDatabaseFactory.CreateAsync(
context,
new LiteralConnectionStringProvider(Configuration.RedisGlobalStoreConnectionString));
if (Configuration.RedisGlobalStoreSecondaryConnectionString != null)
{
RedisDatabaseFactoryForRedisGlobalStoreSecondary = await RedisDatabaseFactory.CreateAsync(
context,
new LiteralConnectionStringProvider(Configuration.RedisGlobalStoreSecondaryConnectionString));
}
}
else
{
// Local location store can only be used if connection string is provided for redis global store
Contract.Assert(!Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore));
}
// Instantiate factories for old redis only when we use old redis.
if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis))
{
Contract.Assert(_contentConnectionStringProvider != null);
Contract.Assert(_machineConnectionStringProvider != null);
RedisDatabaseFactoryForContent = await RedisDatabaseFactory.CreateAsync(context, _contentConnectionStringProvider);
RedisDatabaseFactoryForMachineLocations = await RedisDatabaseFactory.CreateAsync(context, _machineConnectionStringProvider);
}
StartupCompleted = true;
return BoolResult.Success;
}));
}
/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
return(StartupCall <ContentSessionTracer> .RunAsync(
Tracer,
context,
async() =>
{
StartupStarted = true;
var finalResult = BoolResult.Success;
var sessions = SessionsByCacheRoot.Values.ToArray();
for (var i = 0; i < sessions.Length; i++)
{
var canHibernate = sessions[i] is IHibernateContentSession ? "can" : "cannot";
Tracer.Debug(context, $"Session {sessions[i].Name} {canHibernate} hibernate");
var startupResult = await sessions[i].StartupAsync(context).ConfigureAwait(false);
if (!startupResult.Succeeded)
{
finalResult = startupResult;
for (var j = 0; j < i; j++)
{
var shutdownResult = await sessions[j].ShutdownAsync(context).ConfigureAwait(false);
if (!shutdownResult.Succeeded)
{
finalResult = new BoolResult(finalResult, shutdownResult.ErrorMessage);
}
}
}
}
StartupCompleted = true;
return finalResult;
}));
}
/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
StartupStarted = true;
return(StartupCall <CacheTracer> .RunAsync(_tracer, context, async() =>
{
var preStartupResult = await PreStartupAsync(context).ConfigureAwait(false);
if (!preStartupResult.Succeeded)
{
return preStartupResult;
}
var contentStoreTask = Task.Run(() => ContentStore.StartupAsync(context));
var memoizationStoreResult = await MemoizationStore.StartupAsync(context).ConfigureAwait(false);
var contentStoreResult = await contentStoreTask.ConfigureAwait(false);
BoolResult result;
if (!contentStoreResult.Succeeded || !memoizationStoreResult.Succeeded)
{
var sb = new StringBuilder();
if (contentStoreResult.Succeeded)
{
var r = await ContentStore.ShutdownAsync(context).ConfigureAwait(false);
if (!r.Succeeded)
{
sb.AppendFormat($"Content store shutdown failed, error=[{r}]");
}
}
else
{
sb.AppendFormat($"Content store startup failed, error=[{contentStoreResult}]");
}
if (memoizationStoreResult.Succeeded)
{
var r = await MemoizationStore.ShutdownAsync(context).ConfigureAwait(false);
if (!r.Succeeded)
{
sb.Append(sb.Length > 0 ? ", " : string.Empty);
sb.AppendFormat($"Memoization store shutdown failed, error=[{memoizationStoreResult}]");
}
}
else
{
sb.Append(sb.Length > 0 ? ", " : string.Empty);
sb.AppendFormat($"Memoization store startup failed, error=[{memoizationStoreResult}]");
}
result = new BoolResult(sb.ToString());
}
else
{
result = BoolResult.Success;
}
StartupCompleted = true;
return result;
}));
}
/// <inheritdoc />
public Task <BoolResult> StartupAsync(Context context)
{
StartupStarted = true;
return(StartupCall <BuildCacheCacheTracer> .RunAsync(_tracer, context, async() =>
{
BoolResult result;
_tracer.Debug(context, $"Creating ContentHashListAdapterFactory with {nameof(_useBlobContentHashLists)}={_useBlobContentHashLists}");
_contentHashListAdapterFactory = await ContentHashListAdapterFactory.CreateAsync(
context, _buildCacheHttpClientFactory, _useBlobContentHashLists);
Id =
await _contentHashListAdapterFactory.BuildCacheHttpClient.GetBuildCacheServiceDeterminism(_cacheNamespace)
.ConfigureAwait(false);
var backingContentStoreTask = Task.Run(async() => await _backingContentStore.StartupAsync(context).ConfigureAwait(false));
var writeThroughContentStoreResult = _writeThroughContentStore != null
? await _writeThroughContentStore.StartupAsync(context).ConfigureAwait(false)
: BoolResult.Success;
var backingContentStoreResult = await backingContentStoreTask.ConfigureAwait(false);
if (backingContentStoreResult.Succeeded && writeThroughContentStoreResult.Succeeded)
{
result = BoolResult.Success;
}
else
{
var sb = new StringBuilder();
if (backingContentStoreResult.Succeeded)
{
var r = await _backingContentStore.ShutdownAsync(context).ConfigureAwait(false);
if (!r.Succeeded)
{
sb.Append($"Backing content store shutdown failed, error=[{r}]");
}
}
else
{
sb.Append($"Backing content store startup failed, error=[{backingContentStoreResult}]");
}
if (writeThroughContentStoreResult.Succeeded)
{
var r = _writeThroughContentStore != null
? await _writeThroughContentStore.ShutdownAsync(context).ConfigureAwait(false)
: BoolResult.Success;
if (!r.Succeeded)
{
sb.Append(sb.Length > 0 ? ", " : string.Empty);
sb.Append($"Write-through content store shutdown failed, error=[{r}]");
}
}
else
{
sb.Append(sb.Length > 0 ? ", " : string.Empty);
sb.Append($"Write-through content store startup failed, error=[{writeThroughContentStoreResult}]");
}
result = new BoolResult(sb.ToString());
}
StartupCompleted = true;
return result;
}));
}
/// <inheritdoc />
public virtual Task <BoolResult> StartupAsync(Context context)
{
StartupStarted = true;
return(StartupCall <TTracer> .RunAsync(Tracer, context, async() =>
{
var preStartupResult = await PreStartupAsync(context);
if (!preStartupResult.Succeeded)
{
return preStartupResult;
}
CreateMarkerFile(context);
if (_config.BackupDatabase && File.Exists(DatabaseFilePath.Path))
{
try
{
File.Copy(DatabaseFilePath.Path, DatabaseBackupTempPath.Path, overwrite: true);
Tracer.Info(context, "Created a temporary backup of the master DB");
}
catch (Exception e)
{
Tracer.Error(context, e, "Failed to create a copy of the master DB as backup");
}
}
try
{
BoolResult startupAttempt = await AttemptStartupAsync(context);
if (startupAttempt.Succeeded)
{
if (_config.BackupDatabase && File.Exists(DatabaseBackupTempPath.Path))
{
try
{
File.Move(DatabaseBackupTempPath.Path, DatabaseBackupPath.Path); // rename
Tracer.Info(context, "The temporary backup DB has been upgraded to be the primary backup");
}
catch (Exception e)
{
Tracer.Error(context, e, "Failed to upgrade the temporary backup DB to primary backup");
}
}
} // else leave the temp backup file lying around for possible inspection. It may get overwritten at next startup.
return startupAttempt;
}
catch (SQLiteException sqle1)
{
// Leave the temp backup file lying around for possible inspection. It may get overwritten at next startup.
if (sqle1.ResultCode != SQLiteErrorCode.Corrupt && sqle1.ResultCode != SQLiteErrorCode.NotADb)
{
throw;
}
else
{
Tracer.IncrementMasterDBCorruptionCount(context, "The master DB is corrupt.");
CloseConnections();
bool masterDbRestoredFromBackup = false;
if (File.Exists(DatabaseBackupPath.Path))
{
Tracer.Info(context, "Found the backup DB");
try
{
File.Copy(DatabaseBackupPath.Path, DatabaseFilePath.Path, overwrite: true);
masterDbRestoredFromBackup = true;
Tracer.Info(context, "Replaced the master DB with the backup");
}
catch (Exception e)
{
masterDbRestoredFromBackup = false;
Tracer.Error(context, e, "Restoring the master DB from backup failed");
}
}
else
{
masterDbRestoredFromBackup = false;
Tracer.Info(context, "Backup DB does not exist");
}
if (!masterDbRestoredFromBackup)
{
try
{
File.Delete(DatabaseFilePath.Path);
Tracer.Info(context, "Deleted corrupt master DB.");
}
catch (Exception e)
{
Tracer.Error(context, e, "Failed to delete master DB");
}
}
try
{
return await AttemptStartupAsync(context);
}
catch (SQLiteException sqle2)
{
if (sqle2.ResultCode != SQLiteErrorCode.Corrupt && sqle2.ResultCode != SQLiteErrorCode.NotADb)
{
throw;
}
else
{
Tracer.IncrementBackupDBCorruptionCount(context, "The backup DB is corrupt.");
CloseConnections();
try
{
File.Delete(DatabaseFilePath.Path);
Tracer.Info(context, "Deleted corrupt master DB");
}
catch (Exception e)
{
Tracer.Error(context, e, "Failed to delete master DB");
}
try
{
File.Delete(DatabaseBackupPath.Path);
Tracer.Info(context, "Deleted corrupt backup DB");
}
catch (Exception e)
{
Tracer.Error(context, e, "Failed to delete backup DB");
}
return await AttemptStartupAsync(context);
}
}
}
}
}));
}
