private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool clean = false)
{
try
{
var storeLocation = GetStoreLocation(activeSlot);
if (clean && Directory.Exists(storeLocation))
{
BuildXL.Native.IO.FileUtilities.DeleteDirectoryContents(storeLocation, deleteRootDirectory: true);
}
Directory.CreateDirectory(storeLocation);
Tracer.Info(context, $"Creating rocksdb store at '{storeLocation}'.");
var possibleStore = KeyValueStoreAccessor.Open(storeLocation,
additionalColumns: new[] { nameof(Columns.ClusterState), nameof(Columns.Metadata) },
rotateLogs: true,
failureHandler: failureEvent =>
{
// By default, rethrow is true iff it is a user error. We invalidate only if it isn't
failureEvent.Invalidate = !failureEvent.Rethrow;
},
invalidationHandler: failure => OnDatabaseInvalidated(context, failure),
onFailureDeleteExistingStoreAndRetry: _configuration.OnFailureDeleteExistingStoreAndRetry,
onStoreReset: failure => {
Tracer.Error(context, $"RocksDb critical error caused store to reset: {failure.DescribeIncludingInnerFailures()}");
});
if (possibleStore.Succeeded)
{
var oldKeyValueStore = _keyValueStore;
var store = possibleStore.Result;
if (oldKeyValueStore == null)
{
_keyValueStore = new KeyValueStoreGuard(store);
}
else
{
// Just replace the inner accessor
oldKeyValueStore.Replace(store);
}
_activeSlot = activeSlot;
_storeLocation = storeLocation;
}
return(possibleStore.Succeeded ? BoolResult.Success : new BoolResult($"Failed to initialize a RocksDb store at {_storeLocation}:", possibleStore.Failure.DescribeIncludingInnerFailures()));
}
catch (Exception ex) when(ex.IsRecoverableIoException())
{
return(new BoolResult(ex));
}
}
public RocksDbContentMetadataDatabase(IClock clock, RocksDbContentLocationDatabaseConfiguration configuration)
: base(clock, configuration, () => Array.Empty <MachineId>())
{
Contract.Requires(configuration.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep > 0);
_configuration = configuration;
_activeSlotFilePath = (_configuration.StoreLocation / ActiveStoreSlotFileName).ToString();
// this is a hacky way to convince the compiler that the field is initialized.
// Technically, the field is nullable, but keeping it as nullable causes more issues than giving us benefits.
_keyValueStore = null !;
}
private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool clean = false)
{
try
{
var storeLocation = GetStoreLocation(activeSlot);
if (clean && Directory.Exists(storeLocation))
{
BuildXL.Native.IO.FileUtilities.DeleteDirectoryContents(storeLocation, deleteRootDirectory: true);
}
Directory.CreateDirectory(storeLocation);
Tracer.Info(context, $"Creating rocksdb store at '{storeLocation}'.");
var possibleStore = KeyValueStoreAccessor.Open(storeLocation,
additionalColumns: new[] { nameof(ClusterState) });
if (possibleStore.Succeeded)
{
var oldKeyValueStore = _keyValueStore;
var store = possibleStore.Result;
if (oldKeyValueStore == null)
{
_keyValueStore = new KeyValueStoreGuard(store);
}
else
{
// Just replace the inner accessor
oldKeyValueStore.Replace(store);
}
_activeSlot = activeSlot;
_storeLocation = storeLocation;
}
return(possibleStore.Succeeded ? BoolResult.Success : new BoolResult($"Failed to initialize a RocksDb store at {_storeLocation}:", possibleStore.Failure.DescribeIncludingInnerFailures()));
}
catch (Exception ex) when(ex.IsRecoverableIoException())
{
return(new BoolResult(ex));
}
}
private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool clean)
{
try
{
var storeLocation = GetStoreLocation(activeSlot);
if (clean)
{
Counters[ContentLocationDatabaseCounters.DatabaseCleans].Increment();
if (Directory.Exists(storeLocation))
{
FileUtilities.DeleteDirectoryContents(storeLocation, deleteRootDirectory: true);
}
}
bool dbAlreadyExists = Directory.Exists(storeLocation);
Directory.CreateDirectory(storeLocation);
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'. Clean={clean}, Configured Epoch='{_configuration.Epoch}'");
var possibleStore = KeyValueStoreAccessor.Open(
new RocksDbStoreConfiguration(storeLocation)
{
AdditionalColumns = ColumnNames,
RotateLogsMaxFileSizeBytes = _configuration.LogsKeepLongTerm ? 0ul : ((ulong)"1MB".ToSize()),
RotateLogsNumFiles = _configuration.LogsKeepLongTerm ? 60ul : 1,
RotateLogsMaxAge = TimeSpan.FromHours(_configuration.LogsKeepLongTerm ? 12 : 1),
EnableStatistics = true,
FastOpen = true,
// We take the user's word here. This may be completely wrong, but we don't have enough
// information at this point to take a decision here. If a machine is master and demoted to
// worker, EventHub may continue to process events for a little while. If we set this to
// read-only during that checkpoint, those last few events will fail with RocksDbException.
// NOTE: we need to check that the database exists because RocksDb will refuse to open an empty
// read-only instance.
ReadOnly = _configuration.OpenReadOnly && dbAlreadyExists,
// The RocksDb database here is read-only from the perspective of the default column family,
// but read/write from the perspective of the ClusterState (which is rewritten on every
// heartbeat). This means that the database may perform background compactions on the column
// families, possibly triggering a RocksDb corruption "block checksum mismatch" error.
// Since the writes to ClusterState are relatively few, we can make-do with disabling
// compaction here and pretending like we are using a read-only database.
DisableAutomaticCompactions = !IsDatabaseWriteable,
LeveledCompactionDynamicLevelTargetSizes = true,
Compression = _configuration.Compression,
UseReadOptionsWithSetTotalOrderSeekInDbEnumeration = _configuration.UseReadOptionsWithSetTotalOrderSeekInDbEnumeration,
UseReadOptionsWithSetTotalOrderSeekInGarbageCollection = _configuration.UseReadOptionsWithSetTotalOrderSeekInGarbageCollection,
},
// When an exception is caught from within methods using the database, this handler is called to
// decide whether the exception should be rethrown in user code, and the database invalidated. Our
// policy is to only invalidate if it is an exception coming from RocksDb, but not from our code.
failureHandler: failureEvent =>
{
// By default, rethrow is true iff it is a user error. We invalidate only if it isn't
failureEvent.Invalidate = !failureEvent.Rethrow;
},
// The database may be invalidated for a number of reasons, all related to latent bugs in our code.
// For example, exceptions thrown from methods that are operating on the DB. If that happens, we
// call a user-defined handler. This is because the instance is invalid after this happens.
invalidationHandler: failure => OnDatabaseInvalidated(context, failure),
// It is possible we may fail to open an already existing database. This can happen (most commonly)
// due to corruption, among others. If this happens, then we want to recreate it from empty. This
// only helps for the memoization store.
onFailureDeleteExistingStoreAndRetry: _configuration.OnFailureDeleteExistingStoreAndRetry,
// If the previous flag is true, and it does happen that we invalidate the database, we want to log
// it explicitly.
onStoreReset: failure =>
{
Tracer.Error(context, $"RocksDb critical error caused store to reset: {failure.DescribeIncludingInnerFailures()}");
});
if (possibleStore.Succeeded)
{
var oldKeyValueStore = _keyValueStore;
var store = possibleStore.Result;
if (oldKeyValueStore == null)
{
_keyValueStore = new KeyValueStoreGuard(store);
_keyValueStore.UseExclusive((db, state) =>
{
if (db.TryGetValue(nameof(GlobalKeys.ActiveColummGroup), out var activeColumnGroup))
{
_activeColumnsGroup = (ColumnGroup)Enum.Parse(typeof(ColumnGroup), activeColumnGroup);
}
else
{
_activeColumnsGroup = ColumnGroup.One;
}
return(true);
},
this).ThrowOnError();
}
else
{
// Just replace the inner accessor
oldKeyValueStore.Replace(store, db =>
{
if (db.TryGetValue(nameof(GlobalKeys.ActiveColummGroup), out var activeColumnGroup))
{
_activeColumnsGroup = (ColumnGroup)Enum.Parse(typeof(ColumnGroup), activeColumnGroup);
}
else
{
_activeColumnsGroup = ColumnGroup.One;
}
}).ThrowOnError();
}
_activeSlot = activeSlot;
_storeLocation = storeLocation;
}
return(possibleStore.Succeeded ? BoolResult.Success : new BoolResult($"Failed to initialize a RocksDb store at {_storeLocation}:", possibleStore.Failure.DescribeIncludingInnerFailures()));
}
catch (Exception ex) when(ex.IsRecoverableIoException())
{
return(new BoolResult(ex));
}
}
private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool clean)
{
try
{
var storeLocation = GetStoreLocation(activeSlot);
if (Directory.Exists(storeLocation))
{
// We backup right before loading. This means we should never loose any logs, but it also means
// the backup directory will only hold logs for DBs that have already been overwritten.
if (_logManager != null)
{
_logManager.BackupAsync(context, new AbsolutePath(storeLocation), activeSlot.ToString()).Result.IgnoreFailure();
Task.Run(() => _logManager.GarbageCollect(context)).FireAndForget(context, severityOnException: Severity.Error);
}
if (clean)
{
FileUtilities.DeleteDirectoryContents(storeLocation, deleteRootDirectory: true);
}
}
Directory.CreateDirectory(storeLocation);
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'.");
var possibleStore = KeyValueStoreAccessor.Open(
new KeyValueStoreAccessor.RocksDbStoreArguments()
{
StoreDirectory = storeLocation,
AdditionalColumns = new[] { nameof(Columns.ClusterState), nameof(Columns.Metadata) },
RotateLogs = true,
EnableStatistics = true,
FastOpen = true,
},
// When an exception is caught from within methods using the database, this handler is called to
// decide whether the exception should be rethrown in user code, and the database invalidated. Our
// policy is to only invalidate if it is an exception coming from RocksDb, but not from our code.
failureHandler: failureEvent =>
{
// By default, rethrow is true iff it is a user error. We invalidate only if it isn't
failureEvent.Invalidate = !failureEvent.Rethrow;
},
// The database may be invalidated for a number of reasons, all related to latent bugs in our code.
// For example, exceptions thrown from methods that are operating on the DB. If that happens, we
// call a user-defined handler. This is because the instance is invalid after this happens.
invalidationHandler: failure => OnDatabaseInvalidated(context, failure),
// It is possible we may fail to open an already existing database. This can happen (most commonly)
// due to corruption, among others. If this happens, then we want to recreate it from empty. This
// only helps for the memoization store.
onFailureDeleteExistingStoreAndRetry: _configuration.OnFailureDeleteExistingStoreAndRetry,
// If the previous flag is true, and it does happen that we invalidate the database, we want to log
// it explicitly.
onStoreReset: failure =>
{
Tracer.Error(context, $"RocksDb critical error caused store to reset: {failure.DescribeIncludingInnerFailures()}");
});
if (possibleStore.Succeeded)
{
var oldKeyValueStore = _keyValueStore;
var store = possibleStore.Result;
if (oldKeyValueStore == null)
{
_keyValueStore = new KeyValueStoreGuard(store);
}
else
{
// Just replace the inner accessor
oldKeyValueStore.Replace(store);
}
_activeSlot = activeSlot;
_storeLocation = storeLocation;
}
return(possibleStore.Succeeded ? BoolResult.Success : new BoolResult($"Failed to initialize a RocksDb store at {_storeLocation}:", possibleStore.Failure.DescribeIncludingInnerFailures()));
}
catch (Exception ex) when(ex.IsRecoverableIoException())
{
return(new BoolResult(ex));
}
}
private BoolResult Load(OperationContext context, StoreSlot activeSlot, bool clean)
{
try
{
var storeLocation = GetStoreLocation(activeSlot);
if (Directory.Exists(storeLocation))
{
if (clean)
{
FileUtilities.DeleteDirectoryContents(storeLocation, deleteRootDirectory: true);
}
}
Directory.CreateDirectory(storeLocation);
Tracer.Info(context, $"Creating RocksDb store at '{storeLocation}'.");
var possibleStore = KeyValueStoreAccessor.Open(
new KeyValueStoreAccessor.RocksDbStoreArguments()
{
StoreDirectory = storeLocation,
AdditionalColumns = new[] { nameof(Columns.ClusterState), nameof(Columns.Metadata) },
RotateLogsMaxFileSizeBytes = _configuration.LogsKeepLongTerm ? 0ul : ((ulong)"1MB".ToSize()),
RotateLogsNumFiles = _configuration.LogsKeepLongTerm ? 60ul : 1,
RotateLogsMaxAge = TimeSpan.FromHours(_configuration.LogsKeepLongTerm ? 12 : 1),
EnableStatistics = true,
FastOpen = true,
},
// When an exception is caught from within methods using the database, this handler is called to
// decide whether the exception should be rethrown in user code, and the database invalidated. Our
// policy is to only invalidate if it is an exception coming from RocksDb, but not from our code.
failureHandler: failureEvent =>
{
// By default, rethrow is true iff it is a user error. We invalidate only if it isn't
failureEvent.Invalidate = !failureEvent.Rethrow;
},
// The database may be invalidated for a number of reasons, all related to latent bugs in our code.
// For example, exceptions thrown from methods that are operating on the DB. If that happens, we
// call a user-defined handler. This is because the instance is invalid after this happens.
invalidationHandler: failure => OnDatabaseInvalidated(context, failure),
// It is possible we may fail to open an already existing database. This can happen (most commonly)
// due to corruption, among others. If this happens, then we want to recreate it from empty. This
// only helps for the memoization store.
onFailureDeleteExistingStoreAndRetry: _configuration.OnFailureDeleteExistingStoreAndRetry,
// If the previous flag is true, and it does happen that we invalidate the database, we want to log
// it explicitly.
onStoreReset: failure =>
{
Tracer.Error(context, $"RocksDb critical error caused store to reset: {failure.DescribeIncludingInnerFailures()}");
});
if (possibleStore.Succeeded)
{
var oldKeyValueStore = _keyValueStore;
var store = possibleStore.Result;
if (oldKeyValueStore == null)
{
_keyValueStore = new KeyValueStoreGuard(store);
}
else
{
// Just replace the inner accessor
oldKeyValueStore.Replace(store);
}
_activeSlot = activeSlot;
_storeLocation = storeLocation;
}
return(possibleStore.Succeeded ? BoolResult.Success : new BoolResult($"Failed to initialize a RocksDb store at {_storeLocation}:", possibleStore.Failure.DescribeIncludingInnerFailures()));
}
catch (Exception ex) when(ex.IsRecoverableIoException())
{
return(new BoolResult(ex));
}
}
