private static DateTime?GetLastBackup(TransactionOperationContext context, ServerStore serverStore, string databaseName)
{
using (var databaseRecord = serverStore.Cluster.ReadRawDatabaseRecord(context, databaseName, out _))
{
if (databaseRecord == null)
{
return(null); // should not happen
}
var periodicBackupTaskIds = databaseRecord.PeriodicBackupsTaskIds;
if (periodicBackupTaskIds == null || periodicBackupTaskIds.Count == 0)
{
return(null); // no backup
}
var lastBackup = DateTime.MinValue;
foreach (var periodicBackupTaskId in periodicBackupTaskIds)
{
var status = PeriodicBackupRunner.GetBackupStatusFromCluster(serverStore, context, databaseName, periodicBackupTaskId);
if (status == null)
{
continue; // we have a backup task but no backup was ever done
}
var currentLatestBackup = LastBackupDate(status.LastFullBackup, status.LastIncrementalBackup);
if (currentLatestBackup > lastBackup)
{
lastBackup = currentLatestBackup;
}
}
return(lastBackup);
public DynamicJsonValue GenerateDatabaseInfo()
{
var envs = GetAllStoragesEnvironment().ToList();
if (envs.Any(x => x.Environment == null))
{
return(null);
}
var size = new Size(envs.Sum(env => env.Environment.Stats().AllocatedDataFileSizeInBytes));
var databaseInfo = new DynamicJsonValue
{
[nameof(DatabaseInfo.HasRevisionsConfiguration)] = DocumentsStorage.RevisionsStorage.Configuration != null,
[nameof(DatabaseInfo.HasExpirationConfiguration)] = ExpiredDocumentsCleaner != null,
[nameof(DatabaseInfo.IsAdmin)] = true, //TODO: implement me!
[nameof(DatabaseInfo.IsEncrypted)] = DocumentsStorage.Environment.Options.EncryptionEnabled,
[nameof(DatabaseInfo.Name)] = Name,
[nameof(DatabaseInfo.Disabled)] = false, //TODO: this value should be overwritten by the studio since it is cached
[nameof(DatabaseInfo.TotalSize)] = new DynamicJsonValue
{
[nameof(Size.HumaneSize)] = size.HumaneSize,
[nameof(Size.SizeInBytes)] = size.SizeInBytes
},
[nameof(DatabaseInfo.IndexingErrors)] = IndexStore.GetIndexes().Sum(index => index.GetErrorCount()),
[nameof(DatabaseInfo.Alerts)] = NotificationCenter.GetAlertCount(),
[nameof(DatabaseInfo.UpTime)] = null, //it is shutting down
[nameof(DatabaseInfo.BackupInfo)] = PeriodicBackupRunner.GetBackupInfo(),
[nameof(DatabaseInfo.DocumentsCount)] = DocumentsStorage.GetNumberOfDocuments(),
[nameof(DatabaseInfo.IndexesCount)] = IndexStore.GetIndexes().Count(),
[nameof(DatabaseInfo.RejectClients)] = false, //TODO: implement me!
[nameof(DatabaseInfo.IndexingStatus)] = IndexStore.Status.ToString(),
["CachedDatabaseInfo"] = true
};
return(databaseInfo);
}
private void InitializeFromDatabaseRecord(DatabaseRecord record)
{
if (record == null)
{
return;
}
ClientConfiguration = record.Client;
DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(record);
ExpiredDocumentsCleaner = ExpiredDocumentsCleaner.LoadConfigurations(this, record, ExpiredDocumentsCleaner);
PeriodicBackupRunner.UpdateConfigurations(record);
}
public void Initialize(InitializeOptions options = InitializeOptions.None)
{
try
{
NotificationCenter.Initialize(this);
DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId);
TxMerger.Start();
ConfigurationStorage.Initialize();
if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord)
{
return;
}
long index;
DatabaseRecord record;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
record = _serverStore.Cluster.ReadDatabase(context, Name, out index);
if (record == null)
{
DatabaseDoesNotExistException.Throw(Name);
}
PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore);
_indexStoreTask = IndexStore.InitializeAsync(record);
ReplicationLoader?.Initialize(record);
EtlLoader.Initialize(record);
DocumentTombstoneCleaner.Start();
try
{
_indexStoreTask.Wait(DatabaseShutdown);
}
finally
{
_indexStoreTask = null;
}
SubscriptionStorage.Initialize();
NotifyFeaturesAboutStateChange(record, index);
}
catch (Exception)
{
Dispose();
throw;
}
}
public void RunIdleOperations()
{
if (Monitor.TryEnter(_idleLocker) == false)
{
return;
}
try
{
_lastIdleTicks = DateTime.UtcNow.Ticks;
IndexStore?.RunIdleOperations();
Operations?.CleanupOperations();
PeriodicBackupRunner?.RemoveInactiveCompletedTasks();
}
finally
{
Monitor.Exit(_idleLocker);
}
}
private static DateTime?GetLastBackup(TransactionOperationContext context, ServerStore serverStore, string databaseName)
{
using (var databaseRecord = serverStore.Cluster.ReadRawDatabaseRecord(context, databaseName, out _))
{
if (databaseRecord == null)
{
return(null); // should not happen
}
var periodicBackupTaskIds = databaseRecord.GetPeriodicBackupsTaskIds();
if (periodicBackupTaskIds == null || periodicBackupTaskIds.Count == 0)
{
return(null); // no backup
}
foreach (var periodicBackupTaskId in periodicBackupTaskIds)
{
var status = PeriodicBackupRunner.GetBackupStatusFromCluster(serverStore, context, databaseName, periodicBackupTaskId);
if (status == null)
{
return(DateTime.MinValue); // we have a backup task but no backup was ever done?
}
if (status.LastFullBackup == null)
{
return(DateTime.MinValue); // we never did a full backup
}
if (status.LastIncrementalBackup != null)
{
return(status.LastFullBackup > status.LastIncrementalBackup
? status.LastFullBackup
: status.LastIncrementalBackup);
}
return(status.LastFullBackup);
}
return(null);
}
}
private static async Task CheckBackupOperationStatus(OperationStatus expected, OperationStatus actual, DocumentStore store, long taskId, long opId,
PeriodicBackupRunner periodicBackupRunner)
{
if (expected == OperationStatus.Completed && actual == OperationStatus.Faulted)
{
// gather debug info
var operation = new GetPeriodicBackupStatusOperation(taskId);
var status = (await store.Maintenance.SendAsync(operation)).Status;
TryGetBackupStatusFromPeriodicBackupAndPrint(expected, actual, opId, periodicBackupRunner, status, result: null);
Assert.True(false,
$"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Backup status from storage for current operation id: '{opId}':{Environment.NewLine}" +
PrintBackupStatus(status));
}
else if (expected == OperationStatus.Completed && actual == OperationStatus.InProgress)
{
// backup didn't complete in time, try to print running backup status, and backup result
var pb = periodicBackupRunner?.PeriodicBackups.FirstOrDefault(x => x.RunningBackupStatus != null && x.BackupStatus.TaskId == taskId);
if (pb == null)
{
// print previous backup status saved in memory
var operation = new GetPeriodicBackupStatusOperation(taskId);
var status = (await store.Maintenance.SendAsync(operation)).Status;
Assert.True(false,
$"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Could not fetch running backup status for current task id: '{taskId}', previous backup status:{Environment.NewLine}" +
PrintBackupStatus(status));
}
else
{
Assert.True(false,
$"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Running backup status for current task id: '{taskId}':{Environment.NewLine}" +
PrintBackupStatus(pb.RunningBackupStatus));
}
}
}
public unsafe void Dispose()
{
if (_databaseShutdown.IsCancellationRequested)
{
return; // double dispose?
}
lock (this)
{
if (_databaseShutdown.IsCancellationRequested)
{
return; // double dispose?
}
//before we dispose of the database we take its latest info to be displayed in the studio
try
{
var databaseInfo = GenerateDatabaseInfo();
if (databaseInfo != null)
{
DatabaseInfoCache?.InsertDatabaseInfo(databaseInfo, Name);
}
}
catch (Exception e)
{
// if we encountered a catastrophic failure we might not be able to retrieve database info
if (_logger.IsInfoEnabled)
{
_logger.Info("Failed to generate and store database info", e);
}
}
_databaseShutdown.Cancel();
// we'll wait for 1 minute to drain all the requests
// from the database
var sp = Stopwatch.StartNew();
while (sp.ElapsedMilliseconds < 60 * 1000)
{
if (Interlocked.Read(ref _usages) == 0)
{
break;
}
if (_waitForUsagesOnDisposal.Wait(1000))
{
_waitForUsagesOnDisposal.Reset();
}
}
var exceptionAggregator = new ExceptionAggregator(_logger, $"Could not dispose {nameof(DocumentDatabase)} {Name}");
foreach (var connection in RunningTcpConnections)
{
exceptionAggregator.Execute(() =>
{
connection.Dispose();
});
}
exceptionAggregator.Execute(() =>
{
TxMerger?.Dispose();
});
if (_indexStoreTask != null)
{
exceptionAggregator.Execute(() =>
{
_indexStoreTask.Wait(DatabaseShutdown);
});
}
exceptionAggregator.Execute(() =>
{
IndexStore?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ExpiredDocumentsCleaner?.Dispose();
});
exceptionAggregator.Execute(() =>
{
PeriodicBackupRunner?.Dispose();
});
exceptionAggregator.Execute(() =>
{
DocumentTombstoneCleaner?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ReplicationLoader?.Dispose();
});
exceptionAggregator.Execute(() =>
{
EtlLoader?.Dispose();
});
exceptionAggregator.Execute(() =>
{
Operations?.Dispose(exceptionAggregator);
});
exceptionAggregator.Execute(() =>
{
NotificationCenter?.Dispose();
});
exceptionAggregator.Execute(() =>
{
SubscriptionStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ConfigurationStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
DocumentsStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
_databaseShutdown.Dispose();
});
exceptionAggregator.Execute(() =>
{
if (MasterKey == null)
{
return;
}
fixed(byte *pKey = MasterKey)
{
Sodium.sodium_memzero(pKey, (UIntPtr)MasterKey.Length);
}
});
exceptionAggregator.ThrowIfNeeded();
}
}
public void Initialize(InitializeOptions options = InitializeOptions.None)
{
try
{
_addToInitLog("Initializing NotificationCenter");
NotificationCenter.Initialize(this);
_addToInitLog("Initializing DocumentStorage");
DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId);
_addToInitLog("Starting Transaction Merger");
TxMerger.Start();
_addToInitLog("Initializing ConfigurationStorage");
ConfigurationStorage.Initialize();
if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord)
{
return;
}
_addToInitLog("Loading Database");
long index;
DatabaseRecord record;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
record = _serverStore.Cluster.ReadDatabase(context, Name, out index);
if (record == null)
{
DatabaseDoesNotExistException.Throw(Name);
}
PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore);
_addToInitLog("Initializing IndexStore (async)");
_indexStoreTask = IndexStore.InitializeAsync(record);
_addToInitLog("Initializing Replication");
ReplicationLoader?.Initialize(record);
_addToInitLog("Initializing ETL");
EtlLoader.Initialize(record);
DocumentTombstoneCleaner.Start();
try
{
_indexStoreTask.Wait(DatabaseShutdown);
}
finally
{
_addToInitLog("Initializing IndexStore completed");
_indexStoreTask = null;
}
SubscriptionStorage.Initialize();
_addToInitLog("Initializing SubscriptionStorage completed");
TaskExecutor.Execute((state) =>
{
try
{
NotifyFeaturesAboutStateChange(record, index);
}
catch
{
// We ignore the exception since it was caught in the function itself
}
}, null);
}
catch (Exception)
{
Dispose();
throw;
}
}
private static void TryGetBackupStatusFromPeriodicBackupAndPrint(OperationStatus expected, OperationStatus actual, long opId, PeriodicBackupRunner periodicBackupRunner, PeriodicBackupStatus status, BackupResult result)
{
if (status?.LastOperationId != opId)
{
// failed to save backup status, lets fetch it from memory
var pb = periodicBackupRunner?.PeriodicBackups.FirstOrDefault(x => x.BackupStatus != null && x.BackupStatus.LastOperationId == opId);
if (pb == null)
{
Assert.True(false,
$"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Could not fetch backup status for current operation id: '{opId}', previous backup status:{Environment.NewLine}" +
PrintBackupStatus(status) + Environment.NewLine + "BackupResult Messages:" + Environment.NewLine + PrintBackupResultMessagesStatus(result));
}
else
{
Assert.True(false,
$"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Could not fetch backup status from storage for current operation id: '{opId}', current in memory backup status:{Environment.NewLine}" +
PrintBackupStatus(pb.BackupStatus) + Environment.NewLine + "BackupResult Messages:" + Environment.NewLine +
PrintBackupResultMessagesStatus(result));
}
}
}
