private static DateTime?GetLastBackup(TransactionOperationContext context, ServerStore serverStore, string databaseName) { using (var databaseRecord = serverStore.Cluster.ReadRawDatabaseRecord(context, databaseName, out _)) { if (databaseRecord == null) { return(null); // should not happen } var periodicBackupTaskIds = databaseRecord.PeriodicBackupsTaskIds; if (periodicBackupTaskIds == null || periodicBackupTaskIds.Count == 0) { return(null); // no backup } var lastBackup = DateTime.MinValue; foreach (var periodicBackupTaskId in periodicBackupTaskIds) { var status = PeriodicBackupRunner.GetBackupStatusFromCluster(serverStore, context, databaseName, periodicBackupTaskId); if (status == null) { continue; // we have a backup task but no backup was ever done } var currentLatestBackup = LastBackupDate(status.LastFullBackup, status.LastIncrementalBackup); if (currentLatestBackup > lastBackup) { lastBackup = currentLatestBackup; } } return(lastBackup);
public DynamicJsonValue GenerateDatabaseInfo() { var envs = GetAllStoragesEnvironment().ToList(); if (envs.Any(x => x.Environment == null)) { return(null); } var size = new Size(envs.Sum(env => env.Environment.Stats().AllocatedDataFileSizeInBytes)); var databaseInfo = new DynamicJsonValue { [nameof(DatabaseInfo.HasRevisionsConfiguration)] = DocumentsStorage.RevisionsStorage.Configuration != null, [nameof(DatabaseInfo.HasExpirationConfiguration)] = ExpiredDocumentsCleaner != null, [nameof(DatabaseInfo.IsAdmin)] = true, //TODO: implement me! [nameof(DatabaseInfo.IsEncrypted)] = DocumentsStorage.Environment.Options.EncryptionEnabled, [nameof(DatabaseInfo.Name)] = Name, [nameof(DatabaseInfo.Disabled)] = false, //TODO: this value should be overwritten by the studio since it is cached [nameof(DatabaseInfo.TotalSize)] = new DynamicJsonValue { [nameof(Size.HumaneSize)] = size.HumaneSize, [nameof(Size.SizeInBytes)] = size.SizeInBytes }, [nameof(DatabaseInfo.IndexingErrors)] = IndexStore.GetIndexes().Sum(index => index.GetErrorCount()), [nameof(DatabaseInfo.Alerts)] = NotificationCenter.GetAlertCount(), [nameof(DatabaseInfo.UpTime)] = null, //it is shutting down [nameof(DatabaseInfo.BackupInfo)] = PeriodicBackupRunner.GetBackupInfo(), [nameof(DatabaseInfo.DocumentsCount)] = DocumentsStorage.GetNumberOfDocuments(), [nameof(DatabaseInfo.IndexesCount)] = IndexStore.GetIndexes().Count(), [nameof(DatabaseInfo.RejectClients)] = false, //TODO: implement me! [nameof(DatabaseInfo.IndexingStatus)] = IndexStore.Status.ToString(), ["CachedDatabaseInfo"] = true }; return(databaseInfo); }
private void InitializeFromDatabaseRecord(DatabaseRecord record) { if (record == null) { return; } ClientConfiguration = record.Client; DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(record); ExpiredDocumentsCleaner = ExpiredDocumentsCleaner.LoadConfigurations(this, record, ExpiredDocumentsCleaner); PeriodicBackupRunner.UpdateConfigurations(record); }
public void Initialize(InitializeOptions options = InitializeOptions.None) { try { NotificationCenter.Initialize(this); DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId); TxMerger.Start(); ConfigurationStorage.Initialize(); if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord) { return; } long index; DatabaseRecord record; using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) record = _serverStore.Cluster.ReadDatabase(context, Name, out index); if (record == null) { DatabaseDoesNotExistException.Throw(Name); } PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore); _indexStoreTask = IndexStore.InitializeAsync(record); ReplicationLoader?.Initialize(record); EtlLoader.Initialize(record); DocumentTombstoneCleaner.Start(); try { _indexStoreTask.Wait(DatabaseShutdown); } finally { _indexStoreTask = null; } SubscriptionStorage.Initialize(); NotifyFeaturesAboutStateChange(record, index); } catch (Exception) { Dispose(); throw; } }
public void RunIdleOperations() { if (Monitor.TryEnter(_idleLocker) == false) { return; } try { _lastIdleTicks = DateTime.UtcNow.Ticks; IndexStore?.RunIdleOperations(); Operations?.CleanupOperations(); PeriodicBackupRunner?.RemoveInactiveCompletedTasks(); } finally { Monitor.Exit(_idleLocker); } }
private static DateTime?GetLastBackup(TransactionOperationContext context, ServerStore serverStore, string databaseName) { using (var databaseRecord = serverStore.Cluster.ReadRawDatabaseRecord(context, databaseName, out _)) { if (databaseRecord == null) { return(null); // should not happen } var periodicBackupTaskIds = databaseRecord.GetPeriodicBackupsTaskIds(); if (periodicBackupTaskIds == null || periodicBackupTaskIds.Count == 0) { return(null); // no backup } foreach (var periodicBackupTaskId in periodicBackupTaskIds) { var status = PeriodicBackupRunner.GetBackupStatusFromCluster(serverStore, context, databaseName, periodicBackupTaskId); if (status == null) { return(DateTime.MinValue); // we have a backup task but no backup was ever done? } if (status.LastFullBackup == null) { return(DateTime.MinValue); // we never did a full backup } if (status.LastIncrementalBackup != null) { return(status.LastFullBackup > status.LastIncrementalBackup ? status.LastFullBackup : status.LastIncrementalBackup); } return(status.LastFullBackup); } return(null); } }
private static async Task CheckBackupOperationStatus(OperationStatus expected, OperationStatus actual, DocumentStore store, long taskId, long opId, PeriodicBackupRunner periodicBackupRunner) { if (expected == OperationStatus.Completed && actual == OperationStatus.Faulted) { // gather debug info var operation = new GetPeriodicBackupStatusOperation(taskId); var status = (await store.Maintenance.SendAsync(operation)).Status; TryGetBackupStatusFromPeriodicBackupAndPrint(expected, actual, opId, periodicBackupRunner, status, result: null); Assert.True(false, $"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Backup status from storage for current operation id: '{opId}':{Environment.NewLine}" + PrintBackupStatus(status)); } else if (expected == OperationStatus.Completed && actual == OperationStatus.InProgress) { // backup didn't complete in time, try to print running backup status, and backup result var pb = periodicBackupRunner?.PeriodicBackups.FirstOrDefault(x => x.RunningBackupStatus != null && x.BackupStatus.TaskId == taskId); if (pb == null) { // print previous backup status saved in memory var operation = new GetPeriodicBackupStatusOperation(taskId); var status = (await store.Maintenance.SendAsync(operation)).Status; Assert.True(false, $"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Could not fetch running backup status for current task id: '{taskId}', previous backup status:{Environment.NewLine}" + PrintBackupStatus(status)); } else { Assert.True(false, $"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Running backup status for current task id: '{taskId}':{Environment.NewLine}" + PrintBackupStatus(pb.RunningBackupStatus)); } } }
public unsafe void Dispose() { if (_databaseShutdown.IsCancellationRequested) { return; // double dispose? } lock (this) { if (_databaseShutdown.IsCancellationRequested) { return; // double dispose? } //before we dispose of the database we take its latest info to be displayed in the studio try { var databaseInfo = GenerateDatabaseInfo(); if (databaseInfo != null) { DatabaseInfoCache?.InsertDatabaseInfo(databaseInfo, Name); } } catch (Exception e) { // if we encountered a catastrophic failure we might not be able to retrieve database info if (_logger.IsInfoEnabled) { _logger.Info("Failed to generate and store database info", e); } } _databaseShutdown.Cancel(); // we'll wait for 1 minute to drain all the requests // from the database var sp = Stopwatch.StartNew(); while (sp.ElapsedMilliseconds < 60 * 1000) { if (Interlocked.Read(ref _usages) == 0) { break; } if (_waitForUsagesOnDisposal.Wait(1000)) { _waitForUsagesOnDisposal.Reset(); } } var exceptionAggregator = new ExceptionAggregator(_logger, $"Could not dispose {nameof(DocumentDatabase)} {Name}"); foreach (var connection in RunningTcpConnections) { exceptionAggregator.Execute(() => { connection.Dispose(); }); } exceptionAggregator.Execute(() => { TxMerger?.Dispose(); }); if (_indexStoreTask != null) { exceptionAggregator.Execute(() => { _indexStoreTask.Wait(DatabaseShutdown); }); } exceptionAggregator.Execute(() => { IndexStore?.Dispose(); }); exceptionAggregator.Execute(() => { ExpiredDocumentsCleaner?.Dispose(); }); exceptionAggregator.Execute(() => { PeriodicBackupRunner?.Dispose(); }); exceptionAggregator.Execute(() => { DocumentTombstoneCleaner?.Dispose(); }); exceptionAggregator.Execute(() => { ReplicationLoader?.Dispose(); }); exceptionAggregator.Execute(() => { EtlLoader?.Dispose(); }); exceptionAggregator.Execute(() => { Operations?.Dispose(exceptionAggregator); }); exceptionAggregator.Execute(() => { NotificationCenter?.Dispose(); }); exceptionAggregator.Execute(() => { SubscriptionStorage?.Dispose(); }); exceptionAggregator.Execute(() => { ConfigurationStorage?.Dispose(); }); exceptionAggregator.Execute(() => { DocumentsStorage?.Dispose(); }); exceptionAggregator.Execute(() => { _databaseShutdown.Dispose(); }); exceptionAggregator.Execute(() => { if (MasterKey == null) { return; } fixed(byte *pKey = MasterKey) { Sodium.sodium_memzero(pKey, (UIntPtr)MasterKey.Length); } }); exceptionAggregator.ThrowIfNeeded(); } }
public void Initialize(InitializeOptions options = InitializeOptions.None) { try { _addToInitLog("Initializing NotificationCenter"); NotificationCenter.Initialize(this); _addToInitLog("Initializing DocumentStorage"); DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId); _addToInitLog("Starting Transaction Merger"); TxMerger.Start(); _addToInitLog("Initializing ConfigurationStorage"); ConfigurationStorage.Initialize(); if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord) { return; } _addToInitLog("Loading Database"); long index; DatabaseRecord record; using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) record = _serverStore.Cluster.ReadDatabase(context, Name, out index); if (record == null) { DatabaseDoesNotExistException.Throw(Name); } PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore); _addToInitLog("Initializing IndexStore (async)"); _indexStoreTask = IndexStore.InitializeAsync(record); _addToInitLog("Initializing Replication"); ReplicationLoader?.Initialize(record); _addToInitLog("Initializing ETL"); EtlLoader.Initialize(record); DocumentTombstoneCleaner.Start(); try { _indexStoreTask.Wait(DatabaseShutdown); } finally { _addToInitLog("Initializing IndexStore completed"); _indexStoreTask = null; } SubscriptionStorage.Initialize(); _addToInitLog("Initializing SubscriptionStorage completed"); TaskExecutor.Execute((state) => { try { NotifyFeaturesAboutStateChange(record, index); } catch { // We ignore the exception since it was caught in the function itself } }, null); } catch (Exception) { Dispose(); throw; } }
private static void TryGetBackupStatusFromPeriodicBackupAndPrint(OperationStatus expected, OperationStatus actual, long opId, PeriodicBackupRunner periodicBackupRunner, PeriodicBackupStatus status, BackupResult result) { if (status?.LastOperationId != opId) { // failed to save backup status, lets fetch it from memory var pb = periodicBackupRunner?.PeriodicBackups.FirstOrDefault(x => x.BackupStatus != null && x.BackupStatus.LastOperationId == opId); if (pb == null) { Assert.True(false, $"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Could not fetch backup status for current operation id: '{opId}', previous backup status:{Environment.NewLine}" + PrintBackupStatus(status) + Environment.NewLine + "BackupResult Messages:" + Environment.NewLine + PrintBackupResultMessagesStatus(result)); } else { Assert.True(false, $"Backup status expected: '{expected}', actual '{actual}',{Environment.NewLine}Could not fetch backup status from storage for current operation id: '{opId}', current in memory backup status:{Environment.NewLine}" + PrintBackupStatus(pb.BackupStatus) + Environment.NewLine + "BackupResult Messages:" + Environment.NewLine + PrintBackupResultMessagesStatus(result)); } } }