private void ScheduleNextBackup(PeriodicBackup periodicBackup) { try { _serverStore.ConcurrentBackupsCounter.FinishBackup(); periodicBackup.RunningTask = null; periodicBackup.RunningBackupTaskId = null; periodicBackup.CancelToken = null; periodicBackup.RunningBackupStatus = null; if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false) { var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus); periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true); } } catch (Exception e) { var message = $"Failed to schedule next backup for task: '{periodicBackup.Configuration.Name}'"; if (_logger.IsOperationsEnabled) { _logger.Operations(message, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, "Couldn't schedule next backup", message, AlertType.PeriodicBackup, NotificationSeverity.Warning, details: new ExceptionDetails(e))); } }
private void AddAlertOnFailureToReachOtherSide(string msg, Exception e) { _database.NotificationCenter.Add( AlertRaised.Create( _database.Name, AlertTitle, msg, AlertType.Replication, NotificationSeverity.Warning, key: FromToString, details: new ExceptionDetails(e))); }
public RelationalDatabaseWriter(SqlEtl etl, DocumentDatabase database) : base(etl.Configuration.FactoryName) { _etl = etl; _database = database; _logger = LoggingSource.Instance.GetLogger <RelationalDatabaseWriter>(_database.Name); _providerFactory = GetDbProviderFactory(etl.Configuration); _commandBuilder = _providerFactory.InitializeCommandBuilder(); _connection = _providerFactory.CreateConnection(); _connection.ConnectionString = etl.Configuration.Connection.ConnectionString; try { _connection.Open(); } catch (Exception e) { database.NotificationCenter.Add(AlertRaised.Create( database.Name, SqlEtl.SqlEtlTag, $"SQL ETL could not open connection to {_connection.ConnectionString}", AlertType.SqlEtl_ConnectionError, NotificationSeverity.Error, key: _connection.ConnectionString, details: new ExceptionDetails(e))); throw; } _tx = _connection.BeginTransaction(); _stringParserList = GenerateStringParsers(); }
private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs) { try { Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; NativeMemory.EnsureRegistered(); using (_database.PreventFromUnloading()) { tcs.SetResult(backupTask.RunPeriodicBackup(onProgress)); } } catch (OperationCanceledException) { tcs.SetCanceled(); } catch (Exception e) { if (_logger.IsOperationsEnabled) { _logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e); } tcs.SetException(e); } finally { try { _serverStore.ConcurrentBackupsSemaphore.Release(); periodicBackup.RunningTask = null; periodicBackup.RunningBackupTaskId = null; periodicBackup.CancelToken = null; periodicBackup.RunningBackupStatus = null; if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false) { var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus); periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true); } } catch (Exception e) { var msg = $"Failed to schedule next backup for backup thread: '{periodicBackup.Configuration.Name}'"; if (_logger.IsOperationsEnabled) { _logger.Operations(msg, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, "Couldn't schedule next backup.", msg, AlertType.PeriodicBackup, NotificationSeverity.Warning, details: new ExceptionDetails(e))); } } }
public void RecordLoadError(Exception e, int count = 1) { LoadErrors += count; LastErrorTime = SystemTime.UtcNow; LastAlert = AlertRaised.Create(_processType, $"[{_name}] Write error: {e.Message}", AlertType.Etl_LoadError, NotificationSeverity.Error, key: _name, details: new ExceptionDetails(e)); if (LoadErrors < 100) { return; } if (LoadErrors <= LoadSuccesses) { return; } var message = $"[{_name}] Write error hit ratio too high. Could not tolerate write error ratio and stopped current ETL cycle"; LastAlert = AlertRaised.Create(_processType, message, AlertType.Etl_WriteErrorRatio, NotificationSeverity.Error, key: _name, details: new ExceptionDetails(e)); _notificationCenter.Add(LastAlert); throw new InvalidOperationException($"{message}. Current stats: {this}", e); }
private DbProviderFactory GetDbProviderFactory(SqlEtlConfiguration configuration) { DbProviderFactory providerFactory; try { providerFactory = DbProviderFactories.GetFactory(configuration.FactoryName); } catch (Exception e) { var message = $"Could not find provider factory {configuration.FactoryName} to replicate to sql for {configuration.Name}, ignoring."; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, SqlEtl.SqlEtlTag, message, AlertType.SqlEtl_ProviderError, NotificationSeverity.Error, details: new ExceptionDetails(e))); throw; } return(providerFactory); }
private void HandleOnRecoveryError(StorageEnvironmentWithType.StorageEnvironmentType type, string resourceName, object environment, RecoveryErrorEventArgs e) { NotificationCenter.NotificationCenter nc; string title; switch (type) { case StorageEnvironmentWithType.StorageEnvironmentType.Configuration: case StorageEnvironmentWithType.StorageEnvironmentType.Documents: nc = _serverStore?.NotificationCenter; title = $"Database Recovery Error - {resourceName ?? "Unknown Database"}"; if (type == StorageEnvironmentWithType.StorageEnvironmentType.Configuration) { title += " (configuration storage)"; } break; case StorageEnvironmentWithType.StorageEnvironmentType.Index: nc = NotificationCenter; title = $"Index Recovery Error - {resourceName ?? "Unknown Index"}"; break; default: throw new ArgumentOutOfRangeException(nameof(type), type.ToString()); } nc?.Add(AlertRaised.Create(Name, title, $"{e.Message}{Environment.NewLine}{Environment.NewLine}Environment: {environment}", AlertType.RecoveryError, NotificationSeverity.Error, key: resourceName)); }
private void AlertIfDocumentStoreCreationRateIsNotReasonable(string applicationIdentifier, string name) { var q = ServerStore.ClientCreationRate.GetOrCreate(applicationIdentifier); var now = DateTime.UtcNow; q.Enqueue(now); while (q.Count > 20) { if (q.TryDequeue(out var last) && (now - last).TotalMinutes < 1) { q.Clear(); ServerStore.NotificationCenter.Add( AlertRaised.Create( name, "Too many clients creations", "There has been a lot of topology updates (more than 20) for the same client id in less than a minute. " + $"Last one from ({HttpContext.Connection.RemoteIpAddress} as " + $"{HttpContext.Connection.ClientCertificate?.FriendlyName ?? HttpContext.Connection.ClientCertificate?.Thumbprint ?? "<unsecured>"})" + "This is usually an indication that you are creating a large number of DocumentStore instance. " + "Are you creating a Document Store per request, instead of using DocumentStore as a singleton? ", AlertType.HighClientCreationRate, NotificationSeverity.Warning )); } } }
internal void HandleOnRecoveryError(object sender, RecoveryErrorEventArgs e) { _serverStore?.NotificationCenter.Add(AlertRaised.Create($"Database Recovery Error - {Name ?? "Unknown Database"}", e.Message, AlertType.RecoveryError, NotificationSeverity.Error, Name)); }
public static TimeSeriesPolicyRunner LoadConfigurations(DocumentDatabase database, DatabaseRecord dbRecord, TimeSeriesPolicyRunner policyRunner) { try { if (dbRecord.TimeSeries?.Collections == null || dbRecord.TimeSeries.Collections.Count == 0) { policyRunner?.Dispose(); return null; } database.ServerStore.LicenseManager.AssertCanAddTimeSeriesRollupsAndRetention(dbRecord.TimeSeries); if (policyRunner != null) { // no changes if (policyRunner.Configuration.PolicyConfigurationChanged(dbRecord.TimeSeries) == false) return policyRunner; } policyRunner?.Dispose(); var runner = new TimeSeriesPolicyRunner(database, dbRecord.TimeSeries); runner.Start(); return runner; } catch (Exception e) { const string msg = "Cannot enable policy runner as the configuration record is not valid."; if (e is LicenseLimitException lle) { LicenseLimitWarning.AddLicenseLimitNotification(database.ServerStore.NotificationCenter, lle); } else { database.NotificationCenter.Add(AlertRaised.Create( database.Name, $"Time series policy runner for database '{database.Name}' encountered an error", msg, AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, database.Name)); } var logger = LoggingSource.Instance.GetLogger<TimeSeriesPolicyRunner>(database.Name); if (logger.IsOperationsEnabled) logger.Operations(msg, e); try { policyRunner?.Dispose(); } catch (Exception ex) { if (logger.IsOperationsEnabled) logger.Operations("Failed to dispose previous time-series policy runner", ex); } return null; } }
private static List <Notification> CreateSampleNotificationsForFilterOutTest() { return(new List <Notification> { AlertRaised.Create( null, "DatabaseTopologyWarning", "DatabaseTopologyWarning_MSG", AlertType.DatabaseTopologyWarning, NotificationSeverity.Info), DatabaseChanged.Create(null, DatabaseChangeType.Put), // filtered out, DatabaseChange AlertRaised.Create( null, "LicenseManager_AGPL3", "LicenseManager_AGPL3_MSG", AlertType.ClusterTransactionFailure, NotificationSeverity.Info), AlertRaised.Create( null, "LicenseManager_AGPL3", "LicenseManager_AGPL3_MSG", AlertType.LicenseManager_AGPL3, // filtered out explicitly NotificationSeverity.Info), AlertRaised.Create( null, "RevisionsConfigurationNotValid", "RevisionsConfigurationNotValid_MSG", AlertType.RevisionsConfigurationNotValid, // filtered out explicitly NotificationSeverity.Info), AlertRaised.Create( null, "Certificates_ReplaceError", "Certificates_ReplaceError_MSG", AlertType.Certificates_ReplaceError, NotificationSeverity.Info), PerformanceHint.Create( null, "SlowIO", "SlowIO_MSG", PerformanceHintType.SlowIO, // filtered out, PerformanceHint NotificationSeverity.Info, "test"), PerformanceHint.Create( null, "SqlEtl_SlowSql", "SqlEtl_SlowSql_MSG", PerformanceHintType.SqlEtl_SlowSql, // filtered out, PerformanceHint NotificationSeverity.Info, "test"), OperationChanged.Create(null, 1, new Operations.OperationDescription(), new OperationState() { Result = new PersistableResult() }, false), DatabaseChanged.Create(null, DatabaseChangeType.Delete) // filtered out, DatabaseChange }); }
public void Execute(string databaseName, Exception e, Guid environmentId) { var stats = _errorsPerEnvironment.GetOrAdd(environmentId, x => FailureStats.Create(MaxDatabaseUnloads)); if (stats.WillUnloadDatabase == false) { if (DateTime.UtcNow - stats.LastUnloadTime > NoFailurePeriod) { // let it unload again after it was working fine for a given time with no failure stats.NumberOfUnloads = 0; stats.LastUnloadTime = DateTime.MinValue; } else { return; } } stats.DatabaseUnloadTask = Task.Run(async() => { var title = $"Critical error in '{databaseName}'"; const string message = "Database is about to be unloaded due to an encountered error"; try { _serverStore.NotificationCenter.Add(AlertRaised.Create( databaseName, title, message, AlertType.CatastrophicDatabaseFailure, NotificationSeverity.Error, key: databaseName, details: new ExceptionDetails(e))); } catch (Exception) { // exception in raising an alert can't prevent us from unloading a database } if (_logger.IsOperationsEnabled) { _logger.Operations($"{title}. {message}", e); } // let it propagate the exception to the client first and do // the internal failure handling e.g. Index.HandleIndexCorruption await Task.Delay(TimeToWaitBeforeUnloadingDatabase); stats.NumberOfUnloads++; stats.LastUnloadTime = DateTime.UtcNow; (await _databasesLandlord.UnloadAndLockDatabase(databaseName, "CatastrophicFailure"))?.Dispose(); stats.DatabaseUnloadTask = null; }); }
private static AlertRaised GetSampleAlert(string customMessage = null, string customKey = null) { return(AlertRaised.Create( "title", customMessage ?? "Alert #1", 0, //use any type NotificationSeverity.Info, key: customKey ?? "Key", details: new ExceptionDetails(new Exception("Error message")))); }
private void HandleSlowSql(long elapsedMiliseconds, string stmt) { var message = $"[{_etl.Name}] Slow SQL detected. Execution took: {elapsedMiliseconds}ms, statement: {stmt}"; if (_logger.IsInfoEnabled) { _logger.Info(message); } _database.NotificationCenter.Add(AlertRaised.Create(_database.Name, _etl.Tag, message, AlertType.SqlEtl_SlowSql, NotificationSeverity.Warning)); }
public void InitializeFromDatabaseRecord(DatabaseRecord dbRecord) { try { var revisions = dbRecord.Revisions; if (revisions == null || (revisions.Default == null && revisions.Collections.Count == 0)) { Configuration = null; return; } if (revisions.Equals(Configuration)) { return; } Configuration = revisions; using (var tx = _database.DocumentsStorage.Environment.WriteTransaction()) { foreach (var collection in Configuration.Collections) { if (collection.Value.Disabled) { continue; } EnsureRevisionTableCreated(tx, new CollectionName(collection.Key)); } CreateTrees(tx); tx.Commit(); } if (_logger.IsInfoEnabled) { _logger.Info("Revisions configuration changed"); } } catch (Exception e) { var msg = "Cannot enable revisions for documents as the revisions configuration " + "in the database record is missing or not valid."; _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, $"Revisions error in {_database.Name}", msg, AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, _database.Name)); if (_logger.IsOperationsEnabled) { _logger.Operations(msg, e); } } }
public static void AddLicenseLimitNotification(ServerStore serverStore, LicenseLimit licenseLimit) { var alert = AlertRaised.Create( "You've reached your license limit", licenseLimit.Message, AlertType.LicenseManager_LicenseLimit, NotificationSeverity.Warning, details: new LicenseLimitWarning(licenseLimit)); serverStore.NotificationCenter.Add(alert); }
internal void HandleNonDurableFileSystemError(object sender, NonDurabilitySupportEventArgs e) { _serverStore?.NotificationCenter.Add(AlertRaised.Create($"Non Durable File System - {Name ?? "Unknown Database"}", e.Message, AlertType.NonDurableFileSystem, NotificationSeverity.Warning, Name, details: new MessageDetails { Message = e.Details })); }
public static void AddLicenseLimitNotification(NotificationCenter notificationCenter, LicenseLimitException licenseLimit) { var alert = AlertRaised.Create( null, $@"You've reached your license limit ({EnumHelper.GetDescription(licenseLimit.Type)})", licenseLimit.Message, AlertType.LicenseManager_LicenseLimit, NotificationSeverity.Warning, details: new LicenseLimitWarning(licenseLimit)); notificationCenter.Add(alert); }
private void AddAlertOnFailureToReachOtherSide(string msg, Exception e) { using (_database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext)) using (var txw = configurationContext.OpenWriteTransaction()) { _database.NotificationCenter.AddAfterTransactionCommit( AlertRaised.Create(AlertTitle, msg, AlertType.Replication, NotificationSeverity.Warning, key: FromToString, details: new ExceptionDetails(e)), txw); txw.Commit(); } }
private void OpenIndex(PathSetting path, string indexPath, List <Exception> exceptions, string name) { Index index = null; try { index = Index.Open(indexPath, _documentDatabase); index.Start(); if (_logger.IsInfoEnabled) { _logger.Info($"Started {index.Name} from {indexPath}"); } _indexes.Add(index); } catch (Exception e) { var alreadyFaulted = false; if (index != null && _indexes.TryGetByName(index.Name, out Index i)) { if (i is FaultyInMemoryIndex) { alreadyFaulted = true; } } index?.Dispose(); exceptions?.Add(e); if (alreadyFaulted) { return; } var configuration = new FaultyInMemoryIndexConfiguration(path, _documentDatabase.Configuration); var fakeIndex = new FaultyInMemoryIndex(e, name, configuration); var message = $"Could not open index at '{indexPath}'. Created in-memory, fake instance: {fakeIndex.Name}"; if (_logger.IsInfoEnabled) { _logger.Info(message, e); } _documentDatabase.NotificationCenter.Add(AlertRaised.Create( _documentDatabase.Name, "Indexes store initialization error", message, AlertType.IndexStore_IndexCouldNotBeOpened, NotificationSeverity.Error, key: fakeIndex.Name, details: new ExceptionDetails(e))); _indexes.Add(fakeIndex); } }
public static ExpiredDocumentsCleaner LoadConfigurations(DocumentDatabase database, DatabaseRecord dbRecord, ExpiredDocumentsCleaner expiredDocumentsCleaner) { try { if (dbRecord.Expiration == null && dbRecord.Refresh == null) { expiredDocumentsCleaner?.Dispose(); return(null); } if (expiredDocumentsCleaner != null) { // no changes if (Equals(expiredDocumentsCleaner.ExpirationConfiguration, dbRecord.Expiration) && Equals(expiredDocumentsCleaner.RefreshConfiguration, dbRecord.Refresh)) { return(expiredDocumentsCleaner); } } expiredDocumentsCleaner?.Dispose(); var hasExpiration = dbRecord.Expiration?.Disabled == false; var hasRefresh = dbRecord.Refresh?.Disabled == false; if (hasExpiration == false && hasRefresh == false) { return(null); } var cleaner = new ExpiredDocumentsCleaner(database, dbRecord.Expiration, dbRecord.Refresh); cleaner.Start(); return(cleaner); } catch (Exception e) { const string msg = "Cannot enable expired documents cleaner as the configuration record is not valid."; database.NotificationCenter.Add(AlertRaised.Create( database.Name, $"Expiration error in {database.Name}", msg, AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, database.Name)); var logger = LoggingSource.Instance.GetLogger <ExpiredDocumentsCleaner>(database.Name); if (logger.IsOperationsEnabled) { logger.Operations(msg, e); } return(null); } }
private void LogConfigurationWarning <T>(EtlConfiguration <T> config, List <string> warnings) where T : ConnectionString { var warnMessage = $"Warning about ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " + $"Reason{(warnings.Count > 1 ? "s" : string.Empty)}: {string.Join(";", warnings)}."; if (Logger.IsInfoEnabled) { Logger.Info(warnMessage); } var alert = AlertRaised.Create(_database.Name, AlertTitle, warnMessage, AlertType.Etl_Warning, NotificationSeverity.Warning); _database.NotificationCenter.Add(alert); }
private void LogConfigurationError <T>(EtlConfiguration <T> config, List <string> errors) where T : ConnectionString { var errorMessage = $"Invalid ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " + $"Reason{(errors.Count > 1 ? "s" : string.Empty)}: {string.Join(";", errors)}."; if (Logger.IsInfoEnabled) { Logger.Info(errorMessage); } var alert = AlertRaised.Create(_database.Name, AlertTitle, errorMessage, AlertType.Etl_Error, NotificationSeverity.Error); _database.NotificationCenter.Add(alert); }
public void Initialize(StorageEnvironment environment, TransactionContextPool contextPool) { try { _licenseStorage.Initialize(environment, contextPool); var firstServerStartDate = _licenseStorage.GetFirstServerStartDate(); if (firstServerStartDate == null) { firstServerStartDate = SystemTime.UtcNow; _licenseStorage.SetFirstServerStartDate(firstServerStartDate.Value); } _licenseStatus.FirstServerStartDate = firstServerStartDate.Value; var license = _serverStore.LoadLicense(); if (license == null) { return; } _leaseLicenseTimer = new Timer(state => AsyncHelpers.RunSync(LeaseLicense), null, 0, (int)TimeSpan.FromHours(24).TotalMilliseconds); _licenseStatus.Attributes = LicenseValidator.Validate(license, RSAParameters); _licenseStatus.Error = false; _licenseStatus.Message = null; } catch (Exception e) { _licenseStatus.Attributes = null; _licenseStatus.Error = true; _licenseStatus.Message = e.Message; if (Logger.IsInfoEnabled) { Logger.Info("Could not validate license", e); } var alert = AlertRaised.Create( "License manager initialization error", "Could not intitalize the license manager", AlertType.LicenseManager_InitializationError, NotificationSeverity.Warning, details: new ExceptionDetails(e)); _serverStore.NotificationCenter.Add(alert); } }
private void RaiseNoLivingNodesAlert(string alertMsg) { var alert = AlertRaised.Create( "No living nodes in the database topology", alertMsg, AlertType.DatabaseTopologyWarning, NotificationSeverity.Warning ); NotificationCenter.Add(alert); if (_logger.IsOperationsEnabled) { _logger.Operations(alertMsg); } }
private void RaiseNoLivingNodesAlert(string alertMsg, string dbName) { var alert = AlertRaised.Create( $"Could not reach any node of {dbName} database", $"{alertMsg}. {ThingsToCheck}", AlertType.DatabaseTopologyWarning, NotificationSeverity.Warning ); NotificationCenter.Add(alert); if (_logger.IsOperationsEnabled) { _logger.Operations(alertMsg); } }
public void Add(StorageEnvironment environment, Exception exception) { var notificationsMetadata = _notificationsMetadataTable.GetOrCreateValue(environment); if (notificationsMetadata.TryGetValue(exception.GetType(), out var notificationMetadata)) { if (DateTime.Now - notificationMetadata.Time < _updateFrequency) { return; } if (Interlocked.CompareExchange(ref notificationMetadata.IsInProgress, 1, 0) == 1) { return; } notificationMetadata.Time = DateTime.Now; } else { notificationMetadata = new NotificationTime { Time = DateTime.Now, IsInProgress = 1 }; if (notificationsMetadata.TryAdd(exception.GetType(), notificationMetadata) == false) { return; } //We are in low of memory so we want to minimize allocations notificationMetadata.Key = $"{environment}:{exception.GetType()}"; notificationMetadata.Title = $"Out of memory occurred for '{environment}'"; } var alert = AlertRaised.Create( null, notificationMetadata.Title, exception.Message, AlertType.OutOfMemoryException, NotificationSeverity.Error, notificationMetadata.Key, OutOfMemoryDetails(exception)); _notificationsCenter.Add(alert); Volatile.Write(ref notificationMetadata.IsInProgress, 0); }
private void RaiseNodeNotFoundAlert(string alertMsg, string node) { var alert = AlertRaised.Create( null, $"Node {node} not found.", $"{alertMsg}", AlertType.DatabaseTopologyWarning, NotificationSeverity.Warning ); NotificationCenter.Add(alert); if (_logger.IsOperationsEnabled) { _logger.Operations(alertMsg); } }
private TaskStatus GetTaskStatus( DatabaseRecord databaseRecord, PeriodicBackupConfiguration configuration, bool skipErrorLog = false) { if (configuration.Disabled) { return(TaskStatus.Disabled); } if (configuration.HasBackup() == false) { if (skipErrorLog == false) { var message = $"All backup destinations are disabled for backup task id: {configuration.TaskId}"; _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, "Periodic Backup", message, AlertType.PeriodicBackup, NotificationSeverity.Info)); } return(TaskStatus.Disabled); } var backupStatus = GetBackupStatus(configuration.TaskId); var whoseTaskIsIt = _database.WhoseTaskIsIt(databaseRecord.Topology, configuration, backupStatus, useLastResponsibleNodeIfNoAvailableNodes: true); if (whoseTaskIsIt == null) { return(TaskStatus.Disabled); } if (whoseTaskIsIt == _serverStore.NodeTag) { return(TaskStatus.ActiveByCurrentNode); } if (_logger.IsInfoEnabled) { _logger.Info($"Backup job is skipped at {SystemTime.UtcNow}, because it is managed " + $"by '{whoseTaskIsIt}' node and not the current node ({_serverStore.NodeTag})"); } return(TaskStatus.ActiveByOtherNode); }
private NextBackup GetNextBackupDetails( PeriodicBackupConfiguration configuration, PeriodicBackupStatus backupStatus, bool skipErrorLog = false) { var now = SystemTime.UtcNow; var lastFullBackup = backupStatus.LastFullBackupInternal ?? now; var lastIncrementalBackup = backupStatus.LastIncrementalBackupInternal ?? backupStatus.LastFullBackupInternal ?? now; var nextFullBackup = GetNextBackupOccurrence(configuration.FullBackupFrequency, lastFullBackup, configuration, skipErrorLog: skipErrorLog); var nextIncrementalBackup = GetNextBackupOccurrence(configuration.IncrementalBackupFrequency, lastIncrementalBackup, configuration, skipErrorLog: skipErrorLog); if (nextFullBackup == null && nextIncrementalBackup == null) { var message = "Couldn't schedule next backup " + $"full backup frequency: {configuration.FullBackupFrequency}, " + $"incremental backup frequency: {configuration.IncrementalBackupFrequency}"; if (string.IsNullOrWhiteSpace(configuration.Name) == false) { message += $", backup name: {configuration.Name}"; } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, "Couldn't schedule next backup, this shouldn't happen", message, AlertType.PeriodicBackup, NotificationSeverity.Warning)); return(null); } Debug.Assert(configuration.TaskId != 0); var isFullBackup = IsFullBackup(backupStatus, configuration, nextFullBackup, nextIncrementalBackup); var nextBackupDateTime = GetNextBackupDateTime(nextFullBackup, nextIncrementalBackup); var nowLocalTime = now.ToLocalTime(); var nextBackupTimeSpan = (nextBackupDateTime - nowLocalTime).Ticks <= 0 ? TimeSpan.Zero : nextBackupDateTime - nowLocalTime; return(new NextBackup { TimeSpan = nextBackupTimeSpan, DateTime = DateTime.UtcNow.Add(nextBackupTimeSpan), IsFull = isFullBackup }); }