private void ScheduleNextBackup(PeriodicBackup periodicBackup)
{
try
{
_serverStore.ConcurrentBackupsCounter.FinishBackup();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
catch (Exception e)
{
var message = $"Failed to schedule next backup for task: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Warning,
details: new ExceptionDetails(e)));
}
}
private void AddAlertOnFailureToReachOtherSide(string msg, Exception e)
{
_database.NotificationCenter.Add(
AlertRaised.Create(
_database.Name,
AlertTitle, msg, AlertType.Replication, NotificationSeverity.Warning, key: FromToString, details: new ExceptionDetails(e)));
}
public RelationalDatabaseWriter(SqlEtl etl, DocumentDatabase database)
: base(etl.Configuration.FactoryName)
{
_etl = etl;
_database = database;
_logger = LoggingSource.Instance.GetLogger <RelationalDatabaseWriter>(_database.Name);
_providerFactory = GetDbProviderFactory(etl.Configuration);
_commandBuilder = _providerFactory.InitializeCommandBuilder();
_connection = _providerFactory.CreateConnection();
_connection.ConnectionString = etl.Configuration.Connection.ConnectionString;
try
{
_connection.Open();
}
catch (Exception e)
{
database.NotificationCenter.Add(AlertRaised.Create(
database.Name,
SqlEtl.SqlEtlTag,
$"SQL ETL could not open connection to {_connection.ConnectionString}",
AlertType.SqlEtl_ConnectionError,
NotificationSeverity.Error,
key: _connection.ConnectionString,
details: new ExceptionDetails(e)));
throw;
}
_tx = _connection.BeginTransaction();
_stringParserList = GenerateStringParsers();
}
private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs)
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (_database.PreventFromUnloading())
{
tcs.SetResult(backupTask.RunPeriodicBackup(onProgress));
}
}
catch (OperationCanceledException)
{
tcs.SetCanceled();
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e);
}
tcs.SetException(e);
}
finally
{
try
{
_serverStore.ConcurrentBackupsSemaphore.Release();
periodicBackup.RunningTask = null;
periodicBackup.RunningBackupTaskId = null;
periodicBackup.CancelToken = null;
periodicBackup.RunningBackupStatus = null;
if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
{
var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
}
}
catch (Exception e)
{
var msg = $"Failed to schedule next backup for backup thread: '{periodicBackup.Configuration.Name}'";
if (_logger.IsOperationsEnabled)
{
_logger.Operations(msg, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup.",
msg,
AlertType.PeriodicBackup,
NotificationSeverity.Warning,
details: new ExceptionDetails(e)));
}
}
}
public void RecordLoadError(Exception e, int count = 1)
{
LoadErrors += count;
LastErrorTime = SystemTime.UtcNow;
LastAlert = AlertRaised.Create(_processType,
$"[{_name}] Write error: {e.Message}",
AlertType.Etl_LoadError,
NotificationSeverity.Error,
key: _name, details: new ExceptionDetails(e));
if (LoadErrors < 100)
{
return;
}
if (LoadErrors <= LoadSuccesses)
{
return;
}
var message = $"[{_name}] Write error hit ratio too high. Could not tolerate write error ratio and stopped current ETL cycle";
LastAlert = AlertRaised.Create(_processType,
message,
AlertType.Etl_WriteErrorRatio,
NotificationSeverity.Error,
key: _name,
details: new ExceptionDetails(e));
_notificationCenter.Add(LastAlert);
throw new InvalidOperationException($"{message}. Current stats: {this}", e);
}
private DbProviderFactory GetDbProviderFactory(SqlEtlConfiguration configuration)
{
DbProviderFactory providerFactory;
try
{
providerFactory = DbProviderFactories.GetFactory(configuration.FactoryName);
}
catch (Exception e)
{
var message = $"Could not find provider factory {configuration.FactoryName} to replicate to sql for {configuration.Name}, ignoring.";
if (_logger.IsInfoEnabled)
{
_logger.Info(message, e);
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
SqlEtl.SqlEtlTag,
message,
AlertType.SqlEtl_ProviderError,
NotificationSeverity.Error,
details: new ExceptionDetails(e)));
throw;
}
return(providerFactory);
}
private void HandleOnRecoveryError(StorageEnvironmentWithType.StorageEnvironmentType type, string resourceName, object environment, RecoveryErrorEventArgs e)
{
NotificationCenter.NotificationCenter nc;
string title;
switch (type)
{
case StorageEnvironmentWithType.StorageEnvironmentType.Configuration:
case StorageEnvironmentWithType.StorageEnvironmentType.Documents:
nc = _serverStore?.NotificationCenter;
title = $"Database Recovery Error - {resourceName ?? "Unknown Database"}";
if (type == StorageEnvironmentWithType.StorageEnvironmentType.Configuration)
{
title += " (configuration storage)";
}
break;
case StorageEnvironmentWithType.StorageEnvironmentType.Index:
nc = NotificationCenter;
title = $"Index Recovery Error - {resourceName ?? "Unknown Index"}";
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type.ToString());
}
nc?.Add(AlertRaised.Create(Name,
title,
$"{e.Message}{Environment.NewLine}{Environment.NewLine}Environment: {environment}",
AlertType.RecoveryError,
NotificationSeverity.Error,
key: resourceName));
}
private void AlertIfDocumentStoreCreationRateIsNotReasonable(string applicationIdentifier, string name)
{
var q = ServerStore.ClientCreationRate.GetOrCreate(applicationIdentifier);
var now = DateTime.UtcNow;
q.Enqueue(now);
while (q.Count > 20)
{
if (q.TryDequeue(out var last) && (now - last).TotalMinutes < 1)
{
q.Clear();
ServerStore.NotificationCenter.Add(
AlertRaised.Create(
name,
"Too many clients creations",
"There has been a lot of topology updates (more than 20) for the same client id in less than a minute. " +
$"Last one from ({HttpContext.Connection.RemoteIpAddress} as " +
$"{HttpContext.Connection.ClientCertificate?.FriendlyName ?? HttpContext.Connection.ClientCertificate?.Thumbprint ?? "<unsecured>"})" +
"This is usually an indication that you are creating a large number of DocumentStore instance. " +
"Are you creating a Document Store per request, instead of using DocumentStore as a singleton? ",
AlertType.HighClientCreationRate,
NotificationSeverity.Warning
));
}
}
}
internal void HandleOnRecoveryError(object sender, RecoveryErrorEventArgs e)
{
_serverStore?.NotificationCenter.Add(AlertRaised.Create($"Database Recovery Error - {Name ?? "Unknown Database"}",
e.Message,
AlertType.RecoveryError,
NotificationSeverity.Error,
Name));
}
public static TimeSeriesPolicyRunner LoadConfigurations(DocumentDatabase database, DatabaseRecord dbRecord, TimeSeriesPolicyRunner policyRunner)
{
try
{
if (dbRecord.TimeSeries?.Collections == null || dbRecord.TimeSeries.Collections.Count == 0)
{
policyRunner?.Dispose();
return null;
}
database.ServerStore.LicenseManager.AssertCanAddTimeSeriesRollupsAndRetention(dbRecord.TimeSeries);
if (policyRunner != null)
{
// no changes
if (policyRunner.Configuration.PolicyConfigurationChanged(dbRecord.TimeSeries) == false)
return policyRunner;
}
policyRunner?.Dispose();
var runner = new TimeSeriesPolicyRunner(database, dbRecord.TimeSeries);
runner.Start();
return runner;
}
catch (Exception e)
{
const string msg = "Cannot enable policy runner as the configuration record is not valid.";
if (e is LicenseLimitException lle)
{
LicenseLimitWarning.AddLicenseLimitNotification(database.ServerStore.NotificationCenter, lle);
}
else
{
database.NotificationCenter.Add(AlertRaised.Create(
database.Name,
$"Time series policy runner for database '{database.Name}' encountered an error", msg,
AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, database.Name));
}
var logger = LoggingSource.Instance.GetLogger<TimeSeriesPolicyRunner>(database.Name);
if (logger.IsOperationsEnabled)
logger.Operations(msg, e);
try
{
policyRunner?.Dispose();
}
catch (Exception ex)
{
if (logger.IsOperationsEnabled)
logger.Operations("Failed to dispose previous time-series policy runner", ex);
}
return null;
}
}
private static List <Notification> CreateSampleNotificationsForFilterOutTest()
{
return(new List <Notification>
{
AlertRaised.Create(
null,
"DatabaseTopologyWarning",
"DatabaseTopologyWarning_MSG",
AlertType.DatabaseTopologyWarning,
NotificationSeverity.Info),
DatabaseChanged.Create(null, DatabaseChangeType.Put), // filtered out, DatabaseChange
AlertRaised.Create(
null,
"LicenseManager_AGPL3",
"LicenseManager_AGPL3_MSG",
AlertType.ClusterTransactionFailure,
NotificationSeverity.Info),
AlertRaised.Create(
null,
"LicenseManager_AGPL3",
"LicenseManager_AGPL3_MSG",
AlertType.LicenseManager_AGPL3, // filtered out explicitly
NotificationSeverity.Info),
AlertRaised.Create(
null,
"RevisionsConfigurationNotValid",
"RevisionsConfigurationNotValid_MSG",
AlertType.RevisionsConfigurationNotValid, // filtered out explicitly
NotificationSeverity.Info),
AlertRaised.Create(
null,
"Certificates_ReplaceError",
"Certificates_ReplaceError_MSG",
AlertType.Certificates_ReplaceError,
NotificationSeverity.Info),
PerformanceHint.Create(
null,
"SlowIO",
"SlowIO_MSG",
PerformanceHintType.SlowIO, // filtered out, PerformanceHint
NotificationSeverity.Info,
"test"),
PerformanceHint.Create(
null,
"SqlEtl_SlowSql",
"SqlEtl_SlowSql_MSG",
PerformanceHintType.SqlEtl_SlowSql, // filtered out, PerformanceHint
NotificationSeverity.Info,
"test"),
OperationChanged.Create(null, 1, new Operations.OperationDescription(), new OperationState()
{
Result = new PersistableResult()
}, false),
DatabaseChanged.Create(null, DatabaseChangeType.Delete) // filtered out, DatabaseChange
});
}
public void Execute(string databaseName, Exception e, Guid environmentId)
{
var stats = _errorsPerEnvironment.GetOrAdd(environmentId, x => FailureStats.Create(MaxDatabaseUnloads));
if (stats.WillUnloadDatabase == false)
{
if (DateTime.UtcNow - stats.LastUnloadTime > NoFailurePeriod)
{
// let it unload again after it was working fine for a given time with no failure
stats.NumberOfUnloads = 0;
stats.LastUnloadTime = DateTime.MinValue;
}
else
{
return;
}
}
stats.DatabaseUnloadTask = Task.Run(async() =>
{
var title = $"Critical error in '{databaseName}'";
const string message = "Database is about to be unloaded due to an encountered error";
try
{
_serverStore.NotificationCenter.Add(AlertRaised.Create(
databaseName,
title,
message,
AlertType.CatastrophicDatabaseFailure,
NotificationSeverity.Error,
key: databaseName,
details: new ExceptionDetails(e)));
}
catch (Exception)
{
// exception in raising an alert can't prevent us from unloading a database
}
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"{title}. {message}", e);
}
// let it propagate the exception to the client first and do
// the internal failure handling e.g. Index.HandleIndexCorruption
await Task.Delay(TimeToWaitBeforeUnloadingDatabase);
stats.NumberOfUnloads++;
stats.LastUnloadTime = DateTime.UtcNow;
(await _databasesLandlord.UnloadAndLockDatabase(databaseName, "CatastrophicFailure"))?.Dispose();
stats.DatabaseUnloadTask = null;
});
}
private static AlertRaised GetSampleAlert(string customMessage = null, string customKey = null)
{
return(AlertRaised.Create(
"title",
customMessage ?? "Alert #1",
0, //use any type
NotificationSeverity.Info,
key: customKey ?? "Key",
details: new ExceptionDetails(new Exception("Error message"))));
}
private void HandleSlowSql(long elapsedMiliseconds, string stmt)
{
var message = $"[{_etl.Name}] Slow SQL detected. Execution took: {elapsedMiliseconds}ms, statement: {stmt}";
if (_logger.IsInfoEnabled)
{
_logger.Info(message);
}
_database.NotificationCenter.Add(AlertRaised.Create(_database.Name, _etl.Tag, message, AlertType.SqlEtl_SlowSql, NotificationSeverity.Warning));
}
public void InitializeFromDatabaseRecord(DatabaseRecord dbRecord)
{
try
{
var revisions = dbRecord.Revisions;
if (revisions == null ||
(revisions.Default == null && revisions.Collections.Count == 0))
{
Configuration = null;
return;
}
if (revisions.Equals(Configuration))
{
return;
}
Configuration = revisions;
using (var tx = _database.DocumentsStorage.Environment.WriteTransaction())
{
foreach (var collection in Configuration.Collections)
{
if (collection.Value.Disabled)
{
continue;
}
EnsureRevisionTableCreated(tx, new CollectionName(collection.Key));
}
CreateTrees(tx);
tx.Commit();
}
if (_logger.IsInfoEnabled)
{
_logger.Info("Revisions configuration changed");
}
}
catch (Exception e)
{
var msg = "Cannot enable revisions for documents as the revisions configuration " +
"in the database record is missing or not valid.";
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Revisions error in {_database.Name}", msg,
AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, _database.Name));
if (_logger.IsOperationsEnabled)
{
_logger.Operations(msg, e);
}
}
}
public static void AddLicenseLimitNotification(ServerStore serverStore, LicenseLimit licenseLimit)
{
var alert = AlertRaised.Create(
"You've reached your license limit",
licenseLimit.Message,
AlertType.LicenseManager_LicenseLimit,
NotificationSeverity.Warning,
details: new LicenseLimitWarning(licenseLimit));
serverStore.NotificationCenter.Add(alert);
}
internal void HandleNonDurableFileSystemError(object sender, NonDurabilitySupportEventArgs e)
{
_serverStore?.NotificationCenter.Add(AlertRaised.Create($"Non Durable File System - {Name ?? "Unknown Database"}",
e.Message,
AlertType.NonDurableFileSystem,
NotificationSeverity.Warning,
Name,
details: new MessageDetails {
Message = e.Details
}));
}
public static void AddLicenseLimitNotification(NotificationCenter notificationCenter, LicenseLimitException licenseLimit)
{
var alert = AlertRaised.Create(
null,
$@"You've reached your license limit ({EnumHelper.GetDescription(licenseLimit.Type)})",
licenseLimit.Message,
AlertType.LicenseManager_LicenseLimit,
NotificationSeverity.Warning,
details: new LicenseLimitWarning(licenseLimit));
notificationCenter.Add(alert);
}
private void AddAlertOnFailureToReachOtherSide(string msg, Exception e)
{
using (_database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext))
using (var txw = configurationContext.OpenWriteTransaction())
{
_database.NotificationCenter.AddAfterTransactionCommit(
AlertRaised.Create(AlertTitle, msg, AlertType.Replication, NotificationSeverity.Warning, key: FromToString, details: new ExceptionDetails(e)),
txw);
txw.Commit();
}
}
private void OpenIndex(PathSetting path, string indexPath, List <Exception> exceptions, string name)
{
Index index = null;
try
{
index = Index.Open(indexPath, _documentDatabase);
index.Start();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Started {index.Name} from {indexPath}");
}
_indexes.Add(index);
}
catch (Exception e)
{
var alreadyFaulted = false;
if (index != null && _indexes.TryGetByName(index.Name, out Index i))
{
if (i is FaultyInMemoryIndex)
{
alreadyFaulted = true;
}
}
index?.Dispose();
exceptions?.Add(e);
if (alreadyFaulted)
{
return;
}
var configuration = new FaultyInMemoryIndexConfiguration(path, _documentDatabase.Configuration);
var fakeIndex = new FaultyInMemoryIndex(e, name, configuration);
var message = $"Could not open index at '{indexPath}'. Created in-memory, fake instance: {fakeIndex.Name}";
if (_logger.IsInfoEnabled)
{
_logger.Info(message, e);
}
_documentDatabase.NotificationCenter.Add(AlertRaised.Create(
_documentDatabase.Name,
"Indexes store initialization error",
message,
AlertType.IndexStore_IndexCouldNotBeOpened,
NotificationSeverity.Error,
key: fakeIndex.Name,
details: new ExceptionDetails(e)));
_indexes.Add(fakeIndex);
}
}
public static ExpiredDocumentsCleaner LoadConfigurations(DocumentDatabase database, DatabaseRecord dbRecord, ExpiredDocumentsCleaner expiredDocumentsCleaner)
{
try
{
if (dbRecord.Expiration == null && dbRecord.Refresh == null)
{
expiredDocumentsCleaner?.Dispose();
return(null);
}
if (expiredDocumentsCleaner != null)
{
// no changes
if (Equals(expiredDocumentsCleaner.ExpirationConfiguration, dbRecord.Expiration) &&
Equals(expiredDocumentsCleaner.RefreshConfiguration, dbRecord.Refresh))
{
return(expiredDocumentsCleaner);
}
}
expiredDocumentsCleaner?.Dispose();
var hasExpiration = dbRecord.Expiration?.Disabled == false;
var hasRefresh = dbRecord.Refresh?.Disabled == false;
if (hasExpiration == false && hasRefresh == false)
{
return(null);
}
var cleaner = new ExpiredDocumentsCleaner(database, dbRecord.Expiration, dbRecord.Refresh);
cleaner.Start();
return(cleaner);
}
catch (Exception e)
{
const string msg = "Cannot enable expired documents cleaner as the configuration record is not valid.";
database.NotificationCenter.Add(AlertRaised.Create(
database.Name,
$"Expiration error in {database.Name}", msg,
AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, database.Name));
var logger = LoggingSource.Instance.GetLogger <ExpiredDocumentsCleaner>(database.Name);
if (logger.IsOperationsEnabled)
{
logger.Operations(msg, e);
}
return(null);
}
}
private void LogConfigurationWarning <T>(EtlConfiguration <T> config, List <string> warnings) where T : ConnectionString
{
var warnMessage = $"Warning about ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " +
$"Reason{(warnings.Count > 1 ? "s" : string.Empty)}: {string.Join(";", warnings)}.";
if (Logger.IsInfoEnabled)
{
Logger.Info(warnMessage);
}
var alert = AlertRaised.Create(_database.Name, AlertTitle, warnMessage, AlertType.Etl_Warning, NotificationSeverity.Warning);
_database.NotificationCenter.Add(alert);
}
private void LogConfigurationError <T>(EtlConfiguration <T> config, List <string> errors) where T : ConnectionString
{
var errorMessage = $"Invalid ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " +
$"Reason{(errors.Count > 1 ? "s" : string.Empty)}: {string.Join(";", errors)}.";
if (Logger.IsInfoEnabled)
{
Logger.Info(errorMessage);
}
var alert = AlertRaised.Create(_database.Name, AlertTitle, errorMessage, AlertType.Etl_Error, NotificationSeverity.Error);
_database.NotificationCenter.Add(alert);
}
public void Initialize(StorageEnvironment environment, TransactionContextPool contextPool)
{
try
{
_licenseStorage.Initialize(environment, contextPool);
var firstServerStartDate = _licenseStorage.GetFirstServerStartDate();
if (firstServerStartDate == null)
{
firstServerStartDate = SystemTime.UtcNow;
_licenseStorage.SetFirstServerStartDate(firstServerStartDate.Value);
}
_licenseStatus.FirstServerStartDate = firstServerStartDate.Value;
var license = _serverStore.LoadLicense();
if (license == null)
{
return;
}
_leaseLicenseTimer = new Timer(state =>
AsyncHelpers.RunSync(LeaseLicense), null, 0, (int)TimeSpan.FromHours(24).TotalMilliseconds);
_licenseStatus.Attributes = LicenseValidator.Validate(license, RSAParameters);
_licenseStatus.Error = false;
_licenseStatus.Message = null;
}
catch (Exception e)
{
_licenseStatus.Attributes = null;
_licenseStatus.Error = true;
_licenseStatus.Message = e.Message;
if (Logger.IsInfoEnabled)
{
Logger.Info("Could not validate license", e);
}
var alert = AlertRaised.Create(
"License manager initialization error",
"Could not intitalize the license manager",
AlertType.LicenseManager_InitializationError,
NotificationSeverity.Warning,
details: new ExceptionDetails(e));
_serverStore.NotificationCenter.Add(alert);
}
}
private void RaiseNoLivingNodesAlert(string alertMsg)
{
var alert = AlertRaised.Create(
"No living nodes in the database topology",
alertMsg,
AlertType.DatabaseTopologyWarning,
NotificationSeverity.Warning
);
NotificationCenter.Add(alert);
if (_logger.IsOperationsEnabled)
{
_logger.Operations(alertMsg);
}
}
private void RaiseNoLivingNodesAlert(string alertMsg, string dbName)
{
var alert = AlertRaised.Create(
$"Could not reach any node of {dbName} database",
$"{alertMsg}. {ThingsToCheck}",
AlertType.DatabaseTopologyWarning,
NotificationSeverity.Warning
);
NotificationCenter.Add(alert);
if (_logger.IsOperationsEnabled)
{
_logger.Operations(alertMsg);
}
}
public void Add(StorageEnvironment environment, Exception exception)
{
var notificationsMetadata = _notificationsMetadataTable.GetOrCreateValue(environment);
if (notificationsMetadata.TryGetValue(exception.GetType(), out var notificationMetadata))
{
if (DateTime.Now - notificationMetadata.Time < _updateFrequency)
{
return;
}
if (Interlocked.CompareExchange(ref notificationMetadata.IsInProgress, 1, 0) == 1)
{
return;
}
notificationMetadata.Time = DateTime.Now;
}
else
{
notificationMetadata = new NotificationTime
{
Time = DateTime.Now,
IsInProgress = 1
};
if (notificationsMetadata.TryAdd(exception.GetType(), notificationMetadata) == false)
{
return;
}
//We are in low of memory so we want to minimize allocations
notificationMetadata.Key = $"{environment}:{exception.GetType()}";
notificationMetadata.Title = $"Out of memory occurred for '{environment}'";
}
var alert = AlertRaised.Create(
null,
notificationMetadata.Title,
exception.Message,
AlertType.OutOfMemoryException,
NotificationSeverity.Error,
notificationMetadata.Key,
OutOfMemoryDetails(exception));
_notificationsCenter.Add(alert);
Volatile.Write(ref notificationMetadata.IsInProgress, 0);
}
private void RaiseNodeNotFoundAlert(string alertMsg, string node)
{
var alert = AlertRaised.Create(
null,
$"Node {node} not found.",
$"{alertMsg}",
AlertType.DatabaseTopologyWarning,
NotificationSeverity.Warning
);
NotificationCenter.Add(alert);
if (_logger.IsOperationsEnabled)
{
_logger.Operations(alertMsg);
}
}
private TaskStatus GetTaskStatus(
DatabaseRecord databaseRecord,
PeriodicBackupConfiguration configuration,
bool skipErrorLog = false)
{
if (configuration.Disabled)
{
return(TaskStatus.Disabled);
}
if (configuration.HasBackup() == false)
{
if (skipErrorLog == false)
{
var message = $"All backup destinations are disabled for backup task id: {configuration.TaskId}";
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Periodic Backup",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Info));
}
return(TaskStatus.Disabled);
}
var backupStatus = GetBackupStatus(configuration.TaskId);
var whoseTaskIsIt = _database.WhoseTaskIsIt(databaseRecord.Topology, configuration, backupStatus, useLastResponsibleNodeIfNoAvailableNodes: true);
if (whoseTaskIsIt == null)
{
return(TaskStatus.Disabled);
}
if (whoseTaskIsIt == _serverStore.NodeTag)
{
return(TaskStatus.ActiveByCurrentNode);
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Backup job is skipped at {SystemTime.UtcNow}, because it is managed " +
$"by '{whoseTaskIsIt}' node and not the current node ({_serverStore.NodeTag})");
}
return(TaskStatus.ActiveByOtherNode);
}
private NextBackup GetNextBackupDetails(
PeriodicBackupConfiguration configuration,
PeriodicBackupStatus backupStatus,
bool skipErrorLog = false)
{
var now = SystemTime.UtcNow;
var lastFullBackup = backupStatus.LastFullBackupInternal ?? now;
var lastIncrementalBackup = backupStatus.LastIncrementalBackupInternal ?? backupStatus.LastFullBackupInternal ?? now;
var nextFullBackup = GetNextBackupOccurrence(configuration.FullBackupFrequency,
lastFullBackup, configuration, skipErrorLog: skipErrorLog);
var nextIncrementalBackup = GetNextBackupOccurrence(configuration.IncrementalBackupFrequency,
lastIncrementalBackup, configuration, skipErrorLog: skipErrorLog);
if (nextFullBackup == null && nextIncrementalBackup == null)
{
var message = "Couldn't schedule next backup " +
$"full backup frequency: {configuration.FullBackupFrequency}, " +
$"incremental backup frequency: {configuration.IncrementalBackupFrequency}";
if (string.IsNullOrWhiteSpace(configuration.Name) == false)
{
message += $", backup name: {configuration.Name}";
}
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
"Couldn't schedule next backup, this shouldn't happen",
message,
AlertType.PeriodicBackup,
NotificationSeverity.Warning));
return(null);
}
Debug.Assert(configuration.TaskId != 0);
var isFullBackup = IsFullBackup(backupStatus, configuration, nextFullBackup, nextIncrementalBackup);
var nextBackupDateTime = GetNextBackupDateTime(nextFullBackup, nextIncrementalBackup);
var nowLocalTime = now.ToLocalTime();
var nextBackupTimeSpan = (nextBackupDateTime - nowLocalTime).Ticks <= 0 ? TimeSpan.Zero : nextBackupDateTime - nowLocalTime;
return(new NextBackup
{
TimeSpan = nextBackupTimeSpan,
DateTime = DateTime.UtcNow.Add(nextBackupTimeSpan),
IsFull = isFullBackup
});
}