private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs) { try { Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; NativeMemory.EnsureRegistered(); using (_database.PreventFromUnloading()) { tcs.SetResult(backupTask.RunPeriodicBackup(onProgress)); } } catch (OperationCanceledException) { tcs.SetCanceled(); } catch (Exception e) { if (_logger.IsOperationsEnabled) { _logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e); } tcs.SetException(e); } finally { try { _serverStore.ConcurrentBackupsSemaphore.Release(); periodicBackup.RunningTask = null; periodicBackup.RunningBackupTaskId = null; periodicBackup.CancelToken = null; periodicBackup.RunningBackupStatus = null; if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false) { var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus); periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true); } } catch (Exception e) { var msg = $"Failed to schedule next backup for backup thread: '{periodicBackup.Configuration.Name}'"; if (_logger.IsOperationsEnabled) { _logger.Operations(msg, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, "Couldn't schedule next backup.", msg, AlertType.PeriodicBackup, NotificationSeverity.Warning, details: new ExceptionDetails(e))); } } }
private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs) { try { Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; NativeMemory.EnsureRegistered(); using (_database.PreventFromUnloading()) { tcs.SetResult(backupTask.RunPeriodicBackup(onProgress)); } } catch (OperationCanceledException) { tcs.SetCanceled(); } catch (Exception e) { if (_logger.IsOperationsEnabled) { _logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e); } tcs.SetException(e); } finally { ScheduleNextBackup(periodicBackup); } }
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup) { using (periodicBackup.UpdateBackupTask()) { try { if (periodicBackup.Disposed) { throw new InvalidOperationException("Backup task was already disposed"); } if (periodicBackup.RunningTask != null) { return(periodicBackup.RunningBackupTaskId ?? -1); } var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus); var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings); // check if we need to do a new full backup if (backupStatus.LastFullBackup == null || // no full backup was previously performed backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed backupStatus.LastEtag == null || // last document etag wasn't updated backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false) // the local folder already includes a full backup or snapshot { isFullBackup = true; } var operationId = _database.Operations.GetNextOperationId(); var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType); periodicBackup.StartTime = SystemTime.UtcNow; var backupTask = new BackupTask( _serverStore, _database, periodicBackup, isFullBackup, backupToLocalFolder, operationId, _tempBackupPath, _logger, _cancellationToken.Token); periodicBackup.RunningBackupTaskId = operationId; periodicBackup.CancelToken = backupTask.TaskCancelToken; var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'"; var task = _database.Operations.AddOperation( null, backupTaskName, Operations.Operations.OperationType.DatabaseBackup, taskFactory: onProgress => Task.Run(async() => { try { using (_database.PreventFromUnloading()) { return(await backupTask.RunPeriodicBackup(onProgress)); } } finally { periodicBackup.RunningTask = null; periodicBackup.RunningBackupTaskId = null; periodicBackup.CancelToken = null; periodicBackup.RunningBackupStatus = null; if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false) { var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus); periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true); } } }, backupTask.TaskCancelToken.Token), id: operationId, token: backupTask.TaskCancelToken); periodicBackup.RunningTask = task; task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose()); return(operationId); } catch (Exception e) { var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'"; if (_logger.IsOperationsEnabled) { _logger.Operations(message, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, $"Periodic Backup task: '{periodicBackup.Configuration.Name}'", message, AlertType.PeriodicBackup, NotificationSeverity.Error, details: new ExceptionDetails(e))); throw; } } }
private long CreateBackupTask(PeriodicBackup periodicBackup, bool isFullBackup) { using (periodicBackup.UpdateBackupTask()) { if (periodicBackup.Disposed) { throw new InvalidOperationException("Backup task was already disposed"); } if (periodicBackup.RunningTask != null) { return(periodicBackup.RunningBackupTaskId ?? -1); } if (_serverStore.Server.CpuCreditsBalance.BackgroundTasksAlertRaised.IsRaised()) { throw new BackupDelayException( $"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " + $"The task cannot run because the CPU credits allocated to this machine are nearing exhaustion.") { DelayPeriod = _serverStore.Configuration.Server.CpuCreditsExhaustionBackupDelay.AsTimeSpan }; } if (LowMemoryNotification.Instance.LowMemoryState) { throw new BackupDelayException( $"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " + $"The task cannot run because the server is in low memory state.") { DelayPeriod = _serverStore.Configuration.Backup.LowMemoryBackupDelay.AsTimeSpan }; } if (_serverStore.ConcurrentBackupsSemaphore.Wait(0) == false) { throw new BackupDelayException( $"Failed to start Backup Task: '{periodicBackup.Configuration.Name}'. " + $"The task exceeds the maximum number of concurrent backup tasks configured. " + $"Current value of Backup.MaxNumberOfConcurrentBackups is: {_serverStore.Configuration.Backup.MaxNumberOfConcurrentBackups:#,#;;0}") { DelayPeriod = TimeSpan.FromMinutes(1) }; } try { var backupStatus = periodicBackup.BackupStatus = GetBackupStatus(periodicBackup.Configuration.TaskId, periodicBackup.BackupStatus); var backupToLocalFolder = PeriodicBackupConfiguration.CanBackupUsing(periodicBackup.Configuration.LocalSettings); // check if we need to do a new full backup if (backupStatus.LastFullBackup == null || // no full backup was previously performed backupStatus.NodeTag != _serverStore.NodeTag || // last backup was performed by a different node backupStatus.BackupType != periodicBackup.Configuration.BackupType || // backup type has changed backupStatus.LastEtag == null || // last document etag wasn't updated backupToLocalFolder && BackupTask.DirectoryContainsBackupFiles(backupStatus.LocalBackup.BackupDirectory, IsFullBackupOrSnapshot) == false) // the local folder already includes a full backup or snapshot { isFullBackup = true; } var operationId = _database.Operations.GetNextOperationId(); var backupTypeText = GetBackupTypeText(isFullBackup, periodicBackup.Configuration.BackupType); periodicBackup.StartTime = SystemTime.UtcNow; var backupTask = new BackupTask( _serverStore, _database, periodicBackup, isFullBackup, backupToLocalFolder, operationId, _tempBackupPath, _logger, _cancellationToken.Token); periodicBackup.RunningBackupTaskId = operationId; periodicBackup.CancelToken = backupTask.TaskCancelToken; var backupTaskName = $"{backupTypeText} backup task: '{periodicBackup.Configuration.Name}'"; var task = _database.Operations.AddOperation( null, backupTaskName, Operations.Operations.OperationType.DatabaseBackup, taskFactory: onProgress => Task.Run(async() => { try { using (_database.PreventFromUnloading()) { return(await backupTask.RunPeriodicBackup(onProgress)); } } finally { _serverStore.ConcurrentBackupsSemaphore.Release(); periodicBackup.RunningTask = null; periodicBackup.RunningBackupTaskId = null; periodicBackup.CancelToken = null; periodicBackup.RunningBackupStatus = null; if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false) { var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus); periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true); } } }, backupTask.TaskCancelToken.Token), id: operationId, token: backupTask.TaskCancelToken); periodicBackup.RunningTask = task; task.ContinueWith(_ => backupTask.TaskCancelToken.Dispose()); return(operationId); } catch (Exception e) { // releasing the semaphore because we failed to start the backup task _serverStore.ConcurrentBackupsSemaphore.Release(); var message = $"Failed to start the backup task: '{periodicBackup.Configuration.Name}'"; if (_logger.IsOperationsEnabled) { _logger.Operations(message, e); } _database.NotificationCenter.Add(AlertRaised.Create( _database.Name, $"Periodic Backup task: '{periodicBackup.Configuration.Name}'", message, AlertType.PeriodicBackup, NotificationSeverity.Error, details: new ExceptionDetails(e))); throw; } } }