public override async Task <bool> Process(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { var timeoutDateTime = this.RequestDateTime.AddSeconds(this.BackupTimeout.TotalSeconds); var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); if (DateTime.Compare(DateTime.UtcNow, timeoutDateTime) > 0) { await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace)); return(true); } var timeoutSpan = timeoutDateTime.Subtract(DateTime.UtcNow); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "TimeoutSpan for Backup Partition WorkItem {0}", timeoutSpan); var cancellationTokenSource = new CancellationTokenSource(timeoutSpan); try { await this.ProcessBackup(statefulService, timeout, cancellationTokenSource.Token, processQueueTypeTrace); } catch (OperationCanceledException operationCanceledException) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Operation Cancellaed exception Details {0}", operationCanceledException.ToString()); await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace)); } return(true); }
private async Task <bool> IsPartitionBackupSuspended(StatefulService statefulService, string processQueueTypeTrace) { var suspendStore = await SuspendStore.CreateOrGetSuspendStatusStore(statefulService); var fabricKey = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); var containsSuspension = await suspendStore.GetValueAsync(fabricKey) ?? await suspendStore.GetValueAsync(ServiceUri) ?? await suspendStore.GetValueAsync(UtilityHelper.GetApplicationUriFromServiceUri(ServiceUri)); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "IsPartition Suspended {0}", containsSuspension != null); return(containsSuspension != null); }
private async Task <Tuple <BackupMapping, BackupPolicy> > GetEffectiveBackupPartitionAndPolicyForPartition( StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupPolicy backupPolicy = null; var backupMappingStore = await BackupMappingStore.CreateOrGetBackupMappingStore(statefulService); var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(statefulService); var backupMappingKey = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); var backupMapping = (await backupMappingStore.GetValueAsync(backupMappingKey, timeout, cancellationToken) ?? await backupMappingStore.GetValueAsync(this.ServiceUri, timeout, cancellationToken)) ?? await backupMappingStore.GetValueAsync(UtilityHelper.GetApplicationNameFromService(this.ServiceUri), timeout, cancellationToken); if (backupMapping != null) { backupPolicy = await backupPolicyStore.GetValueAsync(backupMapping.BackupPolicyName, timeout, cancellationToken); } BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "GetEffectiveBackupPartitionAndPolicyForPartition resulted as BackupMapping {0} ,BackupPolicy {1} ", backupMapping, backupPolicy); return(Tuple.Create(backupMapping, backupPolicy)); }
public override async Task <bool> Process(Microsoft.ServiceFabric.Services.Runtime.StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { WorkItem workItem = null; WorkItemQueue workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService); BackupMappingStore backupMappingStore = await BackupMappingStore.CreateOrGetBackupMappingStore(statefulService); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Resolving for {0} of type {1} ", this.ApplicationOrServiceUri, this.FabricBackupResourceType); try { if (this.FabricBackupResourceType == FabricBackupResourceType.ApplicationUri) { ServiceList serviceList = await FabricClientHelper.GetServiceList(this.ApplicationOrServiceUri); foreach (Query.Service service in serviceList) { if (service.ServiceKind == ServiceKind.Stateful && (service as StatefulService).HasPersistedState) { BackupMapping specificServiceBackupMapping = await backupMappingStore.GetValueAsync( UtilityHelper.GetBackupMappingKey(service.ServiceName.OriginalString, null)); if (specificServiceBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition) { ServicePartitionList servicePartitionList = await FabricClientHelper.GetPartitionList(service.ServiceName.OriginalString); foreach (Partition servicePartition in servicePartitionList) { BackupMapping specificPartitionBackupMapping = await backupMappingStore.GetValueAsync(UtilityHelper.GetBackupMappingKey( service.ServiceName.OriginalString, servicePartition.PartitionInformation.Id.ToString())); if (specificPartitionBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition) { //Send to Service Partition workItem = new SendToServiceNodeWorkItem (await UtilityHelper.GetCustomServiceUri(service.ServiceName.OriginalString, timeout, cancellationToken), servicePartition.PartitionInformation.Id.ToString(), this.WorkItemInfo); await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken); } } } } } } else if (this.FabricBackupResourceType == FabricBackupResourceType.ServiceUri) { ServicePartitionList servicePartitionList = await FabricClientHelper.GetPartitionList(this.ApplicationOrServiceUri); foreach (Partition servicePartition in servicePartitionList) { BackupMapping specificPartitionBackupMapping = await backupMappingStore.GetValueAsync(UtilityHelper.GetBackupMappingKey( this.ApplicationOrServiceUri, servicePartition.PartitionInformation.Id.ToString())); if (specificPartitionBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition) { //Send to Service Partition workItem = new SendToServiceNodeWorkItem (await UtilityHelper.GetCustomServiceUri(this.ApplicationOrServiceUri, timeout, cancellationToken), servicePartition.PartitionInformation.Id.ToString(), this.WorkItemInfo); await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken); } } } BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Resolving successful"); } catch (Exception exception) { AggregateException aggregateException = exception as AggregateException; if (aggregateException != null) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}", exception.StackTrace); foreach (Exception innerException in aggregateException.InnerExceptions) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Inner Exception : {0} , Message : {1} , Stack Trace : {2} ", innerException.InnerException, innerException.Message, innerException.StackTrace); } } else { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception : {0} , Message : {1} , Stack Trace : {2} ", exception.InnerException, exception.Message, exception.StackTrace); } return(false); } return(true); }
private async Task ProcessBackup(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Backup for request Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); var timeOutPersecCycle = 30000; var backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService); var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); BackupPartitionStatus backupPartitionStatus; do { var backupNowConfiguration = new BackupNowConfiguration() { StoreInformation = this.GetBackupStoreInformationFromBackupStorage(this.BackupStorage), OperationTimeoutMilliseconds = (int)this.BackupTimeout.TotalMilliseconds }; await UtilityHelper.InvokeWithRetry(async() => { await Program.ServiceAgent.BackupPartitionAsync(new Uri(UtilityHelper.GetUriFromCustomUri(this.ServiceUri)), Guid.Parse(this.PartitionId), this.BackupRequestGuid, backupNowConfiguration, timeout, cancellationToken); }); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Requested the BackupPartitionAsync Successfully for Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); await UtilityHelper.InvokeWithRetry(async() => { using (ITransaction transaction = statefulService.StateManager.CreateTransaction()) { var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction); if (currentBackupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted) { var toUpdateBackupPartitionStatus = currentBackupPartitionStatus.ToBuilder().WithState(BackupState.BackupInProgress).Build(); await backupPartitionStore.UpdateValueAsync(fabricUri, toUpdateBackupPartitionStatus, timeout, cancellationToken, transaction); await transaction.CommitAsync(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup partition request to BackupInProcss for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } else { transaction.Abort(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Could not Update the BackupPartition request to BackupInProcess for request Service Uri : {0} , PartitionId {1} because currentBackupPartitionStatus value is {2}", this.ServiceUri, this.PartitionId, currentBackupPartitionStatus); } } } ); await Task.Delay(timeOutPersecCycle, cancellationToken); backupPartitionStatus = await backupPartitionStore.GetValueAsync(fabricUri); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionstatusState {0},isCancellationRequested {1} for Service Uri : {2} , PartitionId {3}", backupPartitionStatus.BackupPartitionStatusState, cancellationToken.IsCancellationRequested, this.ServiceUri, this.PartitionId); } while ((backupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted || backupPartitionStatus.BackupPartitionStatusState == BackupState.BackupInProgress) && this.BackupRequestGuid.Equals(backupPartitionStatus.OperationId) && !cancellationToken.IsCancellationRequested); cancellationToken.ThrowIfCancellationRequested(); }
public async Task ProcessRestore(StatefulService statefulService, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Restore for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); WorkItemStore workItemStore = await WorkItemStore.CreateOrGetWorkItemStore(statefulService); RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService); string fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); PartitionDataLossProgress partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace); if (this.RestoreStatusStateModel == RestoreStatusStateModel.TriggerDataLoss) { if (partitionDataLossProgress == null) { bool isInitiateDataLossSuccess = await this.InitiateDataLoss(processQueueTypeTrace); if (isInitiateDataLossSuccess) { this.RestoreStatusStateModel = RestoreStatusStateModel.DataLossTriggered; } } await workItemStore.UpdateValueAsync(this.WorkItemGuid, this); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "RestoreStatusStateModel as DataLossTriggered for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } if (this.RestoreStatusStateModel == RestoreStatusStateModel.DataLossTriggered) { RestoreStatus restoreStatus = null; do { partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace); switch (partitionDataLossProgress.State) { case TestCommandProgressState.Running: case TestCommandProgressState.RollingBack: case TestCommandProgressState.Completed: break; case TestCommandProgressState.Cancelled: case TestCommandProgressState.Faulted: case TestCommandProgressState.ForceCancelled: this.DataLossGuid = Guid.NewGuid(); await workItemStore.UpdateValueAsync(this.WorkItemGuid, this); await this.InitiateDataLoss(processQueueTypeTrace); break; } await Task.Delay(Constants.CheckRestoreTimeSpan, cancellationToken); restoreStatus = await restoreStore.GetValueAsync(fabricUri); if (restoreStatus.RestoreStatusState == RestoreState.Timeout && restoreStatus.RestoreRequestGuid.Equals(this.RestoreRequestGuid)) { partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace); switch (partitionDataLossProgress.State) { case TestCommandProgressState.Running: case TestCommandProgressState.RollingBack: await this.CancelDataLoss(processQueueTypeTrace); break; case TestCommandProgressState.Cancelled: case TestCommandProgressState.Faulted: case TestCommandProgressState.ForceCancelled: case TestCommandProgressState.Completed: break; } } } while ((restoreStatus.RestoreStatusState == RestoreState.RestoreInProgress || restoreStatus.RestoreStatusState == RestoreState.Accepted) && this.RestoreRequestGuid.Equals(restoreStatus.RestoreRequestGuid) && !cancellationToken.IsCancellationRequested); cancellationToken.ThrowIfCancellationRequested(); } }