public override async Task <bool> Process(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { DateTime timeoutDateTime = this.RequestDateTime.AddSeconds(this.RestoreTimeout.TotalSeconds); string fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService); if (DateTime.Compare(DateTime.UtcNow, timeoutDateTime) > 0) { await UtilityHelper.InvokeWithRetry(() => this.UpdateRestoreStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace)); return(true); } TimeSpan timeoutSpan = timeoutDateTime.Subtract(DateTime.UtcNow); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "TimeoutSpan for Backup Partition WorkItem {0}", timeoutSpan); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(timeoutSpan); try { await this.ProcessRestore(statefulService, cancellationTokenSource.Token, processQueueTypeTrace); } catch (OperationCanceledException operationCanceledException) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Operation Canceled exception Details {0}", operationCanceledException.ToString()); await UtilityHelper.InvokeWithRetry(() => this.UpdateRestoreStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace)); } return(true); }
private async Task UpdateRestoreStatusAsTimeout(StatefulService statefulService, string fabricUri, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updating the Restore status as timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService); WorkItemQueue workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService); WorkItem workItem = new SendToServiceNodeWorkItem(this.ServiceUri, this.PartitionId, new WorkItemInfo() { WorkItemType = WorkItemPropogationType.UpdatePolicyAfterRestore }); using (ITransaction transaction = statefulService.StateManager.CreateTransaction()) { var currentRestoreStatus = await restoreStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "RestoreStatus {0} as timeout for request Service Uri : {1} , PartitionId {2}", currentRestoreStatus, this.ServiceUri, this.PartitionId); if (currentRestoreStatus != null && currentRestoreStatus.RestoreRequestGuid.Equals(this.RestoreRequestGuid) && (currentRestoreStatus.RestoreStatusState == RestoreState.Accepted || currentRestoreStatus.RestoreStatusState == RestoreState.RestoreInProgress)) { var restoreStatus = currentRestoreStatus.ToBuilder() .WithState(RestoreState.Timeout) .WithErrorCode(unchecked ((int)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TIMEOUT)) .WithMessage(StringResources.RestoreTimeout) .Build(); await restoreStore.UpdateValueAsync(fabricUri, restoreStatus, timeout, cancellationToken, transaction); await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken, transaction); await transaction.CommitAsync(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated Restore Status as Timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } else { transaction.Abort(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated Restore Status as Timeout Failed for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } } }
public async Task ProcessRestore(StatefulService statefulService, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Restore for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); WorkItemStore workItemStore = await WorkItemStore.CreateOrGetWorkItemStore(statefulService); RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService); string fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); PartitionDataLossProgress partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace); if (this.RestoreStatusStateModel == RestoreStatusStateModel.TriggerDataLoss) { if (partitionDataLossProgress == null) { bool isInitiateDataLossSuccess = await this.InitiateDataLoss(processQueueTypeTrace); if (isInitiateDataLossSuccess) { this.RestoreStatusStateModel = RestoreStatusStateModel.DataLossTriggered; } } await workItemStore.UpdateValueAsync(this.WorkItemGuid, this); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "RestoreStatusStateModel as DataLossTriggered for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } if (this.RestoreStatusStateModel == RestoreStatusStateModel.DataLossTriggered) { RestoreStatus restoreStatus = null; do { partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace); switch (partitionDataLossProgress.State) { case TestCommandProgressState.Running: case TestCommandProgressState.RollingBack: case TestCommandProgressState.Completed: break; case TestCommandProgressState.Cancelled: case TestCommandProgressState.Faulted: case TestCommandProgressState.ForceCancelled: this.DataLossGuid = Guid.NewGuid(); await workItemStore.UpdateValueAsync(this.WorkItemGuid, this); await this.InitiateDataLoss(processQueueTypeTrace); break; } await Task.Delay(Constants.CheckRestoreTimeSpan, cancellationToken); restoreStatus = await restoreStore.GetValueAsync(fabricUri); if (restoreStatus.RestoreStatusState == RestoreState.Timeout && restoreStatus.RestoreRequestGuid.Equals(this.RestoreRequestGuid)) { partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace); switch (partitionDataLossProgress.State) { case TestCommandProgressState.Running: case TestCommandProgressState.RollingBack: await this.CancelDataLoss(processQueueTypeTrace); break; case TestCommandProgressState.Cancelled: case TestCommandProgressState.Faulted: case TestCommandProgressState.ForceCancelled: case TestCommandProgressState.Completed: break; } } } while ((restoreStatus.RestoreStatusState == RestoreState.RestoreInProgress || restoreStatus.RestoreStatusState == RestoreState.Accepted) && this.RestoreRequestGuid.Equals(restoreStatus.RestoreRequestGuid) && !cancellationToken.IsCancellationRequested); cancellationToken.ThrowIfCancellationRequested(); } }