private async Task UpdateBackupStatusAsTimeout(StatefulService statefulService, string fabricUri, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updating the Backup status as timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); BackupPartitionStore backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService); using (ITransaction transaction = statefulService.StateManager.CreateTransaction()) { var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionStatus {0} as timeout for request Service Uri : {1} , PartitionId {2}", currentBackupPartitionStatus, this.ServiceUri, this.PartitionId); if ((currentBackupPartitionStatus?.BackupPartitionStatusState == BackupState.Accepted || currentBackupPartitionStatus?.BackupPartitionStatusState == BackupState.BackupInProgress) && this.BackupRequestGuid.Equals(currentBackupPartitionStatus.OperationId)) { var backupPartitionStatus = currentBackupPartitionStatus.ToBuilder() .WithState(BackupState.Timeout) .WithErrorCode(unchecked ((int)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TIMEOUT)) .WithMessage(StringResources.BackupTimeout) .Build(); await backupPartitionStore.UpdateValueAsync(fabricUri, backupPartitionStatus, timeout, cancellationToken, transaction); await transaction.CommitAsync(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup status as timeout failed for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } else { transaction.Abort(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updating the Backup status as timeout failed for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } } }
private async Task ProcessBackup(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Backup for request Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); var timeOutPersecCycle = 30000; var backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService); var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); BackupPartitionStatus backupPartitionStatus; do { var backupNowConfiguration = new BackupNowConfiguration() { StoreInformation = this.GetBackupStoreInformationFromBackupStorage(this.BackupStorage), OperationTimeoutMilliseconds = (int)this.BackupTimeout.TotalMilliseconds }; await UtilityHelper.InvokeWithRetry(async() => { await Program.ServiceAgent.BackupPartitionAsync(new Uri(UtilityHelper.GetUriFromCustomUri(this.ServiceUri)), Guid.Parse(this.PartitionId), this.BackupRequestGuid, backupNowConfiguration, timeout, cancellationToken); }); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Requested the BackupPartitionAsync Successfully for Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); await UtilityHelper.InvokeWithRetry(async() => { using (ITransaction transaction = statefulService.StateManager.CreateTransaction()) { var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction); if (currentBackupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted) { var toUpdateBackupPartitionStatus = currentBackupPartitionStatus.ToBuilder().WithState(BackupState.BackupInProgress).Build(); await backupPartitionStore.UpdateValueAsync(fabricUri, toUpdateBackupPartitionStatus, timeout, cancellationToken, transaction); await transaction.CommitAsync(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup partition request to BackupInProcss for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } else { transaction.Abort(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Could not Update the BackupPartition request to BackupInProcess for request Service Uri : {0} , PartitionId {1} because currentBackupPartitionStatus value is {2}", this.ServiceUri, this.PartitionId, currentBackupPartitionStatus); } } } ); await Task.Delay(timeOutPersecCycle, cancellationToken); backupPartitionStatus = await backupPartitionStore.GetValueAsync(fabricUri); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionstatusState {0},isCancellationRequested {1} for Service Uri : {2} , PartitionId {3}", backupPartitionStatus.BackupPartitionStatusState, cancellationToken.IsCancellationRequested, this.ServiceUri, this.PartitionId); } while ((backupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted || backupPartitionStatus.BackupPartitionStatusState == BackupState.BackupInProgress) && this.BackupRequestGuid.Equals(backupPartitionStatus.OperationId) && !cancellationToken.IsCancellationRequested); cancellationToken.ThrowIfCancellationRequested(); }