public override async Task <bool> Process(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { var timeoutDateTime = this.RequestDateTime.AddSeconds(this.BackupTimeout.TotalSeconds); var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); if (DateTime.Compare(DateTime.UtcNow, timeoutDateTime) > 0) { await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace)); return(true); } var timeoutSpan = timeoutDateTime.Subtract(DateTime.UtcNow); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "TimeoutSpan for Backup Partition WorkItem {0}", timeoutSpan); var cancellationTokenSource = new CancellationTokenSource(timeoutSpan); try { await this.ProcessBackup(statefulService, timeout, cancellationTokenSource.Token, processQueueTypeTrace); } catch (OperationCanceledException operationCanceledException) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Operation Cancellaed exception Details {0}", operationCanceledException.ToString()); await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace)); } return(true); }
private async Task ProcessBackup(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Backup for request Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); var timeOutPersecCycle = 30000; var backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService); var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId); BackupPartitionStatus backupPartitionStatus; do { var backupNowConfiguration = new BackupNowConfiguration() { StoreInformation = this.GetBackupStoreInformationFromBackupStorage(this.BackupStorage), OperationTimeoutMilliseconds = (int)this.BackupTimeout.TotalMilliseconds }; await UtilityHelper.InvokeWithRetry(async() => { await Program.ServiceAgent.BackupPartitionAsync(new Uri(UtilityHelper.GetUriFromCustomUri(this.ServiceUri)), Guid.Parse(this.PartitionId), this.BackupRequestGuid, backupNowConfiguration, timeout, cancellationToken); }); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Requested the BackupPartitionAsync Successfully for Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); await UtilityHelper.InvokeWithRetry(async() => { using (ITransaction transaction = statefulService.StateManager.CreateTransaction()) { var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction); if (currentBackupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted) { var toUpdateBackupPartitionStatus = currentBackupPartitionStatus.ToBuilder().WithState(BackupState.BackupInProgress).Build(); await backupPartitionStore.UpdateValueAsync(fabricUri, toUpdateBackupPartitionStatus, timeout, cancellationToken, transaction); await transaction.CommitAsync(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup partition request to BackupInProcss for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId); } else { transaction.Abort(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Could not Update the BackupPartition request to BackupInProcess for request Service Uri : {0} , PartitionId {1} because currentBackupPartitionStatus value is {2}", this.ServiceUri, this.PartitionId, currentBackupPartitionStatus); } } } ); await Task.Delay(timeOutPersecCycle, cancellationToken); backupPartitionStatus = await backupPartitionStore.GetValueAsync(fabricUri); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionstatusState {0},isCancellationRequested {1} for Service Uri : {2} , PartitionId {3}", backupPartitionStatus.BackupPartitionStatusState, cancellationToken.IsCancellationRequested, this.ServiceUri, this.PartitionId); } while ((backupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted || backupPartitionStatus.BackupPartitionStatusState == BackupState.BackupInProgress) && this.BackupRequestGuid.Equals(backupPartitionStatus.OperationId) && !cancellationToken.IsCancellationRequested); cancellationToken.ThrowIfCancellationRequested(); }
public async Task TryToProcessAsync(StatefulService statefulService, string processQueueTypeTrace) { var workItemTimeout = BackupRestoreServiceConfig.WorkItemTimeout; try { workItemInProcessStore = await WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService); bool processStatus = false; try { processStatus = await UtilityHelper.InvokeWithRetry <bool>(() => { var cancellationTokenSource = new CancellationTokenSource(workItemTimeout); return(this.Process(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace)); } ); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "The WorkItem {0} is Processed as {1}", this, processStatus); } catch (Exception exception) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception While TryToProcessAsync"); AggregateException aggregateException = exception as AggregateException; if (aggregateException != null) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}", exception.StackTrace); foreach (Exception innerException in aggregateException.InnerExceptions) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Inner Exception : {0} , Message : {1} , Stack Trace : {2} ", innerException.InnerException, innerException.Message, innerException.StackTrace); } } else { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception : {0} , Message : {1} , Stack Trace : {2} ", exception.InnerException, exception.Message, exception.StackTrace); } } if (processStatus) { var cancellationTokenSource = new CancellationTokenSource(workItemTimeout); await UtilityHelper.InvokeWithRetry((() => this.ProcessSuccessHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace))); } else { var cancellationTokenSource = new CancellationTokenSource(workItemTimeout); await UtilityHelper.InvokeWithRetry((() => this.ProcessFailureHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace))); } } catch (Exception exception) { AggregateException aggregateException = exception as AggregateException; ; if (aggregateException != null) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}", exception.StackTrace); foreach (Exception innerException in aggregateException.InnerExceptions) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Inner Exception : {0} , Message : {1} , Stack Trace : {2} ", innerException.InnerException, innerException.Message, innerException.StackTrace); } } else { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception : {0} , Message : {1} , Stack Trace : {2} ", exception.InnerException, exception.Message, exception.StackTrace); } } }