public override async Task <bool> Process(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
var timeoutDateTime = this.RequestDateTime.AddSeconds(this.BackupTimeout.TotalSeconds);
var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
if (DateTime.Compare(DateTime.UtcNow, timeoutDateTime) > 0)
{
await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace));
return(true);
}
var timeoutSpan = timeoutDateTime.Subtract(DateTime.UtcNow);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "TimeoutSpan for Backup Partition WorkItem {0}", timeoutSpan);
var cancellationTokenSource = new CancellationTokenSource(timeoutSpan);
try
{
await this.ProcessBackup(statefulService, timeout, cancellationTokenSource.Token, processQueueTypeTrace);
}
catch (OperationCanceledException operationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace,
"Operation Cancellaed exception Details {0}",
operationCanceledException.ToString());
await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace));
}
return(true);
}
private async Task ProcessBackup(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Backup for request Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
var timeOutPersecCycle = 30000;
var backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService);
var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
BackupPartitionStatus backupPartitionStatus;
do
{
var backupNowConfiguration =
new BackupNowConfiguration()
{
StoreInformation = this.GetBackupStoreInformationFromBackupStorage(this.BackupStorage),
OperationTimeoutMilliseconds = (int)this.BackupTimeout.TotalMilliseconds
};
await UtilityHelper.InvokeWithRetry(async() =>
{
await
Program.ServiceAgent.BackupPartitionAsync(new Uri(UtilityHelper.GetUriFromCustomUri(this.ServiceUri)),
Guid.Parse(this.PartitionId),
this.BackupRequestGuid, backupNowConfiguration, timeout, cancellationToken);
});
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Requested the BackupPartitionAsync Successfully for Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
await UtilityHelper.InvokeWithRetry(async() =>
{
using (ITransaction transaction = statefulService.StateManager.CreateTransaction())
{
var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction);
if (currentBackupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted)
{
var toUpdateBackupPartitionStatus = currentBackupPartitionStatus.ToBuilder().WithState(BackupState.BackupInProgress).Build();
await backupPartitionStore.UpdateValueAsync(fabricUri, toUpdateBackupPartitionStatus,
timeout,
cancellationToken,
transaction);
await transaction.CommitAsync();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup partition request to BackupInProcss for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
else
{
transaction.Abort();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace,
"Could not Update the BackupPartition request to BackupInProcess for request Service Uri : {0} , PartitionId {1} because currentBackupPartitionStatus value is {2}",
this.ServiceUri, this.PartitionId, currentBackupPartitionStatus);
}
}
}
);
await Task.Delay(timeOutPersecCycle, cancellationToken);
backupPartitionStatus = await backupPartitionStore.GetValueAsync(fabricUri);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionstatusState {0},isCancellationRequested {1} for Service Uri : {2} , PartitionId {3}",
backupPartitionStatus.BackupPartitionStatusState, cancellationToken.IsCancellationRequested, this.ServiceUri, this.PartitionId);
} while ((backupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted ||
backupPartitionStatus.BackupPartitionStatusState == BackupState.BackupInProgress) &&
this.BackupRequestGuid.Equals(backupPartitionStatus.OperationId) &&
!cancellationToken.IsCancellationRequested);
cancellationToken.ThrowIfCancellationRequested();
}
public async Task TryToProcessAsync(StatefulService statefulService, string processQueueTypeTrace)
{
var workItemTimeout = BackupRestoreServiceConfig.WorkItemTimeout;
try
{
workItemInProcessStore = await WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService);
bool processStatus = false;
try
{
processStatus = await UtilityHelper.InvokeWithRetry <bool>(() =>
{
var cancellationTokenSource = new CancellationTokenSource(workItemTimeout);
return(this.Process(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace));
}
);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "The WorkItem {0} is Processed as {1}", this, processStatus);
}
catch (Exception exception)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception While TryToProcessAsync");
AggregateException aggregateException = exception as AggregateException;
if (aggregateException != null)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}",
exception.StackTrace);
foreach (Exception innerException in aggregateException.InnerExceptions)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Inner Exception : {0} , Message : {1} , Stack Trace : {2} ",
innerException.InnerException, innerException.Message, innerException.StackTrace);
}
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Exception : {0} , Message : {1} , Stack Trace : {2} ",
exception.InnerException, exception.Message, exception.StackTrace);
}
}
if (processStatus)
{
var cancellationTokenSource = new CancellationTokenSource(workItemTimeout);
await UtilityHelper.InvokeWithRetry((() => this.ProcessSuccessHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace)));
}
else
{
var cancellationTokenSource = new CancellationTokenSource(workItemTimeout);
await UtilityHelper.InvokeWithRetry((() => this.ProcessFailureHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace)));
}
}
catch (Exception exception)
{
AggregateException aggregateException = exception as AggregateException;
;
if (aggregateException != null)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}",
exception.StackTrace);
foreach (Exception innerException in aggregateException.InnerExceptions)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Inner Exception : {0} , Message : {1} , Stack Trace : {2} ",
innerException.InnerException, innerException.Message, innerException.StackTrace);
}
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Exception : {0} , Message : {1} , Stack Trace : {2} ",
exception.InnerException, exception.Message, exception.StackTrace);
}
}
}