private async Task UpdateBackupStatusAsTimeout(StatefulService statefulService, string fabricUri, TimeSpan timeout, CancellationToken cancellationToken,
string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updating the Backup status as timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
BackupPartitionStore backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService);
using (ITransaction transaction = statefulService.StateManager.CreateTransaction())
{
var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionStatus {0} as timeout for request Service Uri : {1} , PartitionId {2}", currentBackupPartitionStatus, this.ServiceUri, this.PartitionId);
if ((currentBackupPartitionStatus?.BackupPartitionStatusState == BackupState.Accepted ||
currentBackupPartitionStatus?.BackupPartitionStatusState == BackupState.BackupInProgress) &&
this.BackupRequestGuid.Equals(currentBackupPartitionStatus.OperationId))
{
var backupPartitionStatus = currentBackupPartitionStatus.ToBuilder()
.WithState(BackupState.Timeout)
.WithErrorCode(unchecked ((int)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TIMEOUT))
.WithMessage(StringResources.BackupTimeout)
.Build();
await backupPartitionStore.UpdateValueAsync(fabricUri, backupPartitionStatus, timeout, cancellationToken, transaction);
await transaction.CommitAsync();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup status as timeout failed for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
else
{
transaction.Abort();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updating the Backup status as timeout failed for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
}
}
internal static async Task <BackupPartitionStore> CreateOrGetBackupPartitionStore(StatefulService statefulService)
{
if (Store == null)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceRestoreStoreType, "Creating a BackupPartition Store");
IReliableDictionary <string, BackupPartitionStatus> reliableDictionary = await statefulService.StateManager.GetOrAddAsync <IReliableDictionary <string, BackupPartitionStatus> >(BackupPartitionStoreName);
Store = new BackupPartitionStore(reliableDictionary, statefulService);
BackupRestoreTrace.TraceSource.WriteInfo(TraceRestoreStoreType, "Created a BackupPartition Store successfully");
}
return(Store);
}
private async Task ProcessBackup(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Backup for request Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
var timeOutPersecCycle = 30000;
var backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService);
var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
BackupPartitionStatus backupPartitionStatus;
do
{
var backupNowConfiguration =
new BackupNowConfiguration()
{
StoreInformation = this.GetBackupStoreInformationFromBackupStorage(this.BackupStorage),
OperationTimeoutMilliseconds = (int)this.BackupTimeout.TotalMilliseconds
};
await UtilityHelper.InvokeWithRetry(async() =>
{
await
Program.ServiceAgent.BackupPartitionAsync(new Uri(UtilityHelper.GetUriFromCustomUri(this.ServiceUri)),
Guid.Parse(this.PartitionId),
this.BackupRequestGuid, backupNowConfiguration, timeout, cancellationToken);
});
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Requested the BackupPartitionAsync Successfully for Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
await UtilityHelper.InvokeWithRetry(async() =>
{
using (ITransaction transaction = statefulService.StateManager.CreateTransaction())
{
var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction);
if (currentBackupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted)
{
var toUpdateBackupPartitionStatus = currentBackupPartitionStatus.ToBuilder().WithState(BackupState.BackupInProgress).Build();
await backupPartitionStore.UpdateValueAsync(fabricUri, toUpdateBackupPartitionStatus,
timeout,
cancellationToken,
transaction);
await transaction.CommitAsync();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup partition request to BackupInProcss for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
else
{
transaction.Abort();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace,
"Could not Update the BackupPartition request to BackupInProcess for request Service Uri : {0} , PartitionId {1} because currentBackupPartitionStatus value is {2}",
this.ServiceUri, this.PartitionId, currentBackupPartitionStatus);
}
}
}
);
await Task.Delay(timeOutPersecCycle, cancellationToken);
backupPartitionStatus = await backupPartitionStore.GetValueAsync(fabricUri);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionstatusState {0},isCancellationRequested {1} for Service Uri : {2} , PartitionId {3}",
backupPartitionStatus.BackupPartitionStatusState, cancellationToken.IsCancellationRequested, this.ServiceUri, this.PartitionId);
} while ((backupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted ||
backupPartitionStatus.BackupPartitionStatusState == BackupState.BackupInProgress) &&
this.BackupRequestGuid.Equals(backupPartitionStatus.OperationId) &&
!cancellationToken.IsCancellationRequested);
cancellationToken.ThrowIfCancellationRequested();
}