public override async Task <bool> Process(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
DateTime timeoutDateTime = this.RequestDateTime.AddSeconds(this.RestoreTimeout.TotalSeconds);
string fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService);
if (DateTime.Compare(DateTime.UtcNow, timeoutDateTime) > 0)
{
await UtilityHelper.InvokeWithRetry(() => this.UpdateRestoreStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken,
processQueueTypeTrace));
return(true);
}
TimeSpan timeoutSpan = timeoutDateTime.Subtract(DateTime.UtcNow);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "TimeoutSpan for Backup Partition WorkItem {0}", timeoutSpan);
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(timeoutSpan);
try
{
await this.ProcessRestore(statefulService, cancellationTokenSource.Token, processQueueTypeTrace);
}
catch (OperationCanceledException operationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace,
"Operation Canceled exception Details {0}",
operationCanceledException.ToString());
await UtilityHelper.InvokeWithRetry(() => this.UpdateRestoreStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace));
}
return(true);
}
internal static async Task <RestoreStore> CreateOrGetRestoreStatusStore(StatefulService statefulService)
{
if (Store == null)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceRestoreStoreType, "Creating a Restore Store");
IReliableDictionary <string, RestoreStatus> reliableDictionary = await statefulService.StateManager.GetOrAddAsync <IReliableDictionary <string, RestoreStatus> >(RestoreStoreName);
Store = new RestoreStore(reliableDictionary, statefulService);
BackupRestoreTrace.TraceSource.WriteInfo(TraceRestoreStoreType, "Created a Restore Store successfully");
}
return(Store);
}
private async Task UpdateRestoreStatusAsTimeout(StatefulService statefulService, string fabricUri, TimeSpan timeout, CancellationToken cancellationToken,
string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updating the Restore status as timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService);
WorkItemQueue workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService);
WorkItem workItem = new SendToServiceNodeWorkItem(this.ServiceUri, this.PartitionId, new WorkItemInfo()
{
WorkItemType = WorkItemPropogationType.UpdatePolicyAfterRestore
});
using (ITransaction transaction = statefulService.StateManager.CreateTransaction())
{
var currentRestoreStatus = await restoreStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "RestoreStatus {0} as timeout for request Service Uri : {1} , PartitionId {2}", currentRestoreStatus, this.ServiceUri, this.PartitionId);
if (currentRestoreStatus != null &&
currentRestoreStatus.RestoreRequestGuid.Equals(this.RestoreRequestGuid) &&
(currentRestoreStatus.RestoreStatusState == RestoreState.Accepted ||
currentRestoreStatus.RestoreStatusState == RestoreState.RestoreInProgress))
{
var restoreStatus = currentRestoreStatus.ToBuilder()
.WithState(RestoreState.Timeout)
.WithErrorCode(unchecked ((int)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TIMEOUT))
.WithMessage(StringResources.RestoreTimeout)
.Build();
await restoreStore.UpdateValueAsync(fabricUri, restoreStatus, timeout, cancellationToken, transaction);
await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken, transaction);
await transaction.CommitAsync();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated Restore Status as Timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
else
{
transaction.Abort();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated Restore Status as Timeout Failed for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
}
}
public async Task ProcessRestore(StatefulService statefulService, CancellationToken cancellationToken, string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Restore for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
WorkItemStore workItemStore = await WorkItemStore.CreateOrGetWorkItemStore(statefulService);
RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService);
string fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
PartitionDataLossProgress partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace);
if (this.RestoreStatusStateModel == RestoreStatusStateModel.TriggerDataLoss)
{
if (partitionDataLossProgress == null)
{
bool isInitiateDataLossSuccess = await this.InitiateDataLoss(processQueueTypeTrace);
if (isInitiateDataLossSuccess)
{
this.RestoreStatusStateModel = RestoreStatusStateModel.DataLossTriggered;
}
}
await workItemStore.UpdateValueAsync(this.WorkItemGuid, this);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "RestoreStatusStateModel as DataLossTriggered for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
if (this.RestoreStatusStateModel == RestoreStatusStateModel.DataLossTriggered)
{
RestoreStatus restoreStatus = null;
do
{
partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace);
switch (partitionDataLossProgress.State)
{
case TestCommandProgressState.Running:
case TestCommandProgressState.RollingBack:
case TestCommandProgressState.Completed:
break;
case TestCommandProgressState.Cancelled:
case TestCommandProgressState.Faulted:
case TestCommandProgressState.ForceCancelled:
this.DataLossGuid = Guid.NewGuid();
await workItemStore.UpdateValueAsync(this.WorkItemGuid, this);
await this.InitiateDataLoss(processQueueTypeTrace);
break;
}
await Task.Delay(Constants.CheckRestoreTimeSpan, cancellationToken);
restoreStatus = await restoreStore.GetValueAsync(fabricUri);
if (restoreStatus.RestoreStatusState == RestoreState.Timeout && restoreStatus.RestoreRequestGuid.Equals(this.RestoreRequestGuid))
{
partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace);
switch (partitionDataLossProgress.State)
{
case TestCommandProgressState.Running:
case TestCommandProgressState.RollingBack:
await this.CancelDataLoss(processQueueTypeTrace);
break;
case TestCommandProgressState.Cancelled:
case TestCommandProgressState.Faulted:
case TestCommandProgressState.ForceCancelled:
case TestCommandProgressState.Completed:
break;
}
}
} while ((restoreStatus.RestoreStatusState == RestoreState.RestoreInProgress ||
restoreStatus.RestoreStatusState == RestoreState.Accepted) &&
this.RestoreRequestGuid.Equals(restoreStatus.RestoreRequestGuid) &&
!cancellationToken.IsCancellationRequested);
cancellationToken.ThrowIfCancellationRequested();
}
}
