public override async Task <bool> Process(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
var timeoutDateTime = this.RequestDateTime.AddSeconds(this.BackupTimeout.TotalSeconds);
var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
if (DateTime.Compare(DateTime.UtcNow, timeoutDateTime) > 0)
{
await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace));
return(true);
}
var timeoutSpan = timeoutDateTime.Subtract(DateTime.UtcNow);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "TimeoutSpan for Backup Partition WorkItem {0}", timeoutSpan);
var cancellationTokenSource = new CancellationTokenSource(timeoutSpan);
try
{
await this.ProcessBackup(statefulService, timeout, cancellationTokenSource.Token, processQueueTypeTrace);
}
catch (OperationCanceledException operationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace,
"Operation Cancellaed exception Details {0}",
operationCanceledException.ToString());
await UtilityHelper.InvokeWithRetry(() => this.UpdateBackupStatusAsTimeout(statefulService, fabricUri, timeout, cancellationToken, processQueueTypeTrace));
}
return(true);
}
private async Task <bool> IsPartitionBackupSuspended(StatefulService statefulService, string processQueueTypeTrace)
{
var suspendStore = await SuspendStore.CreateOrGetSuspendStatusStore(statefulService);
var fabricKey = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
var containsSuspension = await suspendStore.GetValueAsync(fabricKey) ?? await suspendStore.GetValueAsync(ServiceUri) ?? await suspendStore.GetValueAsync(UtilityHelper.GetApplicationUriFromServiceUri(ServiceUri));
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "IsPartition Suspended {0}", containsSuspension != null);
return(containsSuspension != null);
}
private async Task <Tuple <BackupMapping, BackupPolicy> > GetEffectiveBackupPartitionAndPolicyForPartition(
StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
BackupPolicy backupPolicy = null;
var backupMappingStore = await BackupMappingStore.CreateOrGetBackupMappingStore(statefulService);
var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(statefulService);
var backupMappingKey = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
var backupMapping = (await backupMappingStore.GetValueAsync(backupMappingKey, timeout, cancellationToken) ??
await backupMappingStore.GetValueAsync(this.ServiceUri, timeout, cancellationToken)) ??
await backupMappingStore.GetValueAsync(UtilityHelper.GetApplicationNameFromService(this.ServiceUri), timeout, cancellationToken);
if (backupMapping != null)
{
backupPolicy = await backupPolicyStore.GetValueAsync(backupMapping.BackupPolicyName, timeout, cancellationToken);
}
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace,
"GetEffectiveBackupPartitionAndPolicyForPartition resulted as BackupMapping {0} ,BackupPolicy {1} ", backupMapping, backupPolicy);
return(Tuple.Create(backupMapping, backupPolicy));
}
public override async Task <bool> Process(Microsoft.ServiceFabric.Services.Runtime.StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
WorkItem workItem = null;
WorkItemQueue workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService);
BackupMappingStore backupMappingStore =
await BackupMappingStore.CreateOrGetBackupMappingStore(statefulService);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Resolving for {0} of type {1} ", this.ApplicationOrServiceUri, this.FabricBackupResourceType);
try
{
if (this.FabricBackupResourceType == FabricBackupResourceType.ApplicationUri)
{
ServiceList serviceList =
await FabricClientHelper.GetServiceList(this.ApplicationOrServiceUri);
foreach (Query.Service service in serviceList)
{
if (service.ServiceKind == ServiceKind.Stateful && (service as StatefulService).HasPersistedState)
{
BackupMapping specificServiceBackupMapping = await backupMappingStore.GetValueAsync(
UtilityHelper.GetBackupMappingKey(service.ServiceName.OriginalString, null));
if (specificServiceBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition ||
this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition)
{
ServicePartitionList servicePartitionList =
await
FabricClientHelper.GetPartitionList(service.ServiceName.OriginalString);
foreach (Partition servicePartition in servicePartitionList)
{
BackupMapping specificPartitionBackupMapping =
await backupMappingStore.GetValueAsync(UtilityHelper.GetBackupMappingKey(
service.ServiceName.OriginalString,
servicePartition.PartitionInformation.Id.ToString()));
if (specificPartitionBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition ||
this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition)
{
//Send to Service Partition
workItem = new SendToServiceNodeWorkItem
(await UtilityHelper.GetCustomServiceUri(service.ServiceName.OriginalString, timeout, cancellationToken),
servicePartition.PartitionInformation.Id.ToString(),
this.WorkItemInfo);
await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken);
}
}
}
}
}
}
else if (this.FabricBackupResourceType == FabricBackupResourceType.ServiceUri)
{
ServicePartitionList servicePartitionList =
await
FabricClientHelper.GetPartitionList(this.ApplicationOrServiceUri);
foreach (Partition servicePartition in servicePartitionList)
{
BackupMapping specificPartitionBackupMapping =
await backupMappingStore.GetValueAsync(UtilityHelper.GetBackupMappingKey(
this.ApplicationOrServiceUri, servicePartition.PartitionInformation.Id.ToString()));
if (specificPartitionBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition ||
this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition)
{
//Send to Service Partition
workItem = new SendToServiceNodeWorkItem
(await UtilityHelper.GetCustomServiceUri(this.ApplicationOrServiceUri, timeout, cancellationToken), servicePartition.PartitionInformation.Id.ToString(), this.WorkItemInfo);
await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken);
}
}
}
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Resolving successful");
}
catch (Exception exception)
{
AggregateException aggregateException = exception as AggregateException;
if (aggregateException != null)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}",
exception.StackTrace);
foreach (Exception innerException in aggregateException.InnerExceptions)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Inner Exception : {0} , Message : {1} , Stack Trace : {2} ",
innerException.InnerException, innerException.Message, innerException.StackTrace);
}
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Exception : {0} , Message : {1} , Stack Trace : {2} ",
exception.InnerException, exception.Message, exception.StackTrace);
}
return(false);
}
return(true);
}
private async Task ProcessBackup(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Backup for request Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
var timeOutPersecCycle = 30000;
var backupPartitionStore = await BackupPartitionStore.CreateOrGetBackupPartitionStore(statefulService);
var fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
BackupPartitionStatus backupPartitionStatus;
do
{
var backupNowConfiguration =
new BackupNowConfiguration()
{
StoreInformation = this.GetBackupStoreInformationFromBackupStorage(this.BackupStorage),
OperationTimeoutMilliseconds = (int)this.BackupTimeout.TotalMilliseconds
};
await UtilityHelper.InvokeWithRetry(async() =>
{
await
Program.ServiceAgent.BackupPartitionAsync(new Uri(UtilityHelper.GetUriFromCustomUri(this.ServiceUri)),
Guid.Parse(this.PartitionId),
this.BackupRequestGuid, backupNowConfiguration, timeout, cancellationToken);
});
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Requested the BackupPartitionAsync Successfully for Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
await UtilityHelper.InvokeWithRetry(async() =>
{
using (ITransaction transaction = statefulService.StateManager.CreateTransaction())
{
var currentBackupPartitionStatus = await backupPartitionStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction);
if (currentBackupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted)
{
var toUpdateBackupPartitionStatus = currentBackupPartitionStatus.ToBuilder().WithState(BackupState.BackupInProgress).Build();
await backupPartitionStore.UpdateValueAsync(fabricUri, toUpdateBackupPartitionStatus,
timeout,
cancellationToken,
transaction);
await transaction.CommitAsync();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated the Backup partition request to BackupInProcss for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
else
{
transaction.Abort();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace,
"Could not Update the BackupPartition request to BackupInProcess for request Service Uri : {0} , PartitionId {1} because currentBackupPartitionStatus value is {2}",
this.ServiceUri, this.PartitionId, currentBackupPartitionStatus);
}
}
}
);
await Task.Delay(timeOutPersecCycle, cancellationToken);
backupPartitionStatus = await backupPartitionStore.GetValueAsync(fabricUri);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "BackupPartitionstatusState {0},isCancellationRequested {1} for Service Uri : {2} , PartitionId {3}",
backupPartitionStatus.BackupPartitionStatusState, cancellationToken.IsCancellationRequested, this.ServiceUri, this.PartitionId);
} while ((backupPartitionStatus.BackupPartitionStatusState == BackupState.Accepted ||
backupPartitionStatus.BackupPartitionStatusState == BackupState.BackupInProgress) &&
this.BackupRequestGuid.Equals(backupPartitionStatus.OperationId) &&
!cancellationToken.IsCancellationRequested);
cancellationToken.ThrowIfCancellationRequested();
}
public async Task ProcessRestore(StatefulService statefulService, CancellationToken cancellationToken, string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Processing the Restore for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
WorkItemStore workItemStore = await WorkItemStore.CreateOrGetWorkItemStore(statefulService);
RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService);
string fabricUri = UtilityHelper.GetBackupMappingKey(this.ServiceUri, this.PartitionId);
PartitionDataLossProgress partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace);
if (this.RestoreStatusStateModel == RestoreStatusStateModel.TriggerDataLoss)
{
if (partitionDataLossProgress == null)
{
bool isInitiateDataLossSuccess = await this.InitiateDataLoss(processQueueTypeTrace);
if (isInitiateDataLossSuccess)
{
this.RestoreStatusStateModel = RestoreStatusStateModel.DataLossTriggered;
}
}
await workItemStore.UpdateValueAsync(this.WorkItemGuid, this);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "RestoreStatusStateModel as DataLossTriggered for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
if (this.RestoreStatusStateModel == RestoreStatusStateModel.DataLossTriggered)
{
RestoreStatus restoreStatus = null;
do
{
partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace);
switch (partitionDataLossProgress.State)
{
case TestCommandProgressState.Running:
case TestCommandProgressState.RollingBack:
case TestCommandProgressState.Completed:
break;
case TestCommandProgressState.Cancelled:
case TestCommandProgressState.Faulted:
case TestCommandProgressState.ForceCancelled:
this.DataLossGuid = Guid.NewGuid();
await workItemStore.UpdateValueAsync(this.WorkItemGuid, this);
await this.InitiateDataLoss(processQueueTypeTrace);
break;
}
await Task.Delay(Constants.CheckRestoreTimeSpan, cancellationToken);
restoreStatus = await restoreStore.GetValueAsync(fabricUri);
if (restoreStatus.RestoreStatusState == RestoreState.Timeout && restoreStatus.RestoreRequestGuid.Equals(this.RestoreRequestGuid))
{
partitionDataLossProgress = await this.GetPartitionDataLossProgress(processQueueTypeTrace);
switch (partitionDataLossProgress.State)
{
case TestCommandProgressState.Running:
case TestCommandProgressState.RollingBack:
await this.CancelDataLoss(processQueueTypeTrace);
break;
case TestCommandProgressState.Cancelled:
case TestCommandProgressState.Faulted:
case TestCommandProgressState.ForceCancelled:
case TestCommandProgressState.Completed:
break;
}
}
} while ((restoreStatus.RestoreStatusState == RestoreState.RestoreInProgress ||
restoreStatus.RestoreStatusState == RestoreState.Accepted) &&
this.RestoreRequestGuid.Equals(restoreStatus.RestoreRequestGuid) &&
!cancellationToken.IsCancellationRequested);
cancellationToken.ThrowIfCancellationRequested();
}
}