public override async Task <bool> Process(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
WorkItemQueue workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService);
List <WorkItem> workItemList = new List <WorkItem>();
WorkItem workItem = null;
foreach (string key in this.listOfEnablement)
{
string applicationNameUri = null;
string serviceNameUri = null;
string partitionId = null;
FabricBackupResourceType fabricBackupResourceType = UtilityHelper.GetApplicationAndServicePartitionUri(
key, out applicationNameUri, out serviceNameUri, out partitionId);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Resolving for Key : {0} as {1}", key, fabricBackupResourceType);
switch (fabricBackupResourceType)
{
case FabricBackupResourceType.PartitionUri:
workItem = new SendToServiceNodeWorkItem(serviceNameUri, partitionId, this.WorkItemInfo);
break;
case FabricBackupResourceType.ServiceUri:
workItem = new ResolveToPartitionWorkItem(FabricBackupResourceType.ServiceUri, serviceNameUri,
this.WorkItemInfo);
break;
case FabricBackupResourceType.ApplicationUri:
workItem = new ResolveToPartitionWorkItem(FabricBackupResourceType.ApplicationUri, applicationNameUri,
this.WorkItemInfo);
break;
}
if (workItem != null)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Added WorkItem of Type {0} ", workItem.GetType());
workItemList.Add(workItem);
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Work Item is Null for ApplicationUri = {0} , " +
"ServiceUri = {1} , PartitionUri = {2}", applicationNameUri,
serviceNameUri, partitionId);
}
}
foreach (var workItemInList in workItemList)
{
await workItemQueue.AddWorkItem(workItemInList, timeout, cancellationToken);
}
return(true);
}
internal static async Task <BackupMappingStore> CreateOrGetBackupMappingStore(StatefulService statefulService)
{
if (Store == null)
{
BackupRestoreTrace.TraceSource.WriteNoise(TraceTypeBackupMappingStoreType, "Creating a Backup Protection Store");
IReliableDictionary <string, BackupMapping> reliableDictionary =
await statefulService.StateManager.GetOrAddAsync <IReliableDictionary <string, BackupMapping> >(BackupMappingStoreName);
Store = new BackupMappingStore(reliableDictionary, statefulService);
workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService);
BackupRestoreTrace.TraceSource.WriteNoise(TraceTypeBackupMappingStoreType, "Created a Backup Protection Store successfully");
}
return(Store);
}
internal static async Task <WorkItemQueue> CreateOrGetWorkItemQueue(StatefulService statefulService)
{
if (workItemQueue == null)
{
IReliableQueue <WorkItemProcessInfo> workItemReliableQueue =
await statefulService.StateManager.GetOrAddAsync <IReliableQueue <WorkItemProcessInfo> >(Constants.WorkItemQueue);
lock (syncRoot)
{
if (workItemQueue == null)
{
workItemQueue = new WorkItemQueue(workItemReliableQueue, 0, WorkItemQueueRunType.WorkItemQueue, statefulService);
}
}
}
return(workItemQueue);
}
private async Task UpdateRestoreStatusAsTimeout(StatefulService statefulService, string fabricUri, TimeSpan timeout, CancellationToken cancellationToken,
string processQueueTypeTrace)
{
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updating the Restore status as timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
RestoreStore restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(statefulService);
WorkItemQueue workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService);
WorkItem workItem = new SendToServiceNodeWorkItem(this.ServiceUri, this.PartitionId, new WorkItemInfo()
{
WorkItemType = WorkItemPropogationType.UpdatePolicyAfterRestore
});
using (ITransaction transaction = statefulService.StateManager.CreateTransaction())
{
var currentRestoreStatus = await restoreStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "RestoreStatus {0} as timeout for request Service Uri : {1} , PartitionId {2}", currentRestoreStatus, this.ServiceUri, this.PartitionId);
if (currentRestoreStatus != null &&
currentRestoreStatus.RestoreRequestGuid.Equals(this.RestoreRequestGuid) &&
(currentRestoreStatus.RestoreStatusState == RestoreState.Accepted ||
currentRestoreStatus.RestoreStatusState == RestoreState.RestoreInProgress))
{
var restoreStatus = currentRestoreStatus.ToBuilder()
.WithState(RestoreState.Timeout)
.WithErrorCode(unchecked ((int)NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TIMEOUT))
.WithMessage(StringResources.RestoreTimeout)
.Build();
await restoreStore.UpdateValueAsync(fabricUri, restoreStatus, timeout, cancellationToken, transaction);
await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken, transaction);
await transaction.CommitAsync();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated Restore Status as Timeout for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
else
{
transaction.Abort();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Updated Restore Status as Timeout Failed for request Service Uri : {0} , PartitionId {1}", this.ServiceUri, this.PartitionId);
}
}
}
internal static async Task StartAndScheduleWorkItemHandler(StatefulService statefulService)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Initializing Work Item Queue and Handlers");
workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService);
workItemQueue1Min = await WorkItemQueue1Min.CreateOrGetWorkItemQueue(statefulService);
workItemQueue2Min = await WorkItemQueue2Min.CreateOrGetWorkItemQueue(statefulService);
workItemQueue4Min = await WorkItemQueue4Min.CreateOrGetWorkItemQueue(statefulService);
workItemQueue8Min = await WorkItemQueue8Min.CreateOrGetWorkItemQueue(statefulService);
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Initialized Work Item Queue and Handlers");
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Initializing RecoverInProcessWorkItemWrapper");
await RecoverInProcessWorkItemWrapper(statefulService);
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "RecoverInProcessWorkItemWrapper Completed");
}
public override async Task <bool> Process(Microsoft.ServiceFabric.Services.Runtime.StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
WorkItem workItem = null;
WorkItemQueue workItemQueue = await WorkItemQueue.CreateOrGetWorkItemQueue(statefulService);
BackupMappingStore backupMappingStore =
await BackupMappingStore.CreateOrGetBackupMappingStore(statefulService);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Resolving for {0} of type {1} ", this.ApplicationOrServiceUri, this.FabricBackupResourceType);
try
{
if (this.FabricBackupResourceType == FabricBackupResourceType.ApplicationUri)
{
ServiceList serviceList =
await FabricClientHelper.GetServiceList(this.ApplicationOrServiceUri);
foreach (Query.Service service in serviceList)
{
if (service.ServiceKind == ServiceKind.Stateful && (service as StatefulService).HasPersistedState)
{
BackupMapping specificServiceBackupMapping = await backupMappingStore.GetValueAsync(
UtilityHelper.GetBackupMappingKey(service.ServiceName.OriginalString, null));
if (specificServiceBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition ||
this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition)
{
ServicePartitionList servicePartitionList =
await
FabricClientHelper.GetPartitionList(service.ServiceName.OriginalString);
foreach (Partition servicePartition in servicePartitionList)
{
BackupMapping specificPartitionBackupMapping =
await backupMappingStore.GetValueAsync(UtilityHelper.GetBackupMappingKey(
service.ServiceName.OriginalString,
servicePartition.PartitionInformation.Id.ToString()));
if (specificPartitionBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition ||
this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition)
{
//Send to Service Partition
workItem = new SendToServiceNodeWorkItem
(await UtilityHelper.GetCustomServiceUri(service.ServiceName.OriginalString, timeout, cancellationToken),
servicePartition.PartitionInformation.Id.ToString(),
this.WorkItemInfo);
await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken);
}
}
}
}
}
}
else if (this.FabricBackupResourceType == FabricBackupResourceType.ServiceUri)
{
ServicePartitionList servicePartitionList =
await
FabricClientHelper.GetPartitionList(this.ApplicationOrServiceUri);
foreach (Partition servicePartition in servicePartitionList)
{
BackupMapping specificPartitionBackupMapping =
await backupMappingStore.GetValueAsync(UtilityHelper.GetBackupMappingKey(
this.ApplicationOrServiceUri, servicePartition.PartitionInformation.Id.ToString()));
if (specificPartitionBackupMapping == null || this.WorkItemInfo.WorkItemType == WorkItemPropogationType.SuspendPartition ||
this.WorkItemInfo.WorkItemType == WorkItemPropogationType.ResumePartition)
{
//Send to Service Partition
workItem = new SendToServiceNodeWorkItem
(await UtilityHelper.GetCustomServiceUri(this.ApplicationOrServiceUri, timeout, cancellationToken), servicePartition.PartitionInformation.Id.ToString(), this.WorkItemInfo);
await workItemQueue.AddWorkItem(workItem, timeout, cancellationToken);
}
}
}
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Resolving successful");
}
catch (Exception exception)
{
AggregateException aggregateException = exception as AggregateException;
if (aggregateException != null)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}",
exception.StackTrace);
foreach (Exception innerException in aggregateException.InnerExceptions)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Inner Exception : {0} , Message : {1} , Stack Trace : {2} ",
innerException.InnerException, innerException.Message, innerException.StackTrace);
}
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Exception : {0} , Message : {1} , Stack Trace : {2} ",
exception.InnerException, exception.Message, exception.StackTrace);
}
return(false);
}
return(true);
}
internal void DisposeWorkItemQueue()
{
this.DisposeWorkItemDeQueuer();
workItemQueue = null;
}