internal static async Task <WorkItemInProcessStore> CreateOrGetWorkItemInProcessStore(StatefulService statefulService)
{
if (Store == null)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceWorkItemInProcessStoreType, "Creating a WorkItem In Process Store");
IReliableDictionary <Guid, WorkItemProcessInfo> reliableDictionary = await statefulService.StateManager.GetOrAddAsync <IReliableDictionary <Guid, WorkItemProcessInfo> >(WorkItemStoreName);
Store = new WorkItemInProcessStore(reliableDictionary, statefulService);
BackupRestoreTrace.TraceSource.WriteInfo(TraceWorkItemInProcessStoreType, "Created a WorkItem In Process successfully");
}
return(Store);
}
private static async Task RecoverInProcessWorkItems(StatefulService statefulService)
{
try
{
WorkItemInProcessStore workItemInProcessStore =
await WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService);
WorkItemStore workItemStore = await WorkItemStore.CreateOrGetWorkItemStore(statefulService);
List <WorkItemProcessInfo> workItemProcessInfos = await workItemInProcessStore.GetValuesAsync();
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Work Item to be Recovered Count {0}", workItemProcessInfos.Count);
foreach (WorkItemProcessInfo workItemProcessInfo in workItemProcessInfos)
{
WorkItem workItem = await workItemStore.GetValueAsync(workItemProcessInfo.WorkItemId);
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Work Item Recovered Type : {0} , Id : {1}", workItem.GetType(), workItemProcessInfo.WorkItemId);
Task processTask = workItem.TryToProcessAsync(statefulService, TraceType);
}
}
catch (Exception exception)
{
AggregateException aggregateException = exception as AggregateException;
;
if (aggregateException != null)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "Aggregate Exception Stack Trace : {0}",
exception.StackTrace);
foreach (Exception innerException in aggregateException.InnerExceptions)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType,
"Inner Exception : {0} , Message : {1} , Stack Trace : {2} ",
innerException.InnerException, innerException.Message, innerException.StackTrace);
}
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType,
"Exception : {0} , Message : {1} , Stack Trace : {2} ",
exception.InnerException, exception.Message, exception.StackTrace);
}
}
}
public override async Task ProcessFailureHandle(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace)
{
WorkItemInProcessStore workItemInProcessStore = await
WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService);
WorkItemQueue1Min workItemQueue1Min = await WorkItemQueue1Min.CreateOrGetWorkItemQueue(statefulService);
WorkItemProcessInfo workItemProcessInfo = await workItemInProcessStore.GetValueAsync(this.WorkItemGuid);
using (ITransaction transaction = statefulService.StateManager.CreateTransaction())
{
await workItemInProcessStore.DeleteValueAsync(this.WorkItemGuid, timeout, cancellationToken, transaction);
await workItemQueue1Min.AddWorkItem(workItemProcessInfo, timeout, cancellationToken, transaction);
await transaction.CommitAsync();
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Process Failure Handle WorkItemInProcessStore Delete WorkItem {0} And Enqueue in {1} ", this, workItemQueue1Min.GetType());
}
}
internal BaseWorkItemQueue(IReliableQueue <WorkItemProcessInfo> workItemReliableQueue, int maxWaitTimeInMinutes,
WorkItemQueueRunType workItemQueueRunType, string traceType, StatefulService statefulService)
{
BackupRestoreTrace.TraceSource.WriteInfo(traceType, "Constructing BaseWorkItemQueue with following parameters - workItemReliableQueue: {0}, maxWaitTimeInMinutes: {1}, workItemQueueRunType: {2}",
workItemReliableQueue.Name,
maxWaitTimeInMinutes,
workItemQueueRunType);
this.WorkItemReliableQueue = workItemReliableQueue;
this.StatefulService = statefulService;
this.WorkItemStore = WorkItemStore.CreateOrGetWorkItemStore(this.StatefulService).GetAwaiter().GetResult();
this.WorkItemInProcessStore =
WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(this.StatefulService).GetAwaiter().GetResult();
this.MaxWaitTimeInMilliSeconds = maxWaitTimeInMinutes == 0 ? FastQueueWaitTime : maxWaitTimeInMinutes * 60 * 1000;
this.AllowedTicks = (this.MaxWaitTimeInMilliSeconds * TimeSpan.TicksPerMillisecond) / 20;
this.workItemQueueRunType = workItemQueueRunType;
this.TraceType = traceType;
this.emptyTraceType = string.Format("{0}.{1}", traceType, "Empty");
this.WorkItemHandlerTimer = new Timer(new TimerCallback(this.ProcessWorkItemHandler),
null,
this.MaxWaitTimeInMilliSeconds,
Timeout.Infinite);
}
public async Task TryToProcessAsync(StatefulService statefulService, string processQueueTypeTrace)
{
var workItemTimeout = BackupRestoreServiceConfig.WorkItemTimeout;
try
{
workItemInProcessStore = await WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService);
bool processStatus = false;
try
{
processStatus = await UtilityHelper.InvokeWithRetry <bool>(() =>
{
var cancellationTokenSource = new CancellationTokenSource(workItemTimeout);
return(this.Process(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace));
}
);
BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "The WorkItem {0} is Processed as {1}", this, processStatus);
}
catch (Exception exception)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception While TryToProcessAsync");
AggregateException aggregateException = exception as AggregateException;
if (aggregateException != null)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}",
exception.StackTrace);
foreach (Exception innerException in aggregateException.InnerExceptions)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Inner Exception : {0} , Message : {1} , Stack Trace : {2} ",
innerException.InnerException, innerException.Message, innerException.StackTrace);
}
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Exception : {0} , Message : {1} , Stack Trace : {2} ",
exception.InnerException, exception.Message, exception.StackTrace);
}
}
if (processStatus)
{
var cancellationTokenSource = new CancellationTokenSource(workItemTimeout);
await UtilityHelper.InvokeWithRetry((() => this.ProcessSuccessHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace)));
}
else
{
var cancellationTokenSource = new CancellationTokenSource(workItemTimeout);
await UtilityHelper.InvokeWithRetry((() => this.ProcessFailureHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace)));
}
}
catch (Exception exception)
{
AggregateException aggregateException = exception as AggregateException;
;
if (aggregateException != null)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}",
exception.StackTrace);
foreach (Exception innerException in aggregateException.InnerExceptions)
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Inner Exception : {0} , Message : {1} , Stack Trace : {2} ",
innerException.InnerException, innerException.Message, innerException.StackTrace);
}
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace,
"Exception : {0} , Message : {1} , Stack Trace : {2} ",
exception.InnerException, exception.Message, exception.StackTrace);
}
}
}
