internal static async Task <WorkItemInProcessStore> CreateOrGetWorkItemInProcessStore(StatefulService statefulService) { if (Store == null) { BackupRestoreTrace.TraceSource.WriteInfo(TraceWorkItemInProcessStoreType, "Creating a WorkItem In Process Store"); IReliableDictionary <Guid, WorkItemProcessInfo> reliableDictionary = await statefulService.StateManager.GetOrAddAsync <IReliableDictionary <Guid, WorkItemProcessInfo> >(WorkItemStoreName); Store = new WorkItemInProcessStore(reliableDictionary, statefulService); BackupRestoreTrace.TraceSource.WriteInfo(TraceWorkItemInProcessStoreType, "Created a WorkItem In Process successfully"); } return(Store); }
private static async Task RecoverInProcessWorkItems(StatefulService statefulService) { try { WorkItemInProcessStore workItemInProcessStore = await WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService); WorkItemStore workItemStore = await WorkItemStore.CreateOrGetWorkItemStore(statefulService); List <WorkItemProcessInfo> workItemProcessInfos = await workItemInProcessStore.GetValuesAsync(); BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Work Item to be Recovered Count {0}", workItemProcessInfos.Count); foreach (WorkItemProcessInfo workItemProcessInfo in workItemProcessInfos) { WorkItem workItem = await workItemStore.GetValueAsync(workItemProcessInfo.WorkItemId); BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "Work Item Recovered Type : {0} , Id : {1}", workItem.GetType(), workItemProcessInfo.WorkItemId); Task processTask = workItem.TryToProcessAsync(statefulService, TraceType); } } catch (Exception exception) { AggregateException aggregateException = exception as AggregateException; ; if (aggregateException != null) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "Aggregate Exception Stack Trace : {0}", exception.StackTrace); foreach (Exception innerException in aggregateException.InnerExceptions) { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "Inner Exception : {0} , Message : {1} , Stack Trace : {2} ", innerException.InnerException, innerException.Message, innerException.StackTrace); } } else { BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "Exception : {0} , Message : {1} , Stack Trace : {2} ", exception.InnerException, exception.Message, exception.StackTrace); } } }
public override async Task ProcessFailureHandle(StatefulService statefulService, TimeSpan timeout, CancellationToken cancellationToken, string processQueueTypeTrace) { WorkItemInProcessStore workItemInProcessStore = await WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService); WorkItemQueue1Min workItemQueue1Min = await WorkItemQueue1Min.CreateOrGetWorkItemQueue(statefulService); WorkItemProcessInfo workItemProcessInfo = await workItemInProcessStore.GetValueAsync(this.WorkItemGuid); using (ITransaction transaction = statefulService.StateManager.CreateTransaction()) { await workItemInProcessStore.DeleteValueAsync(this.WorkItemGuid, timeout, cancellationToken, transaction); await workItemQueue1Min.AddWorkItem(workItemProcessInfo, timeout, cancellationToken, transaction); await transaction.CommitAsync(); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "Process Failure Handle WorkItemInProcessStore Delete WorkItem {0} And Enqueue in {1} ", this, workItemQueue1Min.GetType()); } }
internal BaseWorkItemQueue(IReliableQueue <WorkItemProcessInfo> workItemReliableQueue, int maxWaitTimeInMinutes, WorkItemQueueRunType workItemQueueRunType, string traceType, StatefulService statefulService) { BackupRestoreTrace.TraceSource.WriteInfo(traceType, "Constructing BaseWorkItemQueue with following parameters - workItemReliableQueue: {0}, maxWaitTimeInMinutes: {1}, workItemQueueRunType: {2}", workItemReliableQueue.Name, maxWaitTimeInMinutes, workItemQueueRunType); this.WorkItemReliableQueue = workItemReliableQueue; this.StatefulService = statefulService; this.WorkItemStore = WorkItemStore.CreateOrGetWorkItemStore(this.StatefulService).GetAwaiter().GetResult(); this.WorkItemInProcessStore = WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(this.StatefulService).GetAwaiter().GetResult(); this.MaxWaitTimeInMilliSeconds = maxWaitTimeInMinutes == 0 ? FastQueueWaitTime : maxWaitTimeInMinutes * 60 * 1000; this.AllowedTicks = (this.MaxWaitTimeInMilliSeconds * TimeSpan.TicksPerMillisecond) / 20; this.workItemQueueRunType = workItemQueueRunType; this.TraceType = traceType; this.emptyTraceType = string.Format("{0}.{1}", traceType, "Empty"); this.WorkItemHandlerTimer = new Timer(new TimerCallback(this.ProcessWorkItemHandler), null, this.MaxWaitTimeInMilliSeconds, Timeout.Infinite); }
public async Task TryToProcessAsync(StatefulService statefulService, string processQueueTypeTrace) { var workItemTimeout = BackupRestoreServiceConfig.WorkItemTimeout; try { workItemInProcessStore = await WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(statefulService); bool processStatus = false; try { processStatus = await UtilityHelper.InvokeWithRetry <bool>(() => { var cancellationTokenSource = new CancellationTokenSource(workItemTimeout); return(this.Process(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace)); } ); BackupRestoreTrace.TraceSource.WriteInfo(processQueueTypeTrace, "The WorkItem {0} is Processed as {1}", this, processStatus); } catch (Exception exception) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception While TryToProcessAsync"); AggregateException aggregateException = exception as AggregateException; if (aggregateException != null) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}", exception.StackTrace); foreach (Exception innerException in aggregateException.InnerExceptions) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Inner Exception : {0} , Message : {1} , Stack Trace : {2} ", innerException.InnerException, innerException.Message, innerException.StackTrace); } } else { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception : {0} , Message : {1} , Stack Trace : {2} ", exception.InnerException, exception.Message, exception.StackTrace); } } if (processStatus) { var cancellationTokenSource = new CancellationTokenSource(workItemTimeout); await UtilityHelper.InvokeWithRetry((() => this.ProcessSuccessHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace))); } else { var cancellationTokenSource = new CancellationTokenSource(workItemTimeout); await UtilityHelper.InvokeWithRetry((() => this.ProcessFailureHandle(statefulService, workItemTimeout, cancellationTokenSource.Token, processQueueTypeTrace))); } } catch (Exception exception) { AggregateException aggregateException = exception as AggregateException; ; if (aggregateException != null) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Aggregate Exception Stack Trace : {0}", exception.StackTrace); foreach (Exception innerException in aggregateException.InnerExceptions) { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Inner Exception : {0} , Message : {1} , Stack Trace : {2} ", innerException.InnerException, innerException.Message, innerException.StackTrace); } } else { BackupRestoreTrace.TraceSource.WriteWarning(processQueueTypeTrace, "Exception : {0} , Message : {1} , Stack Trace : {2} ", exception.InnerException, exception.Message, exception.StackTrace); } } }