public void ProtectedRun() { DateTime lastException = DateTime.MinValue; while (!_ct.IsCancellationRequested) { try { // any wfs fall through the cracks and are asleep but have triggers? IEnumerable <WorkflowInstanceInfo> sleeping = Enumerable.Empty <WorkflowInstanceInfo>(); var instancesQuery = new QuerySpecification { BookMark = new GenericPageBookmark { PageSize = 100 }, Where = new Filter { Rules = new Comparison[] { new Comparison { Data = WorkflowStatus.Sleeping.ToString(), Field = "Status", Test = Test.Equal } } } }; instancesQuery.BookMark = new GenericPageBookmark { PageSize = 100 }; while (instancesQuery.BookMark.More) { var qr = _instanceData.Query(instancesQuery); if (qr.Items.Any()) { sleeping = sleeping.Concat(qr.Items); } } sleeping = sleeping.ToArray(); Parallel.ForEach(sleeping, nc => { try { // check for triggers IEnumerable <WorkflowMachineState> hasTriggers = Enumerable.Empty <WorkflowMachineState>(); var hasTriggersQuery = new QuerySpecification { BookMark = new GenericPageBookmark { PageSize = 100 }, Where = new Filter { Rules = new Comparison[] { new Comparison { Field = "Parent", Data = nc.Id, Test = Test.Equal } } } }; while (hasTriggersQuery.BookMark.More) { var qr = _machineStateData.Query(hasTriggersQuery); if (qr.Items.Any()) { hasTriggers = hasTriggers.Concat(qr.Items); } } // it has some, try to acquire it if (hasTriggers.Any()) { _ct.ThrowIfCancellationRequested(); _log.InfoFormat( "Sleeping workflow {0} has {1} waiting triggers. Waking.", nc.Id, hasTriggers.Count()); TryAcquireWorkflow(nc.Id); } } catch (Exception ex) { _dblog.ErrorFormat("Problem while waking workflow {0}:{2}", nc.Id, ex.ToString()); } }); // groom completed and data IEnumerable <WorkflowInstanceInfo> completedInstances = Enumerable.Empty <WorkflowInstanceInfo>(); var completedQuery = new QuerySpecification { BookMark = new GenericPageBookmark { PageSize = 100 }, Where = new Filter { Rules = new Comparison[] { new Comparison { Data = WorkflowStatus.Complete.ToString(), Field = "Status", Test = Test.Equal } } } }; completedQuery.BookMark = new GenericPageBookmark { PageSize = 100 }; while (completedQuery.BookMark.More) { var qr = _instanceData.Query(completedQuery); if (qr.Items.Any()) { completedInstances = completedInstances.Concat(qr.Items); } } completedInstances = completedInstances.ToArray(); Parallel.ForEach(completedInstances, ci => { try { _log.InfoFormat("Grooming completed workflow instance {0}", ci.Id); _ct.ThrowIfCancellationRequested(); // delete workspaces var workspace = Catalog.Preconfigure() .Add(WorkspaceLocalConfig.WorkspaceName, WorkflowShared.WorkflowInstanceWorkspaceName(ci.Id)) .ConfiguredResolve <IWorkspace>(_workflowWorkspaceKey); workspace.DeleteWorkspace(); _ct.ThrowIfCancellationRequested(); IEnumerable <WorkflowMachineState> states = Enumerable.Empty <WorkflowMachineState>(); var msQuery = new QuerySpecification { BookMark = new GenericPageBookmark { PageSize = 100 }, Where = new Filter { Rules = new Comparison[] { new Comparison { Data = ci.Id, Field = "Parent", Test = Test.Equal } } } }; msQuery.BookMark = new GenericPageBookmark { PageSize = 100 }; while (msQuery.BookMark.More) { var qr = _machineStateData.Query(msQuery); if (qr.Items.Any()) { states = states.Concat(qr.Items); } } _machineStateData.DeleteBatch(states.ToList()); _instanceData.Delete(ci); } catch (Exception ex) { var es = string.Format( "Problem while grooming completed workflow {0}:{1}", ci.Id, ex.ToString()); _dblog.Error(es); var on = Catalog.Factory.Resolve <IApplicationAlert>(); on.RaiseAlert(ApplicationAlertKind.Unknown, es); } }); var chill = TimeSpan.FromMinutes((30.0 * _rndGroomTime.NextDouble()) + 1.0); _log.InfoFormat("Waiting {0} minutes until next sweep cycle.", chill.TotalMinutes); _ct.WaitHandle.WaitOne(chill); } catch (Exception ex) { var es = string.Format("exception running workflow host {0}", ex.TraceInformation()); _dblog.Error(es); if (DateTime.UtcNow - lastException > TimeSpan.FromMinutes(1.0)) { _log.TraceException(ex); var alert = Catalog.Factory.Resolve <IApplicationAlert>(); alert.RaiseAlert(ApplicationAlertKind.System, es); } } } }