public bool OnStart() { _listener.Listen( new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WakeWorkflowJob), WakeWorkflow), new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WorkflowCreate), NewWorkflowInstance), new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WorkflowStateMachineRetry), FireRetryMsg), new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(FireWorkflowTriggerJob), FireTriggerJobMsg)); _broadcastListener.Listen(new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WorkflowTrigger), DistributeWorkflowTriggers)); var hostEnvironment = Catalog.Factory.Resolve <IHostEnvironment>(); var hostId = hostEnvironment.GetCurrentHostIdentifier(HostEnvironmentConstants.DefaultHostScope); var instancesQuery = new QuerySpecification { BookMark = new GenericPageBookmark { PageSize = 100 }, Where = new Filter { Rules = new Comparison[] { new Comparison { Data = WorkflowStatus.Complete.ToString(), Field = "Status", Test = Test.NotEqual } } } }; Task.Factory.StartNew(() => { // get this host's chunk of workflow instances to run. Random rnd = GoodSeedRandom.Create(); bool done = false; bool first = true; DateTime lastException = DateTime.MinValue; while (!done) { try { int acquired = 0; IEnumerable <WorkflowInstanceInfo> instances = Enumerable.Empty <WorkflowInstanceInfo>(); instancesQuery.BookMark = new GenericPageBookmark { PageSize = 100 }; while (instancesQuery.BookMark.More) { var qr = _instanceData.Query(instancesQuery); if (qr.Items.Any()) { instances = instances.Concat(qr.Items); } } instances = instances.ToArray(); IEnumerable <WorkflowInstanceInfo> tryInstances; // first pass, pick out existing workflows in a pattern less // likely to interfere with other workflow hosts trying to acquire if (first) { int startAt; startAt = rnd.Next(3) * 3; if (startAt > instances.Count()) { startAt = rnd.Next(instances.Count() / 3); } tryInstances = instances.TakeEvery(startAt, 3); first = false; } else { // subsequent passes, be greedy tryInstances = instances; } int waitDur; foreach (var inst in tryInstances) { if (TryAcquireWorkflow(inst.Id) != null) { acquired++; _log.InfoFormat("Host {0} acquired workflow instance {1} ...", hostId, inst.Id); // give other hosts a little space to take some. waitDur = rnd.Next(160); Thread.Sleep(waitDur); } else { _log.InfoFormat( "Host {0} could not acquire workflow instance {1} ...", hostId, inst.Id); } } if (!first && acquired == 0) { done = true; } waitDur = rnd.Next(7654); _log.InfoFormat( "Host {0} acquired {1} pre-existing workflows, waiting {2} milliseconds to try to attain more.", hostId, acquired, waitDur); Thread.Sleep(waitDur); } catch (Exception ex) { var es = string.Format("exception running workflow host {0}", ex.TraceInformation()); _dblog.Error(es); if (DateTime.UtcNow - lastException > TimeSpan.FromMinutes(1.0)) { _log.TraceException(ex); var alert = Catalog.Factory.Resolve <IApplicationAlert>(); alert.RaiseAlert(ApplicationAlertKind.System, es); } } } }); return(true); }