public FDMessageInbox(string busName, string exchangeName, string queueName)
{
_busName = busName;
_exchangeName = exchangeName;
_queueName = queueName;
_rnd = GoodSeedRandom.Create();
}
public StochasticRecurringWorker()
{
_log = ClassLogger.Create(GetType());
_dblog = DebugOnlyLogger.Create(_log);
var rand = GoodSeedRandom.Create();
_baseTime = DateTime.UtcNow + TimeSpan.FromMinutes(rand.Next(_delay));
}
public RavenMessageInbox()
{
var config = Catalog.Factory.Resolve <IConfig>(SpecialFactoryContexts.Routed);
var name = config[MessageBoxLocalConfig.Name];
Name = name;
_rnd = GoodSeedRandom.Create();
}
public static IEnumerable <string> BindMessageToQueues(string key, ExchangeTypes ex,
IEnumerable <IQueueSpecifier> qs)
{
switch (ex)
{
case ExchangeTypes.Direct:
{
var routeBound =
qs.Where(
q =>
!q.Bindings.Any() ||
q.Bindings.Any(k => string.Compare(key, k, true, CultureInfo.InvariantCulture) == 0));
if (routeBound.Count() == 1)
{
return(EnumerableEx.OfOne(routeBound.First().Name));
}
if (routeBound.Any())
{
var matches = (from q in routeBound
select q.Name).Distinct().Shuffle(GoodSeedRandom.Create());
return(matches.Take(1));
}
return(Enumerable.Empty <string>());
}
//break;
case ExchangeTypes.Fanout:
{
return(qs.Select(q => q.Name));
}
//break;
case ExchangeTypes.Topic:
{
return((from q in qs
where q.Bindings.Any(k => TopicMatch(key, k))
select q.Name).Distinct());
}
//break;
}
return(Enumerable.Empty <string>());
}
public RecurringWork()
{
rand = GoodSeedRandom.Create();
}
private static IEnumerable <T> Shuffle <T>(IEnumerable <T> input)
{
Random rnd = GoodSeedRandom.Create();
return(input.OrderBy(x => rnd.Next()));
}
public bool OnStart()
{
_listener.Listen(
new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WakeWorkflowJob), WakeWorkflow),
new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WorkflowCreate), NewWorkflowInstance),
new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WorkflowStateMachineRetry), FireRetryMsg),
new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(FireWorkflowTriggerJob), FireTriggerJobMsg));
_broadcastListener.Listen(new KeyValuePair <Type, Action <object, CancellationToken, IMessageAcknowledge> >(typeof(WorkflowTrigger), DistributeWorkflowTriggers));
var hostEnvironment = Catalog.Factory.Resolve <IHostEnvironment>();
var hostId = hostEnvironment.GetCurrentHostIdentifier(HostEnvironmentConstants.DefaultHostScope);
var instancesQuery = new QuerySpecification
{
BookMark = new GenericPageBookmark {
PageSize = 100
},
Where = new Filter
{
Rules = new Comparison[]
{
new Comparison
{
Data = WorkflowStatus.Complete.ToString(),
Field = "Status",
Test = Test.NotEqual
}
}
}
};
Task.Factory.StartNew(() =>
{
// get this host's chunk of workflow instances to run.
Random rnd = GoodSeedRandom.Create();
bool done = false;
bool first = true;
DateTime lastException = DateTime.MinValue;
while (!done)
{
try
{
int acquired = 0;
IEnumerable <WorkflowInstanceInfo> instances = Enumerable.Empty <WorkflowInstanceInfo>();
instancesQuery.BookMark = new GenericPageBookmark {
PageSize = 100
};
while (instancesQuery.BookMark.More)
{
var qr = _instanceData.Query(instancesQuery);
if (qr.Items.Any())
{
instances = instances.Concat(qr.Items);
}
}
instances = instances.ToArray();
IEnumerable <WorkflowInstanceInfo> tryInstances;
// first pass, pick out existing workflows in a pattern less
// likely to interfere with other workflow hosts trying to acquire
if (first)
{
int startAt;
startAt = rnd.Next(3) * 3;
if (startAt > instances.Count())
{
startAt = rnd.Next(instances.Count() / 3);
}
tryInstances = instances.TakeEvery(startAt, 3);
first = false;
}
else
{
// subsequent passes, be greedy
tryInstances = instances;
}
int waitDur;
foreach (var inst in tryInstances)
{
if (TryAcquireWorkflow(inst.Id) != null)
{
acquired++;
_log.InfoFormat("Host {0} acquired workflow instance {1} ...",
hostId, inst.Id);
// give other hosts a little space to take some.
waitDur = rnd.Next(160);
Thread.Sleep(waitDur);
}
else
{
_log.InfoFormat(
"Host {0} could not acquire workflow instance {1} ...", hostId,
inst.Id);
}
}
if (!first && acquired == 0)
{
done = true;
}
waitDur = rnd.Next(7654);
_log.InfoFormat(
"Host {0} acquired {1} pre-existing workflows, waiting {2} milliseconds to try to attain more.",
hostId, acquired, waitDur);
Thread.Sleep(waitDur);
}
catch (Exception ex)
{
var es = string.Format("exception running workflow host {0}",
ex.TraceInformation());
_dblog.Error(es);
if (DateTime.UtcNow - lastException > TimeSpan.FromMinutes(1.0))
{
_log.TraceException(ex);
var alert = Catalog.Factory.Resolve <IApplicationAlert>();
alert.RaiseAlert(ApplicationAlertKind.System, es);
}
}
}
});
return(true);
}
private void ScheduleRetry(string machine, int retryCount, int minMinutes, int maxMinutes, int delta, bool retrySleep)
{
int currentRetryCount = 0;
try
{
if (!_wfLock.Wait(TimeSpan.FromSeconds(60.0)))
{
throw new SynchronizationLockException();
}
Workspace.Batch(wsd =>
{
currentRetryCount = wsd.Get(WorkflowWorkspace.CurrentRetryCount, 1);
currentRetryCount++;
});
}
catch (SynchronizationLockException)
{
var es = string.Format("Timed out waiting to write workspace data to {0}", Id);
_log.Error(es);
var on = Catalog.Factory.Resolve <IApplicationAlert>();
on.RaiseAlert(ApplicationAlertKind.System, es);
}
finally
{
_wfLock.Release();
}
if (currentRetryCount < retryCount)
{
var rand = GoodSeedRandom.Create();
var incrementSeconds = (int)((Math.Pow(2, currentRetryCount) - 1) * rand.Next((int)(delta * 60 * 0.8),
(int)(delta * 60 * 1.2)));
var timeToSleepSec = Math.Min((minMinutes * 60) + incrementSeconds, (maxMinutes * 60));
var retryInterval = TimeSpan.FromSeconds(timeToSleepSec);
var retrySchedule = DateTime.UtcNow + retryInterval;
var js = Catalog.Factory.Resolve <IJobScheduler>();
js.ScheduleJob(
new WorkflowStateMachineRetry
{
Id = Id,
Machine = machine
},
retrySchedule,
null,
WorkflowShared.WorkflowJobsRoute);
try
{
if (!_wfLock.Wait(TimeSpan.FromSeconds(60.0)))
{
throw new SynchronizationLockException();
}
Workspace.Batch(wsd => wsd.Put(WorkflowWorkspace.CurrentRetryCount, currentRetryCount));
}catch (SynchronizationLockException)
{
var es = string.Format("Timed out waiting to write workspace data to {0}", Id);
_log.Error(es);
var on = Catalog.Factory.Resolve <IApplicationAlert>();
on.RaiseAlert(ApplicationAlertKind.System, es);
}
finally
{
_wfLock.Release();
}
if (retrySleep)
{
Sleep();
}
}
else
{
StateMachines[machine].Fire(CommonTriggers.StateFailTrigger);
}
}
public void Start()
{
var rnd = GoodSeedRandom.Create();
_updateCycle.Recur(TimeSpan.FromSeconds(90.0 + (rnd.NextDouble() * 30.0)), Action, null);
}
private readonly Guid _reservationId = Guid.NewGuid(); // yeah. Each inbox reserves messages
public RavenMessageInbox(string name)
{
Name = name;
_rnd = GoodSeedRandom.Create();
}
