public void GivenIHaveQueuedTargets(Table table)
{
var queueTargets = TableAid.ObjectCreator <QueueTarget>(table, (row, target) =>
{
RecallAid.It[row["!this"]] = target;
});
var countById = queueTargets.GroupBy(x => x.Id);
foreach (var g in countById)
{
Assert.AreEqual(1, g.Count(), string.Format("More than one QueueTarget exists with ID {0}.", g.Key));
}
TargetQueue.Setup(tq => tq.Targets).Returns(queueTargets.ToList().AsReadOnly());
}
protected void ExtractTargetsFromCurrentAndEnqueue()
{
string[] more = ExtractTargets(Current);
for (int i = 0; i < more.Length; i++)
{
string extracted = more[i];
if (string.IsNullOrEmpty(extracted))
{
continue;
}
string compareTo = IsCaseSensitive ? extracted.ToLowerInvariant() : extracted;
if (!WasProcessed(compareTo))
{
if (MaxQueueSize == 0 || (MaxQueueSize > 0 && TargetQueue.Count < MaxQueueSize))
{
TargetQueue.Enqueue(extracted);
}
}
}
}
public override Action GetAction(CancellationToken cancellation)
{
return(() =>
{
var nextMessages = TargetQueue.RemoveMessages(100);
if (nextMessages != null && nextMessages.Any())
{
logger.Info(@"[ServiceStack.Webhooks.Azure.Worker.BaseQueueProcessor] Processing {0} messages from queue {1}".Fmt(nextMessages.Count, TargetQueue.QueueName));
nextMessages.ForEach(msg =>
{
var handled = TryProcessMessage(msg);
if (!handled)
{
PlaceMessageOnUnhandled(msg);
}
});
}
});
}
public void Crawl(string rootTarget)
{
Exec.Start(ThreadName, () =>
{
Root = rootTarget;
_processed = new HashSet <string>();
TargetQueue.Enqueue(Root);
string current;
while (TargetQueue.TryDequeue(out current))
{
Current = current;
CurrentAction = CrawlerState.Action.Extracting;
ExtractTargetsFromCurrentAndEnqueue();
CurrentAction = CrawlerState.Action.Processing;
ProcessTarget(Current);
_processed.Add(IsCaseSensitive ? Current : Current.ToLowerInvariant());
}
CurrentAction = CrawlerState.Action.Idle;
});
}