private async Task SendUris(ILinkExtractor reader)
{
var l = new List <QueueItem>();
if (reader == null)
{
return;
}
Uri uri = reader.NextUri();
while (uri != null && l.Count < FetchoConfiguration.Current.MaxLinksToExtractFromOneResource * 10)
{
var item = new QueueItem()
{
SourceUri = reader.CurrentSourceUri,
TargetUri = uri
};
l.Add(item);
uri = reader.NextUri();
}
LinksExtracted += l.Count;
// effectively block until the URLs are accepted
await PrioritisationBuffer.SendOrWaitAsync(l.Randomise().Take(FetchoConfiguration.Current.MaxLinksToExtractFromOneResource)).ConfigureAwait(false);
}
private void OutputUris(ILinkExtractor reader)
{
if (reader == null)
{
return;
}
Uri uri = reader.NextUri();
while (uri != null)
{
Console.WriteLine("{0}\t{1}", reader.CurrentSourceUri, uri);
uri = reader.NextUri();
}
}