public override IEnumerable <FreeDocument> Generate(IFreeDocument document = null)
{
InsertQueue(StartURL);
while (Enabled)
{
string currentURL;
if (queue.TryDequeue(out currentURL))
{
var urlhash = currentURL.GetHashCode();
if (StartURL != currentURL && URLHash.Contains(urlhash))
{
continue;
}
URLHash.Add(urlhash);
var doc = new FreeDocument();
doc.Add(Column, currentURL);
yield return(doc);
}
else
{
if (DelayTime > 0)
{
Thread.Sleep(1000);
XLogSys.Print.Debug("empty queue,wait 1s");
}
}
}
}
public void InsertQueue(string url)
{
if (url == null)
{
return;
}
if (!URLHash.Contains(url.GetHashCode()))
{
queue.Enqueue(url);
}
}
public override bool Init(IEnumerable <IFreeDocument> datas)
{
URLHash.Clear();
queue = new ConcurrentQueue <string>();
return(true);
}