public WebSite Fetch(string url, int depth = 0)
{
var webSite = new WebSite();
webSite.Domain = WebTools.DomainHelper.GetDomain(url);
var pages = new List<WebPage>();
webSite.Pages = pages;
var queue = new ConcurrentQueue<FetchItem>();
var visited = new HashSet<string>();
queue.Enqueue(new FetchItem { Depth = 0, Url = url });
Semaphore semaphore = new Semaphore(1, MaxThreads);
FetchItem item;
var mutex = new object();
while (true)
{
lock(mutex)
{
semaphore.WaitOne();
if (!queue.TryDequeue(out item))
{
break;
}
new Thread(() => FetchAndAdd(semaphore, item, depth, webSite.Domain, visited, queue, webSite.Pages)).Start();
}
}
return webSite;
}
public Prediction Classify(WebSite webSite)
{
var featureVector = this.Featurizer.CreateFeatureVector(webSite, this.FeatureSpace);
var confidences = this.regression.Compute(featureVector);
var prediction = new Prediction();
prediction.Confidences = new Dictionary<Target, double>();
var mapping = Model.GetTargetToInt(this.Targets);
foreach (var target in this.Targets)
{
prediction.Confidences.Add(target, confidences[mapping[target]]);
}
return prediction;
}