private async Task Brush(CombLink destination, string urlDomain, int maxDepth = 1, int depth = 0)
{
if (destination.Combed)
{
return;
}
await Task.Run(async() =>
{
var htmlContent = await httpClient.FetchHtmlAsync(destination.Value);
if (string.IsNullOrEmpty(htmlContent))
{
return;
}
var matches = regex.Matches(htmlContent);
var tasks = new List <Task>();
depth++;
foreach (Match match in matches)
{
var link = match.ToLink(urlDomain, destination);
if (link.IsDescendent && depth <= maxDepth)
{
link.SetCombed();
tasks.Add(Brush(link, urlDomain, maxDepth, depth));
}
destination.AddDescendent(link.Value, link);
}
Task.WaitAll(tasks.ToArray());
});
}
public void AddDescendent(string key, CombLink value)
{
Links.TryAdd(key, value);
if (!Children.ContainsKey(key))
{
// Core 2.2 Descendents.TryAdd(key, value);
Children.Add(key, value);
}
}
public async Task <CombResponse> Brush(CombRequest request)
{
var response = Validate(request, new CombResponse());
if (!response.Success)
{
return(response);
}
var urlDomain = GetUrlDomain(request);
var link = new CombLink(request.Url, urlDomain);
await Brush(link, urlDomain, request.Depth);
response.Result = link;
return(response);
}
public CombLink(string value, string domain, CombLink parent) : base(value, domain, parent.Links)
{
this.parent = parent;
Children = new Dictionary <string, CombLink>();
}
public static CombLink ToLink(this Match match, string domain, CombLink parent)
{
return(new CombLink(match.Value, domain, parent));
}