private async Task <HostSource> LoadHostFileAsync(string url)
{
var response = await _policy.ExecuteAndCaptureAsync(() => _httpClient.GetAsync(url));
if (response.Outcome == OutcomeType.Failure)
{
return(null);
}
var content = await response.Result.Content.ReadAsStringAsync();
var rawHosts = HostParser.Parse(content);
return(new HostSource
{
Hosts = rawHosts
});
}
/// <summary>
/// Given a set of host sources will iterate them and append them to the existing set of hosts and return the modified object.
/// </summary>
/// <param name="blocklist"></param>
/// <param name="newSources"></param>
/// <returns></returns>
public async Task <Blocklist> AppendNewAsync(Blocklist blocklist, HostSource newSources)
{
foreach (var host in newSources.Hosts)
{
blocklist.Hostnames.Add(host);
}
foreach (var host in HostParser.Parse(newSources.Raw))
{
blocklist.Hostnames.Add(host);
}
foreach (var url in newSources.Links)
{
using (_log.MeasureDuration($"FetchHosts - {url}"))
{
var src = await LoadHostFileAsync(url);
_log.LogInformation($"Adding {src.Hosts.Length} hosts from {url}");
// don't parallelize, otherwise we would need to synchronize the hashset!
blocklist = await AppendNewAsync(blocklist, src);
}
}
return(blocklist);
}