private async Task <HostSource> LoadHostFileAsync(string url) { var response = await _policy.ExecuteAndCaptureAsync(() => _httpClient.GetAsync(url)); if (response.Outcome == OutcomeType.Failure) { return(null); } var content = await response.Result.Content.ReadAsStringAsync(); var rawHosts = HostParser.Parse(content); return(new HostSource { Hosts = rawHosts }); }
/// <summary> /// Given a set of host sources will iterate them and append them to the existing set of hosts and return the modified object. /// </summary> /// <param name="blocklist"></param> /// <param name="newSources"></param> /// <returns></returns> public async Task <Blocklist> AppendNewAsync(Blocklist blocklist, HostSource newSources) { foreach (var host in newSources.Hosts) { blocklist.Hostnames.Add(host); } foreach (var host in HostParser.Parse(newSources.Raw)) { blocklist.Hostnames.Add(host); } foreach (var url in newSources.Links) { using (_log.MeasureDuration($"FetchHosts - {url}")) { var src = await LoadHostFileAsync(url); _log.LogInformation($"Adding {src.Hosts.Length} hosts from {url}"); // don't parallelize, otherwise we would need to synchronize the hashset! blocklist = await AppendNewAsync(blocklist, src); } } return(blocklist); }