protected async void Process(Page page, Parameters parameters) { var cts = new CancellationTokenSource(); ServicePointManager.DefaultConnectionLimit = 10; ServicePointManager.MaxServicePointIdleTime = 100; ServicePointManager.Expect100Continue = false; ServicePointManager.CheckCertificateRevocationList = false; _throttle = new SemaphoreSlim(initialCount: 24); await _throttle.WaitAsync(); page.Html.Parse() .Standartize(parameters.Root) .Where(url => url.StartsWith(parameters.Root) && !Regex.Match(url, _filter).Success) .AsParallel() .ForAll(url => _tasks.Add(Task.Run(async () => { try { var response = await _client.PostAsync<Parameters>(_host, new Parameters { Url = url }, new JsonMediaTypeFormatter(), cts.Token).ConfigureAwait(false); } catch (TaskCanceledException) { cts.Cancel(); } catch (Exception ex) { _logger.Error(ex); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine(ex.Message); } finally { _throttle.Release(); } }))); }
public static void Push(string url, Page page) { if (_frontier.ContainsKey(url) && page != null) { return; } if (page != null) { _frontier.AddOrUpdate(url, page, (key, p) => page); _logger.Trace(string.Format("{0}", url)); Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("pushed: {0} :{1}", url, _frontier.Count); return; } if (!_frontier.ContainsKey(url)) { _queue.TryAdd(url); } }