Пример #1
0
        protected async void Process(Page page, Parameters parameters)
        {
            var cts = new CancellationTokenSource();
            ServicePointManager.DefaultConnectionLimit = 10;
            ServicePointManager.MaxServicePointIdleTime = 100;
            ServicePointManager.Expect100Continue = false;
            ServicePointManager.CheckCertificateRevocationList = false;

            _throttle = new SemaphoreSlim(initialCount: 24);

            await _throttle.WaitAsync();

            page.Html.Parse()
            .Standartize(parameters.Root)
            .Where(url => url.StartsWith(parameters.Root) && !Regex.Match(url, _filter).Success)
            .AsParallel()
            .ForAll(url =>
                _tasks.Add(Task.Run(async () =>
                {
                    try
                    {
                        var response = await _client.PostAsync<Parameters>(_host, new Parameters { Url = url }, new JsonMediaTypeFormatter(), cts.Token).ConfigureAwait(false);
                    }
                    catch (TaskCanceledException)
                    {
                        cts.Cancel();
                    }
                    catch (Exception ex)
                    {
                        _logger.Error(ex);
                        Console.ForegroundColor = ConsoleColor.Red;
                        Console.WriteLine(ex.Message);
                    }
                    finally
                    {
                        _throttle.Release();
                    }
                })));
        }
Пример #2
0
        public static void Push(string url, Page page)
        {
            if (_frontier.ContainsKey(url) && page != null)
            {
                return;
            }

            if (page != null)
            {
                _frontier.AddOrUpdate(url, page, (key, p) => page);
                _logger.Trace(string.Format("{0}", url));
                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine("pushed: {0} :{1}", url, _frontier.Count);

                return;
            }

            if (!_frontier.ContainsKey(url))
            {
                _queue.TryAdd(url);
            }
        }