Пример #1
0
        /// <summary>
        /// Выполняет function до успешного завершения, либо истечения количества попыток
        /// </summary>
        /// <param name="function">функция для выполнения</param>
        /// <param name="attemptsCount">количество попыток</param>
        /// <param name="timeInterval">временной интервал</param>
        /// <returns></returns>
        public async static Task Execute(ExecuteParams executeParams)
        {
            bool needRepeat = false;
            try
            {
                await executeParams.RequestFunction(executeParams.Url, executeParams.Proxy, executeParams.Step);
                needRepeat = false;
            }
            catch (Exception ex)
            {
                executeParams.AttemptsCount--;
                if (executeParams.AttemptsCount > 0)
                    needRepeat = true;
                else
                    executeParams.ExceptionAction(executeParams.Url, executeParams.Proxy, executeParams.Step, ex);
            }

            if (needRepeat)
            {
                await Task.Delay(executeParams.TimeInterval);
                await Execute(executeParams);
            }
        }
Пример #2
0
        async Task<List<String>> GetAtpUrls(List<String> regionUrls)
        {
            var index = 0;
            var atpUrls = new ConcurrentQueue<String>();
            var tasks = new List<Task>();
            foreach (var regionUrl in regionUrls)
            //var regionUrl = regionUrls.First();
            {
                var executeParams = new ExecuteParams()
                {
                    Step = ++index,
                    Url = regionUrl,
                    TimeInterval = new TimeSpan(0, 0, 1),
                    AttemptsCount = 3,
                    RequestFunction = async (url, proxy, step) =>
                    {
                        var html = await HttpHelper.HttpGet(url);

                        foreach (var atpUrl in ParseAtpUrlsFromPage(html))
                            atpUrls.Enqueue(atpUrl);

                        Logger.Instance.Info(String.Format("{0}/{1} получено страниц с АТП {2}", step, regionUrls.Count, url));
                    },
                    ExceptionAction = (url, proxy, step, ex) =>
                    {
                        Logger.Instance.Error("Не удалось получить страницу с АТП " + url, ex);
                    }
                };

                tasks.Add(TasksExecuteHelper.Execute(executeParams));

                await Task.Delay(Properties.Settings.Default.OrgPageRegionRequestDelay);
            }

            Task.WaitAll(tasks.ToArray());

            return atpUrls.ToList();
        }
Пример #3
0
        async Task<List<Atp>> GetAtps(List<String> atpUrls)
        {
            var index = 0;
            var atpsDict = new ConcurrentDictionary<String, Atp>();
            var tasks = new List<Task>();
            foreach (var atpUrl in atpUrls)
            //var atpUrl = atpUrls.First();
            {
                var executeParams = new ExecuteParams()
                {
                    Step = ++index,
                    Url = atpUrl,
                    TimeInterval = new TimeSpan(0, 0, 1),
                    AttemptsCount = 3,
                    RequestFunction = async (url, proxy, step) =>
                    {
                        var html = await HttpHelper.HttpGet(url);
                        var atp = new Atp()
                        {
                            Url = url
                        };

                        ParseAtpFromPage(html, atp);

                        atpsDict[atp.Url] = atp;

                        Logger.Instance.Info(String.Format("{0}/{1} получено АТП {2}", step, atpUrls.Count, atp.Url));
                    },
                    ExceptionAction = (url, proxy, step, ex) =>
                    {
                        Logger.Instance.Error("Не удалось получить АТП " + url, ex);
                    }
                };

                tasks.Add(TasksExecuteHelper.Execute(executeParams));

                await Task.Delay(Properties.Settings.Default.OrgPageAtpRequestDelay);
            }

            Task.WaitAll(tasks.ToArray());

            return atpsDict.Values.ToList();
        }