private async Task <Tuple <Request, Response, Item> > NextRequestFromScheduler() { var request = this._scheduler.Pop(); if (request == null) { return(null); } try { this._actives.Add(request); var response = await this._downloader.AsyncFetch(request, _spider); if (response == null) { return(null); } var spiderResponse = _spider.Extract(response); var tasks = new List <Task>(); if (spiderResponse.NewRequests != null) { tasks.Add(Task.Run(() => { foreach (var newRequest in spiderResponse.NewRequests) { newRequest.Deep = request.Deep + 1; this._scheduler.Push(newRequest); } })); } if (this._piplelines != null && this._piplelines.Length > 0 && spiderResponse.Item != null) { tasks.Add(Task.Run(async() => { foreach (var pipleline in this._piplelines) { await pipleline.Process(spiderResponse.Item, this._spider); } })); } if (tasks.Count > 0) { await Task.WhenAll(tasks); } return(Tuple.Create(request, response, spiderResponse.Item)); } catch (Exception ex) { _logger.Error(new EngineException(ex.Message + ("\n URL:" + request.Url), ex)); this._uncompleted.Add(request); return(null); } finally { this._actives.TryTake(out request); } }