protected override Task SuccessPipelineAsync(SpiderContext context) { locations.Add(Tuple.Create(context.Data.latitude, context.Data.longitude)); //Console.WriteLine(context.Data.latitude); //Console.WriteLine(context.Data.longitude); //Console.WriteLine(context.Data.name); //Console.WriteLine("-------------------------------"); return(base.SuccessPipelineAsync(context)); }
protected async virtual Task ErrorPipelineAsync(SpiderContext context) { Log.Debug("Error Pipeline"); }
protected async virtual Task SuccessPipelineAsync(SpiderContext context) { Log.Debug("Sucess Pipeline"); }
public async Task <bool> RunAsync() { Log.Information($"Starting RunAsync ... {this.SpiderName}"); try { this.ct = new SpiderContext(); ct.Url = this.url; ct.Spider = this.SpiderName; ct.Bag = JObject.FromObject(ViewBag); await SetupBeforeRunAsync(); var hasNextPage = false; do { await RunDownloaderAsync(); Log.Information("Running... " + this.SpiderName); ct.Url = this.url; ct.Spider = this.SpiderName; ct.Bag = JObject.FromObject(ViewBag); var obj = OnRun(); foreach (var item in obj) { ct = new SpiderContext(); ct.Url = this.url; ct.Spider = this.SpiderName; ct.Bag = JObject.FromObject(ViewBag); try { // After(item.Data); if (item.Error == null) { await SuccessPipelineAsync(item); } else { await ErrorPipelineAsync(item); } } catch (Exception ex) { Console.WriteLine(JsonConvert.SerializeObject(this)); Console.WriteLine(JsonConvert.SerializeObject(item)); Console.WriteLine(JsonConvert.SerializeObject(ex)); Log.Error(ex, "Error looping spider engine"); } } System.Console.WriteLine("Finish... " + this.SpiderName); if (!nofollow) { var nextPage = this.FollowPage(); if (this.Downloader.HttpMethod == "POST") { // post the nextPage can be same of url hasNextPage = !string.IsNullOrEmpty(nextPage); } else { // get required that nextPage different of url hasNextPage = !string.IsNullOrEmpty(nextPage) && nextPage != url; } if (hasNextPage) { this.SetUrl(nextPage); } } } while (hasNextPage); return(true); } catch (Exception ex) { ct.Data.SpiderEngine = JsonConvert.SerializeObject(this); ct.Data.Exception = JsonConvert.SerializeObject(ex); //ct.RunEmbedMetadataPipeline(); Log.Error(ex, ct.Data.ToString()); throw ex; } return(false); }