protected override Task SuccessPipelineAsync(SpiderContext context)
{
locations.Add(Tuple.Create(context.Data.latitude, context.Data.longitude));
//Console.WriteLine(context.Data.latitude);
//Console.WriteLine(context.Data.longitude);
//Console.WriteLine(context.Data.name);
//Console.WriteLine("-------------------------------");
return(base.SuccessPipelineAsync(context));
}
protected async virtual Task ErrorPipelineAsync(SpiderContext context)
{
Log.Debug("Error Pipeline");
}
protected async virtual Task SuccessPipelineAsync(SpiderContext context)
{
Log.Debug("Sucess Pipeline");
}
public async Task <bool> RunAsync()
{
Log.Information($"Starting RunAsync ... {this.SpiderName}");
try
{
this.ct = new SpiderContext();
ct.Url = this.url;
ct.Spider = this.SpiderName;
ct.Bag = JObject.FromObject(ViewBag);
await SetupBeforeRunAsync();
var hasNextPage = false;
do
{
await RunDownloaderAsync();
Log.Information("Running... " + this.SpiderName);
ct.Url = this.url;
ct.Spider = this.SpiderName;
ct.Bag = JObject.FromObject(ViewBag);
var obj = OnRun();
foreach (var item in obj)
{
ct = new SpiderContext();
ct.Url = this.url;
ct.Spider = this.SpiderName;
ct.Bag = JObject.FromObject(ViewBag);
try
{
// After(item.Data);
if (item.Error == null)
{
await SuccessPipelineAsync(item);
}
else
{
await ErrorPipelineAsync(item);
}
}
catch (Exception ex)
{
Console.WriteLine(JsonConvert.SerializeObject(this));
Console.WriteLine(JsonConvert.SerializeObject(item));
Console.WriteLine(JsonConvert.SerializeObject(ex));
Log.Error(ex, "Error looping spider engine");
}
}
System.Console.WriteLine("Finish... " + this.SpiderName);
if (!nofollow)
{
var nextPage = this.FollowPage();
if (this.Downloader.HttpMethod == "POST")
{
// post the nextPage can be same of url
hasNextPage = !string.IsNullOrEmpty(nextPage);
}
else
{
// get required that nextPage different of url
hasNextPage = !string.IsNullOrEmpty(nextPage) && nextPage != url;
}
if (hasNextPage)
{
this.SetUrl(nextPage);
}
}
} while (hasNextPage);
return(true);
}
catch (Exception ex)
{
ct.Data.SpiderEngine = JsonConvert.SerializeObject(this);
ct.Data.Exception = JsonConvert.SerializeObject(ex);
//ct.RunEmbedMetadataPipeline();
Log.Error(ex, ct.Data.ToString());
throw ex;
}
return(false);
}
