/// <summary>
/// Create and start a Crawl Job with the specific parameters.
/// If the job already exists, updates its status
/// </summary>
/// <param name="settings">Settings for the crawl</param>
/// <returns>An instance of CrawlJob with the settings and update status</returns>
public async Task <CrawlJob> CreateUpdateJobAsync(CrawlbotSettings settings)
{
StringBuilder url = new StringBuilder();
url.AppendFormat("v{0}/crawl", this.version);
string separator = "?";
foreach (var k in JObject.FromObject(settings))
{
if (k.Value != null)
{
url.AppendFormat("{0}{1}={2}", separator, k.Key, k.Value);
if (separator == "?")
{
separator = "&";
}
}
}
var result = await httpClient.GetStringAsync(url.ToString());
if (!string.IsNullOrWhiteSpace(result))
{
CrawlJob job = JsonConvert.DeserializeObject <CrawlJob>(result);
job.Token = settings.Token;
return(job);
}
else
{
return(null);
}
}
/// <summary>
/// Resume a job
/// </summary>
/// <param name="job">An instance of the CrawlJob</param>
/// <returns>Returns the updated job</returns>
public async Task <CrawlJob> ResumeAsync(CrawlJob job)
{
Dictionary <string, string> parameters = new Dictionary <string, string>();
parameters.Add("pause", "0");
return(await CrawlOperations(job, parameters));
}
private async Task <CrawlJob> CrawlOperations(CrawlJob job, Dictionary <string, string> parameters)
{
try
{
parameters.Add("token", job.Token);
parameters.Add("name", job.Name);
var response = await diffbotCall.GetAsync("bulk", this.version.Value, parameters);
if (response["Error"] != null)
{
CrawlJob jobResponse = new CrawlJob();
jobResponse.Error = response["StatusCode"].ToString() + ": " + response["Error"].ToString();
return(jobResponse);
}
else
{
CrawlJob jobResponse = JsonConvert.DeserializeObject <CrawlJob>(response.ToString());
jobResponse.Token = job.Token;
return(jobResponse);
}
}
catch (Exception ex)
{
CrawlJob jobResponse = new CrawlJob();
jobResponse.Error = ex.InnerException != null ? " " + ex.InnerException.Message : ex.Message;
return(jobResponse);
}
}