/// <summary>
/// Walk through pages of results
/// </summary>
/// <param name="url"></param>
/// <param name="depth"></param>
/// <returns></returns>
private List <GoogleSearchResult> RecursiveResults(string url, int depth = 1)
{
var ret = new List <GoogleSearchResult>();
GoogleJsonFormat jResult = null;
try
{
using (var webClient = new WebClient())
{
webClient.Encoding = Encoding.UTF8;
var json = webClient.DownloadString(url);
jResult = JsonConvert.DeserializeObject <GoogleJsonFormat>(json);
}
}
catch (Exception)
{
//This happens if you reach your daily or user limit
}
if (jResult != null)
{
if (jResult.items != null)
{
ret = jResult.items.Select(r => new GoogleSearchResult()
{
Mime = r.mime,
Url = r.link,
Title = r.title,
Description = r.snippet
}).ToList();
}
if (depth < _maxPages && jResult.queries.nextPage != null && jResult.queries.nextPage.Any())
{
//there is a next page
string oldstart = null;
if (Options.ContainsKey("start"))
{
oldstart = Options["start"];
}
Options["start"] = jResult.queries.nextPage[0].startIndex.ToString();
try
{
var nextUrl = QueryUrl();
var theRestOfThePages = RecursiveResults(nextUrl, ++depth);
ret.AddRange(theRestOfThePages);
}
catch (ArgumentOutOfRangeException)
{
//it won't search past 100 results
}
if (oldstart != null)
{
Options["start"] = oldstart;
}
else
{
Options.Remove("start");
}
}
return(ret);
}
return(ret);
}
/// <summary>
/// Used to search multiple pages and combine them with details
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
private GoogleSearchResults RecursiveResultDetailed(string url)
{
var ret = new GoogleSearchResults
{
Results = new List <GoogleSearchResult>(),
Promotions = new List <GoogleSearchResult>(),
Labels = new Dictionary <string, string>(),
TotalResults = 0,
SearchTime = 0
};
GoogleJsonFormat jResult = null;
try
{
using (var webClient = new WebClient())
{
webClient.Encoding = Encoding.UTF8;
var json = webClient.DownloadString(url);
jResult = JsonConvert.DeserializeObject <GoogleJsonFormat>(json);
}
}
catch (Exception)
{
//This happens if you reach your daily or user limit
}
if (jResult != null)
{
//handle the first page
if (jResult.items != null)
{
ret.Results = jResult.items.Select(r => new GoogleSearchResult()
{
Mime = r.mime,
Url = r.link,
Title = r.title,
Description = r.snippet
}).ToList();
}
if (jResult.queries.nextPage != null && jResult.queries.nextPage.Any())
{
//there is a next page
string oldstart = null;
if (Options.ContainsKey("start"))
{
oldstart = Options["start"];
}
Options["start"] = jResult.queries.nextPage[0].startIndex.ToString();
try
{
//let the simpler function handle the next page.
var nextUrl = QueryUrl();
var theRestOfThePages = RecursiveResults(nextUrl, 2);
ret.Results.AddRange(theRestOfThePages);
}
catch (ArgumentOutOfRangeException)
{
//it won't search past 100 results
}
if (oldstart != null)
{
Options["start"] = oldstart;
}
else
{
Options.Remove("start");
}
}
if (jResult.promotions != null)
{
ret.Promotions = jResult.promotions.Select(r => new GoogleSearchResult()
{
Mime = "text/html",
Url = r.link,
Title = r.title,
Description = r.bodyLines[0].title
}).ToList();
}
if (jResult.context.facets != null)
{
foreach (var labelGroup in jResult.context.facets)
{
foreach (var label in labelGroup)
{
ret.Labels.Add(label.label, label.anchor);
}
}
}
ret.TotalResults = long.Parse(jResult.searchInformation.totalResults);
ret.SearchTime = jResult.searchInformation.searchTime;
}
return(ret);
}
