public SiteScraper(string url)
//take care of any initializing
: this()
{
//start the link tree
try
{
LinkTree.Add(new LinkNode(url));
}
catch (ArgumentException e)
{
Console.WriteLine(e.Message);
}
//get the links from the current page
HtmlDocument page = GetPage(url);
IEnumerable <string> pageLinks = ExtractLinks(page, url);
//add these links to the LinkTree
foreach (string link in pageLinks)
{
try
{
LinkTree.Add(new LinkNode(link, url));
}
catch (ArgumentException e)
{
Console.WriteLine(e.Message);
}
}
}
protected void Page_Load(object sender, EventArgs e)
{
ReadQueryString();
var tree = new LinkTree(Subject, _subjectType);
tree.Populate(_depth, _maxNodes);
var r = new TreeNode(tree.Root.Name, tree.Root.Type.ToString(), string.Empty, string.Empty, string.Empty);
TreeView1.Nodes.Add(r);
AddNodesRecursive(r, tree.Root);
}
/// <summary>
/// Gets the links from a specific subreddit from Reddit
/// </summary>
/// <param name="target"></param>
/// <param name="parameters"></param>
/// <returns></returns>
public async Task <Tuple <List <Link>, string> > GetHot(Subreddit target, string parameters)
{
List <Link> links = new List <Link>();
string url = (target.id == null) ? "/hot" : target.url + "hot";
url = (parameters.Equals("")) ? url : url + "?" + parameters;
RestClient listings_request = new RestClient(url, state.access_token);
string result = await listings_request.MakeRequest(parameters);
LinkTree link_tree = JsonConvert.DeserializeObject <LinkTree>(result);
foreach (LinkChild link in link_tree.data.children)
{
links.Add(link.data);
}
return(new Tuple <List <Link>, string>(links, link_tree.data.after));
}
protected void Page_Load(object sender, EventArgs e)
{
if (!ReadQueryString())
{
return;
}
var tree = new LinkTree(_subject, _subjectType);
tree.Populate(_depth, _maxNodes);
Response.ClearContent();
Response.ClearHeaders();
switch (_view)
{
case "text":
Response.ContentType = "text/plain";
tree.ToStringTabSepparatedTable(Response.Output);
break;
case "xml":
Response.ContentType = "text/xml";
tree.ToXml(Response.Output);
break;
}
}
