public SiteScraper(string url) //take care of any initializing : this() { //start the link tree try { LinkTree.Add(new LinkNode(url)); } catch (ArgumentException e) { Console.WriteLine(e.Message); } //get the links from the current page HtmlDocument page = GetPage(url); IEnumerable <string> pageLinks = ExtractLinks(page, url); //add these links to the LinkTree foreach (string link in pageLinks) { try { LinkTree.Add(new LinkNode(link, url)); } catch (ArgumentException e) { Console.WriteLine(e.Message); } } }
protected void Page_Load(object sender, EventArgs e) { ReadQueryString(); var tree = new LinkTree(Subject, _subjectType); tree.Populate(_depth, _maxNodes); var r = new TreeNode(tree.Root.Name, tree.Root.Type.ToString(), string.Empty, string.Empty, string.Empty); TreeView1.Nodes.Add(r); AddNodesRecursive(r, tree.Root); }
/// <summary> /// Gets the links from a specific subreddit from Reddit /// </summary> /// <param name="target"></param> /// <param name="parameters"></param> /// <returns></returns> public async Task <Tuple <List <Link>, string> > GetHot(Subreddit target, string parameters) { List <Link> links = new List <Link>(); string url = (target.id == null) ? "/hot" : target.url + "hot"; url = (parameters.Equals("")) ? url : url + "?" + parameters; RestClient listings_request = new RestClient(url, state.access_token); string result = await listings_request.MakeRequest(parameters); LinkTree link_tree = JsonConvert.DeserializeObject <LinkTree>(result); foreach (LinkChild link in link_tree.data.children) { links.Add(link.data); } return(new Tuple <List <Link>, string>(links, link_tree.data.after)); }
protected void Page_Load(object sender, EventArgs e) { if (!ReadQueryString()) { return; } var tree = new LinkTree(_subject, _subjectType); tree.Populate(_depth, _maxNodes); Response.ClearContent(); Response.ClearHeaders(); switch (_view) { case "text": Response.ContentType = "text/plain"; tree.ToStringTabSepparatedTable(Response.Output); break; case "xml": Response.ContentType = "text/xml"; tree.ToXml(Response.Output); break; } }