public void DefaultAspxLinksTest() { HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create("http://localhost/DBlog/Default.aspx"); request.Method = "GET"; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Stream s = response.GetResponseStream(); StreamReader sr = new StreamReader(s); Assert.IsTrue(response.StatusCode == HttpStatusCode.OK, string.Format("Response code was {0}", response.StatusCode)); // look for link rel="service" type="application/atomsvc+xml" List <HtmlUri> uris = HtmlUriExtractor.Extract(sr.ReadToEnd()); HtmlUri linkRelAtomSvc = null; foreach (HtmlUri uri in uris) { if (uri.Control.TagName == "link") { string rel = uri.Control.Attributes["rel"]; string type = uri.Control.Attributes["type"]; if (rel == "service" && type == "application/atomsvc+xml") { linkRelAtomSvc = uri; break; } } } Assert.IsNotNull(linkRelAtomSvc); Console.WriteLine(linkRelAtomSvc.Uri); }
public void TestPage(Uri root, Uri uri, Cookie cookie, out List <Uri> links, out double ts) { DateTime start = DateTime.UtcNow; Console.Write("{0} (from {1}) ...", uri.PathAndQuery, root.PathAndQuery); CookieContainer cookies = null; if (cookie != null) { cookies = new CookieContainer(); cookies.Add(cookie); } HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(uri); request.Headers.Add("SnCore.Web.UserAgent", "SnCore/1.0"); if (cookies != null) { request.CookieContainer = cookies; } request.Method = "GET"; request.AllowAutoRedirect = true; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Stream s = response.GetResponseStream(); StreamReader sr = new StreamReader(s); string data = sr.ReadToEnd(); Assert.IsTrue(response.StatusCode == HttpStatusCode.OK, string.Format("Response code was {0}", response.StatusCode)); Assert.IsFalse(string.IsNullOrEmpty(data)); ts = DateTime.UtcNow.Subtract(start).TotalSeconds; Console.Write("[{0} bytes][{1} sec.]", data.Length, ts.ToString("0.00")); Assert.IsTrue(data.Length > 0, "Downloaded file size is zero."); if (response.ContentType != "text/html; charset=utf-8") { Console.WriteLine("[ignore: {0}]", response.ContentType); links = new List <Uri>(); return; } int error_start = data.IndexOf("<div id=\"ctl00_noticeMenu_panelNotice\" class=\"sncore_notice_error\""); if (error_start > 0) { int error_end = data.IndexOf("</div>", error_start); string error = data.Substring(error_start, error_end - error_start); Assert.IsTrue(error_start < 0, string.Format("{0}: error: {1}", uri, error)); } links = HtmlUriExtractor.Extract(data, root); // Console.WriteLine("\tExtracted {0} links.", links.Count); Console.WriteLine("[done]"); }
protected List <TransitAccountEvent> discoverRel(string url) { List <TransitAccountEvent> result = new List <TransitAccountEvent>(); HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url); request.UserAgent = SessionManager.GetCachedConfiguration("SnCore.Web.UserAgent", "SnCore/1.0"); request.Accept = "*/*"; WebResponse response = request.GetResponse(); string content; using (StreamReader sr = new StreamReader(response.GetResponseStream())) { content = sr.ReadToEnd(); sr.Close(); } List <HtmlLinkControl> links = HtmlLinkExtractor.Extract(content, new Uri(url)); foreach (HtmlLinkControl link in links) { try { switch (link.Type.ToLower()) { case "text/calendar": TransitAccountEventICALEmitter emitter = TransitAccountEventICALEmitter.Parse( link.Href, SessionManager.GetCachedConfiguration("SnCore.Web.UserAgent", "SnCore/1.0")); emitter.AccountEvent.Website = link.Href; AddUnique(result, emitter.AccountEvent); break; } } catch { } } List <Uri> webcallinks = HtmlUriExtractor.Extract(content, new Uri(url)); foreach (Uri link in webcallinks) { try { if (link.Scheme.ToLower() == "webcal" || link.GetLeftPart(UriPartial.Path).EndsWith(".ics")) { TransitAccountEventICALEmitter emitter = TransitAccountEventICALEmitter.Parse( link.ToString(), SessionManager.GetCachedConfiguration("SnCore.Web.UserAgent", "SnCore/1.0")); emitter.AccountEvent.Website = link.ToString(); AddUnique(result, emitter.AccountEvent); } } catch { } } return(result); }