public void Playback(Uri sourceSite, string stateFile) { Uri pingbackApi; if (!TryGetPingbackFromHeader(out pingbackApi)) { throw new ArgumentException("The site does not have an X-Pingback header."); } if (!Directory.Exists(Path.GetDirectoryName(stateFile))) { Directory.CreateDirectory(Path.GetDirectoryName(stateFile)); } using (Stream stream = new FileStream(stateFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) { stream.Seek(0, SeekOrigin.End); HttpRequestUtil http = new HttpRequestUtil(sourceSite); http.RequestHeaders["If-None-Match"] = String.Format("\"position:{0}\"", stream.Position); if (http.Get(sourceSite.PathAndQuery) == HttpStatusCode.NotModified) { return; } if (http.StatusCode != HttpStatusCode.OK) { throw new ApplicationException(String.Format("Unexpected http result: {0}/{1}", (int)http.StatusCode, http.StatusCode)); } PingbackRecord response = PingbackRecord.ParseFrom(http.Content); foreach (PingbackInfo ping in response.RecordsList) { try { int errorCode; string serverMessage; Uri source = new Uri(ping.SourceUri, UriKind.Absolute); Uri target = new Uri(_targetLink, ping.TargetUri); if (!SendPingback(source, target, pingbackApi, LogError, out errorCode, out serverMessage)) { LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, serverMessage)); } } catch (Exception e) { LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, e.Message)); } } response.WriteTo(stream); } }
public bool ReadRobotsFile(Uri baseUri, string userAgent) { HttpRequestUtil http = new HttpRequestUtil(baseUri); if (http.Get("/robots.txt") != HttpStatusCode.OK) { return(false); } using (TextReader rdr = new StreamReader(new MemoryStream(http.Content), Encoding.UTF8)) { string line; bool matched = false; char[] divide = new[] { ':' }; while (null != (line = rdr.ReadLine())) { if (line.Length == 0 || line[0] == '#') { continue; } string[] parts = line.Split(divide, 2); if (parts.Length != 2) { continue; } parts[0] = parts[0].Trim().ToLower(); switch (parts[0]) { case "user-agent": matched = parts[1].Trim() == "*" || parts[1].Trim() == userAgent; break; case "disallow": if (matched) { Add(parts[1]); } break; } } } return(true); }
public bool TryGetPingbackFromHtml(out Uri pingbackApi) { HttpRequestUtil http = new HttpRequestUtil(_targetLink); if (http.Get(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK) { LogError(String.Format("GET {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode)); } else if (!http.ContentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)) { LogError("Invalid content-type, expected text/html, found: " + http.ContentType); } else { try { HtmlLightDocument htmlDoc = new HtmlLightDocument(Encoding.UTF8.GetString(http.Content)); XmlLightElement link = htmlDoc.SelectSingleNode("/html/head/link[@rel='pingback']"); if (link == null) { LogError("Unable to locate <link rel=\"pingback\" ... in header."); } else { string pingback; if (!link.Attributes.TryGetValue("href", out pingback)) { LogError("Link for rel=pingback is missing the href attribute."); } else { LogInfo("Found rel=pingback: " + pingback); return(Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi)); } } } catch (Exception e) { LogError(e.Message); } } pingbackApi = null; return(false); }
public bool ReadRobotsFile(Uri baseUri, string userAgent) { HttpRequestUtil http = new HttpRequestUtil(baseUri); if (http.Get("/robots.txt") != HttpStatusCode.OK) return false; using(TextReader rdr = new StreamReader(new MemoryStream(http.Content), Encoding.UTF8)) { string line; bool matched = false; char[] divide = new[] {':'}; while(null != (line = rdr.ReadLine())) { if (line.Length == 0 || line[0] == '#') continue; string[] parts = line.Split(divide, 2); if(parts.Length != 2) continue; parts[0] = parts[0].Trim().ToLower(); switch(parts[0]) { case "user-agent": matched = parts[1].Trim() == "*" || parts[1].Trim() == userAgent; break; case "disallow": if(matched) Add(parts[1]); break; } } } return true; }
public bool TryGetPingbackFromHtml(out Uri pingbackApi) { HttpRequestUtil http = new HttpRequestUtil(_targetLink); if (http.Get(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK) LogError(String.Format("GET {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode)); else if (!http.ContentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)) LogError("Invalid content-type, expected text/html, found: " + http.ContentType); else { try { HtmlLightDocument htmlDoc = new HtmlLightDocument(Encoding.UTF8.GetString(http.Content)); XmlLightElement link = htmlDoc.SelectSingleNode("/html/head/link[@rel='pingback']"); if (link == null) LogError("Unable to locate <link rel=\"pingback\" ... in header."); else { string pingback; if (!link.Attributes.TryGetValue("href", out pingback)) LogError("Link for rel=pingback is missing the href attribute."); else { LogInfo("Found rel=pingback: " + pingback); return Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi); } } } catch (Exception e) { LogError(e.Message); } } pingbackApi = null; return false; }
public void Playback(Uri sourceSite, string stateFile) { Uri pingbackApi; if(!TryGetPingbackFromHeader(out pingbackApi)) throw new ArgumentException("The site does not have an X-Pingback header."); if (!Directory.Exists(Path.GetDirectoryName(stateFile))) Directory.CreateDirectory(Path.GetDirectoryName(stateFile)); using(Stream stream = new FileStream(stateFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) { stream.Seek(0, SeekOrigin.End); HttpRequestUtil http = new HttpRequestUtil(sourceSite); http.RequestHeaders["If-None-Match"] = String.Format("\"position:{0}\"", stream.Position); if (http.Get(sourceSite.PathAndQuery) == HttpStatusCode.NotModified) return; if (http.StatusCode != HttpStatusCode.OK) throw new ApplicationException(String.Format("Unexpected http result: {0}/{1}", (int)http.StatusCode, http.StatusCode)); PingbackRecord response = PingbackRecord.ParseFrom(http.Content); foreach (PingbackInfo ping in response.RecordsList) { try { int errorCode; string serverMessage; Uri source = new Uri(ping.SourceUri, UriKind.Absolute); Uri target = new Uri(_targetLink, ping.TargetUri); if (!SendPingback(source, target, pingbackApi, LogError, out errorCode, out serverMessage)) LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, serverMessage)); } catch (Exception e) { LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, e.Message)); } } response.WriteTo(stream); } }