private string RenderResults(string description, string terms, int page, int pageSize, int total, IEnumerable <ContentSearchResult> results) { using (StringWriter wtr = new StringWriter()) { var summary = new Dictionary <string, Action <TextWriter> >(); summary.Add("search-terms", w => w.Write(HttpRequestUtil.HtmlEncode(description))); try { if (total > 0) { summary.Add("search-result", w => WriteResultHtml(w, terms, page, pageSize, total, results)); } else { summary.Add("search-result", w => WriteErrorHtml(w, "No results found.")); } RenderTemplate(wtr, terms, summary); } catch { wtr.GetStringBuilder().Length = 0; summary.Remove("search-result"); RenderTemplate(wtr, terms, summary); } return(wtr.ToString()); } }
public SaveContent(SiteCollector collector, string path, Action<Action> queue, HttpRequestUtil request) { _collector = collector; _path = path; _queue = queue; _request = request; }
public string RenderResults(string searchTerms, int page) { string errorMessage = ""; if (searchTerms.Length > 128) { errorMessage = "The query is too long."; } else { try { return(RenderResultsInternal(searchTerms, page)); } catch (ParseException) { errorMessage = String.Format("Unable to parse query '{0}', be sure to match quotes and parentheses.", searchTerms); } } var summary = new Dictionary <string, Action <TextWriter> >(); summary.Add("search-terms", w => w.Write(HttpRequestUtil.HtmlEncode(errorMessage))); summary.Add("search-result", w => { }); using (StringWriter sw = new StringWriter()) { RenderTemplate(sw, searchTerms, summary); return(sw.ToString()); } }
public static bool SendPingback(Uri source, Uri target, Uri pingbackUri, Action <string> logError, out int errorCode, out string serverMessage) { HttpRequestUtil http = new HttpRequestUtil(pingbackUri); string xmlrpc = @"<?xml version=""1.0""?><methodCall><methodName>pingback.ping</methodName><params>" + @"<param><value><string>{0}</string></value></param>" + @"<param><value><string>{1}</string></value></param></params></methodCall>"; xmlrpc = String.Format(xmlrpc, HttpUtility.HtmlEncode(source.AbsoluteUri), HttpUtility.HtmlEncode(target.AbsoluteUri)); byte[] postdata = Encoding.UTF8.GetBytes(xmlrpc); if (http.Post(pingbackUri.PathAndQuery, "application/xml", postdata, postdata.Length) != HttpStatusCode.OK) { logError(String.Format("POST {0}: {1}/{2}", pingbackUri, (int)http.StatusCode, http.StatusCode)); errorCode = (int)http.StatusCode; serverMessage = "The server returned an http error " + (int)http.StatusCode; } else { if (!http.ContentType.StartsWith("text/xml", StringComparison.OrdinalIgnoreCase) && !http.ContentType.StartsWith("application/xml", StringComparison.OrdinalIgnoreCase)) { logError("Expected content type: application/xml, found: " + http.ContentType); } try { XmlLightDocument doc = new XmlLightDocument(Encoding.UTF8.GetString(http.Content)); XmlLightElement error = doc.SelectSingleNode("methodResponse/fault"); if (error != null) { XmlLightElement faultCode = error.SelectSingleNode("value/struct/member[name/text()='faultCode']/value/int"); XmlLightElement faultString = error.SelectSingleNode("value/struct/member[name/text()='faultString']/value"); if (faultCode == null || !int.TryParse(faultCode.InnerText, out errorCode)) { errorCode = 0; } serverMessage = faultString != null ? faultString.InnerText : "Unknown Error"; } else { serverMessage = doc.SelectRequiredNode("/methodResponse/params/param/value").InnerText; errorCode = 0; return(true); } } catch (Exception e) { logError("Error parsing response: " + e.Message); serverMessage = "Unable to parse xml response."; errorCode = 0; } } return(false); }
public FetchUrl(SiteCollector collector, string path, string etag, Action<Action> enqueue) { _collector = collector; _path = path; _enqueue = enqueue; _request = new HttpRequestUtil(collector.BaseUri); if (!String.IsNullOrEmpty(etag)) _request.RequestHeaders["If-None-Match"] = etag; }
public void Playback(Uri sourceSite, string stateFile) { Uri pingbackApi; if (!TryGetPingbackFromHeader(out pingbackApi)) { throw new ArgumentException("The site does not have an X-Pingback header."); } if (!Directory.Exists(Path.GetDirectoryName(stateFile))) { Directory.CreateDirectory(Path.GetDirectoryName(stateFile)); } using (Stream stream = new FileStream(stateFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) { stream.Seek(0, SeekOrigin.End); HttpRequestUtil http = new HttpRequestUtil(sourceSite); http.RequestHeaders["If-None-Match"] = String.Format("\"position:{0}\"", stream.Position); if (http.Get(sourceSite.PathAndQuery) == HttpStatusCode.NotModified) { return; } if (http.StatusCode != HttpStatusCode.OK) { throw new ApplicationException(String.Format("Unexpected http result: {0}/{1}", (int)http.StatusCode, http.StatusCode)); } PingbackRecord response = PingbackRecord.ParseFrom(http.Content); foreach (PingbackInfo ping in response.RecordsList) { try { int errorCode; string serverMessage; Uri source = new Uri(ping.SourceUri, UriKind.Absolute); Uri target = new Uri(_targetLink, ping.TargetUri); if (!SendPingback(source, target, pingbackApi, LogError, out errorCode, out serverMessage)) { LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, serverMessage)); } } catch (Exception e) { LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, e.Message)); } } response.WriteTo(stream); } }
public bool ReadRobotsFile(Uri baseUri, string userAgent) { HttpRequestUtil http = new HttpRequestUtil(baseUri); if (http.Get("/robots.txt") != HttpStatusCode.OK) { return(false); } using (TextReader rdr = new StreamReader(new MemoryStream(http.Content), Encoding.UTF8)) { string line; bool matched = false; char[] divide = new[] { ':' }; while (null != (line = rdr.ReadLine())) { if (line.Length == 0 || line[0] == '#') { continue; } string[] parts = line.Split(divide, 2); if (parts.Length != 2) { continue; } parts[0] = parts[0].Trim().ToLower(); switch (parts[0]) { case "user-agent": matched = parts[1].Trim() == "*" || parts[1].Trim() == userAgent; break; case "disallow": if (matched) { Add(parts[1]); } break; } } } return(true); }
public bool TryGetPingbackFromHtml(out Uri pingbackApi) { HttpRequestUtil http = new HttpRequestUtil(_targetLink); if (http.Get(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK) { LogError(String.Format("GET {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode)); } else if (!http.ContentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)) { LogError("Invalid content-type, expected text/html, found: " + http.ContentType); } else { try { HtmlLightDocument htmlDoc = new HtmlLightDocument(Encoding.UTF8.GetString(http.Content)); XmlLightElement link = htmlDoc.SelectSingleNode("/html/head/link[@rel='pingback']"); if (link == null) { LogError("Unable to locate <link rel=\"pingback\" ... in header."); } else { string pingback; if (!link.Attributes.TryGetValue("href", out pingback)) { LogError("Link for rel=pingback is missing the href attribute."); } else { LogInfo("Found rel=pingback: " + pingback); return(Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi)); } } } catch (Exception e) { LogError(e.Message); } } pingbackApi = null; return(false); }
public bool TryGetPingbackFromHeader(out Uri pingbackApi) { HttpRequestUtil http = new HttpRequestUtil(_targetLink); if (http.Head(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK) LogError(String.Format("HEAD {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode)); else { string pingback = http.ResponseHeaders["X-PINGBACK"]; if (String.IsNullOrEmpty(pingback)) LogError("X-Pingback header not found."); else { LogInfo("Found X-Pingback: " + pingback); return Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi); } } pingbackApi = null; return false; }
public bool ReadRobotsFile(Uri baseUri, string userAgent) { HttpRequestUtil http = new HttpRequestUtil(baseUri); if (http.Get("/robots.txt") != HttpStatusCode.OK) return false; using(TextReader rdr = new StreamReader(new MemoryStream(http.Content), Encoding.UTF8)) { string line; bool matched = false; char[] divide = new[] {':'}; while(null != (line = rdr.ReadLine())) { if (line.Length == 0 || line[0] == '#') continue; string[] parts = line.Split(divide, 2); if(parts.Length != 2) continue; parts[0] = parts[0].Trim().ToLower(); switch(parts[0]) { case "user-agent": matched = parts[1].Trim() == "*" || parts[1].Trim() == userAgent; break; case "disallow": if(matched) Add(parts[1]); break; } } } return true; }
public bool TryGetPingbackFromHeader(out Uri pingbackApi) { HttpRequestUtil http = new HttpRequestUtil(_targetLink); if (http.Head(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK) { LogError(String.Format("HEAD {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode)); } else { string pingback = http.ResponseHeaders["X-PINGBACK"]; if (String.IsNullOrEmpty(pingback)) { LogError("X-Pingback header not found."); } else { LogInfo("Found X-Pingback: " + pingback); return(Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi)); } } pingbackApi = null; return(false); }
private void RenderTemplate(TextWriter wtr, string terms, IDictionary <string, Action <TextWriter> > values) { string[] parts = GetTemplate(); if (((parts.Length - 1) % 4) != 0) { throw new ApplicationException("Invalid template format."); } if (_valuePartIndex >= 0 && _valuePartOffset >= 0) { parts = (string[])parts.Clone(); parts[_valuePartIndex] = parts[_valuePartIndex].Insert(_valuePartOffset, HttpRequestUtil.HtmlEncode(terms)); } wtr.Write(parts[0]); for (int ix = 1; ix < parts.Length; ix += 4) { if (parts[ix + 2] != '/' + parts[ix]) { throw new ApplicationException("Invalid template format."); } Action <TextWriter> fn; if (values.TryGetValue(parts[ix], out fn)) { fn(wtr); } else { wtr.Write(parts[ix + 1]); } wtr.Write(parts[ix + 3]); } }
public void Publish(Uri destination) { if(_rsaKeyPair == null) throw new FileNotFoundException("You must have a client publication key, see the command 'createkeys'", _keyfile); Console.WriteLine("Creating site archive..."); string file = CreateArchiveFile(); Console.WriteLine("Connecting to server..."); SecureTransfer.Client client = new SecureTransfer.Client( _rsaKeyPair.ClientPrivateKey, _rsaKeyPair.ServerPublicKey, (transferid, location, request) => { byte[] body = IOStream.ReadAllBytes(request); HttpRequestUtil http = new HttpRequestUtil(destination); if (http.Post(_publishUri, "application/binary", body, body.Length) != HttpStatusCode.OK) throw new WebException(String.Format( "The server returned an invalid response: {0}/{1}", (int) http.StatusCode, http.StatusCode)); return new MemoryStream(http.Content); } ); client.ProgressChanged += ProgressChanged; try { client.Upload(Path.GetFileName(file), file); } finally { Console.WriteLine(); } Console.WriteLine("Complete."); }
public bool TryGetPingbackFromHtml(out Uri pingbackApi) { HttpRequestUtil http = new HttpRequestUtil(_targetLink); if (http.Get(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK) LogError(String.Format("GET {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode)); else if (!http.ContentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)) LogError("Invalid content-type, expected text/html, found: " + http.ContentType); else { try { HtmlLightDocument htmlDoc = new HtmlLightDocument(Encoding.UTF8.GetString(http.Content)); XmlLightElement link = htmlDoc.SelectSingleNode("/html/head/link[@rel='pingback']"); if (link == null) LogError("Unable to locate <link rel=\"pingback\" ... in header."); else { string pingback; if (!link.Attributes.TryGetValue("href", out pingback)) LogError("Link for rel=pingback is missing the href attribute."); else { LogInfo("Found rel=pingback: " + pingback); return Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi); } } } catch (Exception e) { LogError(e.Message); } } pingbackApi = null; return false; }
public void Playback(Uri sourceSite, string stateFile) { Uri pingbackApi; if(!TryGetPingbackFromHeader(out pingbackApi)) throw new ArgumentException("The site does not have an X-Pingback header."); if (!Directory.Exists(Path.GetDirectoryName(stateFile))) Directory.CreateDirectory(Path.GetDirectoryName(stateFile)); using(Stream stream = new FileStream(stateFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) { stream.Seek(0, SeekOrigin.End); HttpRequestUtil http = new HttpRequestUtil(sourceSite); http.RequestHeaders["If-None-Match"] = String.Format("\"position:{0}\"", stream.Position); if (http.Get(sourceSite.PathAndQuery) == HttpStatusCode.NotModified) return; if (http.StatusCode != HttpStatusCode.OK) throw new ApplicationException(String.Format("Unexpected http result: {0}/{1}", (int)http.StatusCode, http.StatusCode)); PingbackRecord response = PingbackRecord.ParseFrom(http.Content); foreach (PingbackInfo ping in response.RecordsList) { try { int errorCode; string serverMessage; Uri source = new Uri(ping.SourceUri, UriKind.Absolute); Uri target = new Uri(_targetLink, ping.TargetUri); if (!SendPingback(source, target, pingbackApi, LogError, out errorCode, out serverMessage)) LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, serverMessage)); } catch (Exception e) { LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, e.Message)); } } response.WriteTo(stream); } }
public static bool SendPingback(Uri source, Uri target, Uri pingbackUri, Action<string> logError, out int errorCode, out string serverMessage) { HttpRequestUtil http = new HttpRequestUtil(pingbackUri); string xmlrpc = @"<?xml version=""1.0""?><methodCall><methodName>pingback.ping</methodName><params>" + @"<param><value><string>{0}</string></value></param>" + @"<param><value><string>{1}</string></value></param></params></methodCall>"; xmlrpc = String.Format(xmlrpc, HttpUtility.HtmlEncode(source.AbsoluteUri), HttpUtility.HtmlEncode(target.AbsoluteUri)); byte[] postdata = Encoding.UTF8.GetBytes(xmlrpc); if (http.Post(pingbackUri.PathAndQuery, "application/xml", postdata, postdata.Length) != HttpStatusCode.OK) { logError(String.Format("POST {0}: {1}/{2}", pingbackUri, (int)http.StatusCode, http.StatusCode)); errorCode = (int)http.StatusCode; serverMessage = "The server returned an http error " + (int)http.StatusCode; } else { if (!http.ContentType.StartsWith("text/xml", StringComparison.OrdinalIgnoreCase) && !http.ContentType.StartsWith("application/xml", StringComparison.OrdinalIgnoreCase)) { logError("Expected content type: application/xml, found: " + http.ContentType); } try { XmlLightDocument doc = new XmlLightDocument(Encoding.UTF8.GetString(http.Content)); XmlLightElement error = doc.SelectSingleNode("methodResponse/fault"); if (error != null) { XmlLightElement faultCode = error.SelectSingleNode("value/struct/member[name/text()='faultCode']/value/int"); XmlLightElement faultString = error.SelectSingleNode("value/struct/member[name/text()='faultString']/value"); if (faultCode == null || !int.TryParse(faultCode.InnerText, out errorCode)) errorCode = 0; serverMessage = faultString != null ? faultString.InnerText : "Unknown Error"; } else { serverMessage = doc.SelectRequiredNode("/methodResponse/params/param/value").InnerText; errorCode = 0; return true; } } catch (Exception e) { logError("Error parsing response: " + e.Message); serverMessage = "Unable to parse xml response."; errorCode = 0; } } return false; }