private string RenderResults(string description, string terms, int page, int pageSize, int total, IEnumerable <ContentSearchResult> results)
{
using (StringWriter wtr = new StringWriter())
{
var summary = new Dictionary <string, Action <TextWriter> >();
summary.Add("search-terms", w => w.Write(HttpRequestUtil.HtmlEncode(description)));
try
{
if (total > 0)
{
summary.Add("search-result", w => WriteResultHtml(w, terms, page, pageSize, total, results));
}
else
{
summary.Add("search-result", w => WriteErrorHtml(w, "No results found."));
}
RenderTemplate(wtr, terms, summary);
}
catch
{
wtr.GetStringBuilder().Length = 0;
summary.Remove("search-result");
RenderTemplate(wtr, terms, summary);
}
return(wtr.ToString());
}
}
public SaveContent(SiteCollector collector, string path, Action<Action> queue, HttpRequestUtil request)
{
_collector = collector;
_path = path;
_queue = queue;
_request = request;
}
public string RenderResults(string searchTerms, int page)
{
string errorMessage = "";
if (searchTerms.Length > 128)
{
errorMessage = "The query is too long.";
}
else
{
try
{
return(RenderResultsInternal(searchTerms, page));
}
catch (ParseException)
{
errorMessage = String.Format("Unable to parse query '{0}', be sure to match quotes and parentheses.", searchTerms);
}
}
var summary = new Dictionary <string, Action <TextWriter> >();
summary.Add("search-terms", w => w.Write(HttpRequestUtil.HtmlEncode(errorMessage)));
summary.Add("search-result", w => { });
using (StringWriter sw = new StringWriter())
{
RenderTemplate(sw, searchTerms, summary);
return(sw.ToString());
}
}
public static bool SendPingback(Uri source, Uri target, Uri pingbackUri, Action <string> logError, out int errorCode, out string serverMessage)
{
HttpRequestUtil http = new HttpRequestUtil(pingbackUri);
string xmlrpc = @"<?xml version=""1.0""?><methodCall><methodName>pingback.ping</methodName><params>" +
@"<param><value><string>{0}</string></value></param>" +
@"<param><value><string>{1}</string></value></param></params></methodCall>";
xmlrpc = String.Format(xmlrpc, HttpUtility.HtmlEncode(source.AbsoluteUri), HttpUtility.HtmlEncode(target.AbsoluteUri));
byte[] postdata = Encoding.UTF8.GetBytes(xmlrpc);
if (http.Post(pingbackUri.PathAndQuery, "application/xml", postdata, postdata.Length) != HttpStatusCode.OK)
{
logError(String.Format("POST {0}: {1}/{2}", pingbackUri, (int)http.StatusCode, http.StatusCode));
errorCode = (int)http.StatusCode;
serverMessage = "The server returned an http error " + (int)http.StatusCode;
}
else
{
if (!http.ContentType.StartsWith("text/xml", StringComparison.OrdinalIgnoreCase) &&
!http.ContentType.StartsWith("application/xml", StringComparison.OrdinalIgnoreCase))
{
logError("Expected content type: application/xml, found: " + http.ContentType);
}
try
{
XmlLightDocument doc = new XmlLightDocument(Encoding.UTF8.GetString(http.Content));
XmlLightElement error = doc.SelectSingleNode("methodResponse/fault");
if (error != null)
{
XmlLightElement faultCode = error.SelectSingleNode("value/struct/member[name/text()='faultCode']/value/int");
XmlLightElement faultString = error.SelectSingleNode("value/struct/member[name/text()='faultString']/value");
if (faultCode == null || !int.TryParse(faultCode.InnerText, out errorCode))
{
errorCode = 0;
}
serverMessage = faultString != null ? faultString.InnerText : "Unknown Error";
}
else
{
serverMessage = doc.SelectRequiredNode("/methodResponse/params/param/value").InnerText;
errorCode = 0;
return(true);
}
}
catch (Exception e)
{
logError("Error parsing response: " + e.Message);
serverMessage = "Unable to parse xml response.";
errorCode = 0;
}
}
return(false);
}
public FetchUrl(SiteCollector collector, string path, string etag, Action<Action> enqueue)
{
_collector = collector;
_path = path;
_enqueue = enqueue;
_request = new HttpRequestUtil(collector.BaseUri);
if (!String.IsNullOrEmpty(etag))
_request.RequestHeaders["If-None-Match"] = etag;
}
public void Playback(Uri sourceSite, string stateFile)
{
Uri pingbackApi;
if (!TryGetPingbackFromHeader(out pingbackApi))
{
throw new ArgumentException("The site does not have an X-Pingback header.");
}
if (!Directory.Exists(Path.GetDirectoryName(stateFile)))
{
Directory.CreateDirectory(Path.GetDirectoryName(stateFile));
}
using (Stream stream = new FileStream(stateFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
stream.Seek(0, SeekOrigin.End);
HttpRequestUtil http = new HttpRequestUtil(sourceSite);
http.RequestHeaders["If-None-Match"] = String.Format("\"position:{0}\"", stream.Position);
if (http.Get(sourceSite.PathAndQuery) == HttpStatusCode.NotModified)
{
return;
}
if (http.StatusCode != HttpStatusCode.OK)
{
throw new ApplicationException(String.Format("Unexpected http result: {0}/{1}", (int)http.StatusCode, http.StatusCode));
}
PingbackRecord response = PingbackRecord.ParseFrom(http.Content);
foreach (PingbackInfo ping in response.RecordsList)
{
try
{
int errorCode;
string serverMessage;
Uri source = new Uri(ping.SourceUri, UriKind.Absolute);
Uri target = new Uri(_targetLink, ping.TargetUri);
if (!SendPingback(source, target, pingbackApi, LogError, out errorCode, out serverMessage))
{
LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, serverMessage));
}
}
catch (Exception e)
{
LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, e.Message));
}
}
response.WriteTo(stream);
}
}
public bool ReadRobotsFile(Uri baseUri, string userAgent)
{
HttpRequestUtil http = new HttpRequestUtil(baseUri);
if (http.Get("/robots.txt") != HttpStatusCode.OK)
{
return(false);
}
using (TextReader rdr = new StreamReader(new MemoryStream(http.Content), Encoding.UTF8))
{
string line;
bool matched = false;
char[] divide = new[] { ':' };
while (null != (line = rdr.ReadLine()))
{
if (line.Length == 0 || line[0] == '#')
{
continue;
}
string[] parts = line.Split(divide, 2);
if (parts.Length != 2)
{
continue;
}
parts[0] = parts[0].Trim().ToLower();
switch (parts[0])
{
case "user-agent":
matched = parts[1].Trim() == "*" || parts[1].Trim() == userAgent;
break;
case "disallow":
if (matched)
{
Add(parts[1]);
}
break;
}
}
}
return(true);
}
public bool TryGetPingbackFromHtml(out Uri pingbackApi)
{
HttpRequestUtil http = new HttpRequestUtil(_targetLink);
if (http.Get(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK)
{
LogError(String.Format("GET {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode));
}
else if (!http.ContentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase))
{
LogError("Invalid content-type, expected text/html, found: " + http.ContentType);
}
else
{
try
{
HtmlLightDocument htmlDoc = new HtmlLightDocument(Encoding.UTF8.GetString(http.Content));
XmlLightElement link = htmlDoc.SelectSingleNode("/html/head/link[@rel='pingback']");
if (link == null)
{
LogError("Unable to locate <link rel=\"pingback\" ... in header.");
}
else
{
string pingback;
if (!link.Attributes.TryGetValue("href", out pingback))
{
LogError("Link for rel=pingback is missing the href attribute.");
}
else
{
LogInfo("Found rel=pingback: " + pingback);
return(Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi));
}
}
}
catch (Exception e)
{
LogError(e.Message);
}
}
pingbackApi = null;
return(false);
}
public bool TryGetPingbackFromHeader(out Uri pingbackApi)
{
HttpRequestUtil http = new HttpRequestUtil(_targetLink);
if (http.Head(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK)
LogError(String.Format("HEAD {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode));
else
{
string pingback = http.ResponseHeaders["X-PINGBACK"];
if (String.IsNullOrEmpty(pingback))
LogError("X-Pingback header not found.");
else
{
LogInfo("Found X-Pingback: " + pingback);
return Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi);
}
}
pingbackApi = null;
return false;
}
public bool ReadRobotsFile(Uri baseUri, string userAgent)
{
HttpRequestUtil http = new HttpRequestUtil(baseUri);
if (http.Get("/robots.txt") != HttpStatusCode.OK)
return false;
using(TextReader rdr = new StreamReader(new MemoryStream(http.Content), Encoding.UTF8))
{
string line;
bool matched = false;
char[] divide = new[] {':'};
while(null != (line = rdr.ReadLine()))
{
if (line.Length == 0 || line[0] == '#')
continue;
string[] parts = line.Split(divide, 2);
if(parts.Length != 2)
continue;
parts[0] = parts[0].Trim().ToLower();
switch(parts[0])
{
case "user-agent":
matched = parts[1].Trim() == "*" || parts[1].Trim() == userAgent;
break;
case "disallow":
if(matched)
Add(parts[1]);
break;
}
}
}
return true;
}
public bool TryGetPingbackFromHeader(out Uri pingbackApi)
{
HttpRequestUtil http = new HttpRequestUtil(_targetLink);
if (http.Head(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK)
{
LogError(String.Format("HEAD {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode));
}
else
{
string pingback = http.ResponseHeaders["X-PINGBACK"];
if (String.IsNullOrEmpty(pingback))
{
LogError("X-Pingback header not found.");
}
else
{
LogInfo("Found X-Pingback: " + pingback);
return(Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi));
}
}
pingbackApi = null;
return(false);
}
private void RenderTemplate(TextWriter wtr, string terms, IDictionary <string, Action <TextWriter> > values)
{
string[] parts = GetTemplate();
if (((parts.Length - 1) % 4) != 0)
{
throw new ApplicationException("Invalid template format.");
}
if (_valuePartIndex >= 0 && _valuePartOffset >= 0)
{
parts = (string[])parts.Clone();
parts[_valuePartIndex] = parts[_valuePartIndex].Insert(_valuePartOffset, HttpRequestUtil.HtmlEncode(terms));
}
wtr.Write(parts[0]);
for (int ix = 1; ix < parts.Length; ix += 4)
{
if (parts[ix + 2] != '/' + parts[ix])
{
throw new ApplicationException("Invalid template format.");
}
Action <TextWriter> fn;
if (values.TryGetValue(parts[ix], out fn))
{
fn(wtr);
}
else
{
wtr.Write(parts[ix + 1]);
}
wtr.Write(parts[ix + 3]);
}
}
public void Publish(Uri destination)
{
if(_rsaKeyPair == null)
throw new FileNotFoundException("You must have a client publication key, see the command 'createkeys'", _keyfile);
Console.WriteLine("Creating site archive...");
string file = CreateArchiveFile();
Console.WriteLine("Connecting to server...");
SecureTransfer.Client client =
new SecureTransfer.Client(
_rsaKeyPair.ClientPrivateKey,
_rsaKeyPair.ServerPublicKey,
(transferid, location, request) =>
{
byte[] body = IOStream.ReadAllBytes(request);
HttpRequestUtil http = new HttpRequestUtil(destination);
if (http.Post(_publishUri, "application/binary", body, body.Length) != HttpStatusCode.OK)
throw new WebException(String.Format(
"The server returned an invalid response: {0}/{1}",
(int) http.StatusCode,
http.StatusCode));
return new MemoryStream(http.Content);
}
);
client.ProgressChanged += ProgressChanged;
try { client.Upload(Path.GetFileName(file), file); }
finally { Console.WriteLine(); }
Console.WriteLine("Complete.");
}
public bool TryGetPingbackFromHtml(out Uri pingbackApi)
{
HttpRequestUtil http = new HttpRequestUtil(_targetLink);
if (http.Get(_targetLink.PathAndQuery) != System.Net.HttpStatusCode.OK)
LogError(String.Format("GET {0}: {1}/{2}", _targetLink, (int)http.StatusCode, http.StatusCode));
else if (!http.ContentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase))
LogError("Invalid content-type, expected text/html, found: " + http.ContentType);
else
{
try
{
HtmlLightDocument htmlDoc = new HtmlLightDocument(Encoding.UTF8.GetString(http.Content));
XmlLightElement link = htmlDoc.SelectSingleNode("/html/head/link[@rel='pingback']");
if (link == null)
LogError("Unable to locate <link rel=\"pingback\" ... in header.");
else
{
string pingback;
if (!link.Attributes.TryGetValue("href", out pingback))
LogError("Link for rel=pingback is missing the href attribute.");
else
{
LogInfo("Found rel=pingback: " + pingback);
return Uri.TryCreate(pingback, UriKind.Absolute, out pingbackApi);
}
}
}
catch (Exception e)
{
LogError(e.Message);
}
}
pingbackApi = null;
return false;
}
public void Playback(Uri sourceSite, string stateFile)
{
Uri pingbackApi;
if(!TryGetPingbackFromHeader(out pingbackApi))
throw new ArgumentException("The site does not have an X-Pingback header.");
if (!Directory.Exists(Path.GetDirectoryName(stateFile)))
Directory.CreateDirectory(Path.GetDirectoryName(stateFile));
using(Stream stream = new FileStream(stateFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
stream.Seek(0, SeekOrigin.End);
HttpRequestUtil http = new HttpRequestUtil(sourceSite);
http.RequestHeaders["If-None-Match"] = String.Format("\"position:{0}\"", stream.Position);
if (http.Get(sourceSite.PathAndQuery) == HttpStatusCode.NotModified)
return;
if (http.StatusCode != HttpStatusCode.OK)
throw new ApplicationException(String.Format("Unexpected http result: {0}/{1}", (int)http.StatusCode, http.StatusCode));
PingbackRecord response = PingbackRecord.ParseFrom(http.Content);
foreach (PingbackInfo ping in response.RecordsList)
{
try
{
int errorCode;
string serverMessage;
Uri source = new Uri(ping.SourceUri, UriKind.Absolute);
Uri target = new Uri(_targetLink, ping.TargetUri);
if (!SendPingback(source, target, pingbackApi, LogError, out errorCode, out serverMessage))
LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, serverMessage));
}
catch (Exception e)
{
LogError(String.Format("Failed to register pingback from {0}, {1}", ping.SourceUri, e.Message));
}
}
response.WriteTo(stream);
}
}
public static bool SendPingback(Uri source, Uri target, Uri pingbackUri, Action<string> logError, out int errorCode, out string serverMessage)
{
HttpRequestUtil http = new HttpRequestUtil(pingbackUri);
string xmlrpc = @"<?xml version=""1.0""?><methodCall><methodName>pingback.ping</methodName><params>" +
@"<param><value><string>{0}</string></value></param>" +
@"<param><value><string>{1}</string></value></param></params></methodCall>";
xmlrpc = String.Format(xmlrpc, HttpUtility.HtmlEncode(source.AbsoluteUri), HttpUtility.HtmlEncode(target.AbsoluteUri));
byte[] postdata = Encoding.UTF8.GetBytes(xmlrpc);
if (http.Post(pingbackUri.PathAndQuery, "application/xml", postdata, postdata.Length) != HttpStatusCode.OK)
{
logError(String.Format("POST {0}: {1}/{2}", pingbackUri, (int)http.StatusCode, http.StatusCode));
errorCode = (int)http.StatusCode;
serverMessage = "The server returned an http error " + (int)http.StatusCode;
}
else
{
if (!http.ContentType.StartsWith("text/xml", StringComparison.OrdinalIgnoreCase) &&
!http.ContentType.StartsWith("application/xml", StringComparison.OrdinalIgnoreCase))
{
logError("Expected content type: application/xml, found: " + http.ContentType);
}
try
{
XmlLightDocument doc = new XmlLightDocument(Encoding.UTF8.GetString(http.Content));
XmlLightElement error = doc.SelectSingleNode("methodResponse/fault");
if (error != null)
{
XmlLightElement faultCode = error.SelectSingleNode("value/struct/member[name/text()='faultCode']/value/int");
XmlLightElement faultString = error.SelectSingleNode("value/struct/member[name/text()='faultString']/value");
if (faultCode == null || !int.TryParse(faultCode.InnerText, out errorCode))
errorCode = 0;
serverMessage = faultString != null ? faultString.InnerText : "Unknown Error";
}
else
{
serverMessage = doc.SelectRequiredNode("/methodResponse/params/param/value").InnerText;
errorCode = 0;
return true;
}
}
catch (Exception e)
{
logError("Error parsing response: " + e.Message);
serverMessage = "Unable to parse xml response.";
errorCode = 0;
}
}
return false;
}
