public override Page getPage(PageRequestMessage pageRequest)
{
Logger.log($"Getting {pageRequest.title} page from disk");
FileInfo file = path.GetFiles().FirstOrDefault(f => f.Name.Equals($"{pageRequest.title}.json", StringComparison.InvariantCultureIgnoreCase));
Page page = null;
if (file != null)
{
string fileName = $"{pageRequest.title.ToLower()}.json";
string json = File.ReadAllText(Path.Combine(path.FullName, fileName));
page = JsonConvert.DeserializeObject <Page>(json);
}
if (file == null)
{
Logger.log($"{pageRequest.title} page not on disk");
page = backup?.getPage(pageRequest);
if (page != null)
{
savePage(page);
}
}
return(page);
}
public override Page getPage(PageRequestMessage pageRequest)
{
Logger.log($"Getting {pageRequest.title} page from source");
Dictionary <string, string> values = new Dictionary <string, string>
{
{ "pages", pageRequest.title },
{ "curonly", "1" }
};
FormUrlEncodedContent content = new FormUrlEncodedContent(values);
HttpResponseMessage response = client.PostAsync("https://en.wikipedia.org/wiki/Special:Export", content).Result;
string responseString = response.Content.ReadAsStringAsync().Result;
XmlDocument doc = new XmlDocument();
doc.LoadXml(responseString);
XmlNamespaceManager nsManager = new XmlNamespaceManager(doc.NameTable);
nsManager.AddNamespace("wiki", "http://www.mediawiki.org/xml/export-0.10/");
XmlNode pageXML = doc.DocumentElement.SelectSingleNode("wiki:page", nsManager);
string pageTitle = pageXML.SelectSingleNode("wiki:title", nsManager).InnerText;
int id = int.Parse(pageXML.SelectSingleNode("wiki:id", nsManager).InnerText);
XmlNode revision = pageXML.SelectSingleNode("wiki:revision", nsManager);
string text = revision.SelectSingleNode("wiki:text", nsManager).InnerText;
string sha1 = revision.SelectSingleNode("wiki:sha1", nsManager).InnerText;
DateTime timestamp = DateTime.Parse(revision.SelectSingleNode("wiki:timestamp", nsManager).InnerText);
return(new Page()
{
id = id,
title = pageTitle,
text = text,
timestamp = timestamp,
path = $"wiki/{pageTitle}",
sha1 = sha1
});
}
public override Page getPage(PageRequestMessage pageRequest)
{
Logger.log($"Getting {pageRequest.title} page from memory");
Page page = pages.FirstOrDefault(p => p.title.Equals(pageRequest.title, StringComparison.InvariantCultureIgnoreCase));
if (page == null)
{
Logger.log($"{pageRequest.title} page not in memory");
page = backup?.getPage(pageRequest);
if (page != null)
{
savePage(page);
}
}
return(page);
}
public override Page getPage(PageRequestMessage pageRequest)
{
Logger.log($"Getting {pageRequest.title} page from pool");
Page page = pool.requestPage(pageRequest);
if (page == null)
{
Logger.log($"{pageRequest.title} page not in pool");
page = backup?.getPage(pageRequest);
}
if (page != null)
{
savePage(page);
}
return(page);
}
public Page requestPage(PageRequestMessage pageRequest)
{
pageRequest.ignorantPeerUris.Add(memberInfo.uri.toStringWithPort());
PoolMember nonIgnorantPoolMember = members.Where(m => !pageRequest.ignorantPeerUris.Contains(m.uri.toStringWithPort())).getRandomElement();
Page page = null;
if (nonIgnorantPoolMember != null)
{
StringContent form = new StringContent(JsonConvert.SerializeObject(pageRequest.ignorantPeerUris));
Logger.log($"Asking {nonIgnorantPoolMember.uri.toStringWithPort()} for the {pageRequest.title} page");
HttpResponseMessage pageResponse = client.PostAsync($"http://{nonIgnorantPoolMember.uri.toStringWithPort()}/data/{pageRequest.title}", form).Result;
string responseString = pageResponse.Content.ReadAsStringAsync().Result;
page = JsonConvert.DeserializeObject <Page>(responseString);
}
return(page);
}
public abstract Page getPage(PageRequestMessage pageRequest);