public static PageModel GetContentFromPage(string url)
{
var pageType = url.Split('/').LastOrDefault().Split('.').FirstOrDefault();
var path = "Resources/xml/" + pageType + ".xml";
//If the requested file doesn't exist, or if the file is more than 12 hours old, try getting new file
if (!File.Exists(path)
|| File.GetLastWriteTimeUtc(path) < DateTime.UtcNow.Subtract(new TimeSpan(1, 0, 0)))
{
try
{
//keep a list of all running threads
var threadList = new List<Thread>();
var html = new HtmlWeb();
var page = html.Load(url);
var itemList =
page.GetElementbyId("contentPrimary").Elements("div").Where(
node => node.Attributes["class"].Value == "item").ToList();
//Create file structure
var xml = new XDocument(
new XElement("PageModel", new XAttribute("id", pageType),
new XElement("TextList"),
new XElement("ImageList"),
new XElement("LinksList")
)
);
var nNode = 1;
foreach (var htmlNode in itemList)
{
//Add element to text list for each htmlNode, with title, info, date and place
var element = xml.Descendants("TextList").FirstOrDefault();
element.Add(
new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "title"),
htmlNode.Descendants("a").FirstOrDefault().InnerText.Replace("&", "and")));
element.Add(
new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "info"),
htmlNode.Descendants("p").FirstOrDefault().InnerText.Replace("&", "and")));
var content = htmlNode.Descendants("small").FirstOrDefault().InnerText.Replace("&", "and");
element.Add(
new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "date"),
(pageType == "Events" ? content.Substring(0, content.IndexOf("\r\nWhere: ")) : content)));
element.Add(
new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "place"),
(pageType == "Events" ? content.Substring(content.IndexOf("Where: ")) : "")));
//build path for image
var imagePath = "images/" + pageType + "/" +
htmlNode.Descendants("a")
.FirstOrDefault()
.InnerText.Replace("/", "-")
.Replace("\\", "-")
.Replace(" ", "") + ".jpg";
imagePath = imagePath.Replace(":", " -");
//if the image is not already in cache, create a new thread and download
if (!File.Exists("Resources/" + imagePath))
{
//Check if folder structure is there, if not create folder
if (!Directory.Exists("Resources/images/" + pageType))
Directory.CreateDirectory("Resources/images/" + pageType);
//create a new ThreadedDataFetcherObject
var isImage = htmlNode.Descendants("img").FirstOrDefault();
if (isImage != null)
{
ThreadedDataFetcher fetcher =
new ThreadedDataFetcher(
new Uri("http://www.childcancer.org.nz" +
htmlNode.Descendants("img").FirstOrDefault().Attributes["src"].Value),
"Resources/" + imagePath);
//add the downloadFile method to a thread
var th = new Thread(fetcher.downloadFile);
//add the thread to the threadList
threadList.Add(th);
th.Start();
}
else
{
imagePath = "images/logoCCF.png";
}
}
xml.Descendants("ImageList").FirstOrDefault().Add(
new XElement("Image", new XAttribute("node", nNode), imagePath));
nNode++;
}
//write all text to filesure
xml.WriteXml("Resources/xml/" + pageType + ".xml");
//join all the threads to make they have all finished
foreach (var t in threadList)
{
t.Join();
}
}
catch (Exception)
{
//If an Exception happens while getting data, return existing data, or Error data (create if it does not exist)
if (File.Exists(path))
return GetContentFromFile(path);
if (!File.Exists("Resources/xml/Error.xml"))
{
var xml = new XDocument(
new XElement("PageModel", new XAttribute("id", "Error"),
new XElement("TextList",
new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "title"),
"Content could not be received"),
new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "info"),
"Content could not be retrieved from the web. Please connect to the internet and try again."),
new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "date")),
new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "place"))
),
new XElement("ImageList"),
new XElement("LinksList")
)
);
xml.WriteXml("Resources/xml/Error.xml");
}
return GetContentFromFile("Resources/xml/Error.xml");
}
}
return GetContentFromFile(path);
}
public static void ResolveWithStandardMergetool(
Repository repository,
string fullConflictPath,
XDocument baseContent,
XDocument localContent,
XDocument incomingContent,
Logger logger,
string conflict )
{
// Run the standard mergetool to deal with any remaining issues.
var basePath = fullConflictPath + "_base";
var localPath = fullConflictPath + "_local";
var incomingPath = fullConflictPath + "_incoming";
baseContent.WriteXml( basePath );
localContent.WriteXml( localPath );
incomingContent.WriteXml( incomingPath );
if ( RunStandardMergetool( repository, basePath, localPath, incomingPath, fullConflictPath ) == 0 ) {
// The merge tool reports that the conflict was resolved
logger.Info( "Resolved " + fullConflictPath + " using standard merge tool" );
File.Delete( fullConflictPath );
File.Move( localPath, fullConflictPath );
repository.Stage( conflict );
} else {
logger.Info( "Did not resolve " + fullConflictPath );
throw new OperationCanceledException();
}
File.Delete( basePath );
File.Delete( incomingPath );
}
