public static PageModel GetContentFromPage(string url) { var pageType = url.Split('/').LastOrDefault().Split('.').FirstOrDefault(); var path = "Resources/xml/" + pageType + ".xml"; //If the requested file doesn't exist, or if the file is more than 12 hours old, try getting new file if (!File.Exists(path) || File.GetLastWriteTimeUtc(path) < DateTime.UtcNow.Subtract(new TimeSpan(1, 0, 0))) { try { //keep a list of all running threads var threadList = new List<Thread>(); var html = new HtmlWeb(); var page = html.Load(url); var itemList = page.GetElementbyId("contentPrimary").Elements("div").Where( node => node.Attributes["class"].Value == "item").ToList(); //Create file structure var xml = new XDocument( new XElement("PageModel", new XAttribute("id", pageType), new XElement("TextList"), new XElement("ImageList"), new XElement("LinksList") ) ); var nNode = 1; foreach (var htmlNode in itemList) { //Add element to text list for each htmlNode, with title, info, date and place var element = xml.Descendants("TextList").FirstOrDefault(); element.Add( new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "title"), htmlNode.Descendants("a").FirstOrDefault().InnerText.Replace("&", "and"))); element.Add( new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "info"), htmlNode.Descendants("p").FirstOrDefault().InnerText.Replace("&", "and"))); var content = htmlNode.Descendants("small").FirstOrDefault().InnerText.Replace("&", "and"); element.Add( new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "date"), (pageType == "Events" ? content.Substring(0, content.IndexOf("\r\nWhere: ")) : content))); element.Add( new XElement("Text", new XAttribute("node", nNode), new XAttribute("type", "place"), (pageType == "Events" ? content.Substring(content.IndexOf("Where: ")) : ""))); //build path for image var imagePath = "images/" + pageType + "/" + htmlNode.Descendants("a") .FirstOrDefault() .InnerText.Replace("/", "-") .Replace("\\", "-") .Replace(" ", "") + ".jpg"; imagePath = imagePath.Replace(":", " -"); //if the image is not already in cache, create a new thread and download if (!File.Exists("Resources/" + imagePath)) { //Check if folder structure is there, if not create folder if (!Directory.Exists("Resources/images/" + pageType)) Directory.CreateDirectory("Resources/images/" + pageType); //create a new ThreadedDataFetcherObject var isImage = htmlNode.Descendants("img").FirstOrDefault(); if (isImage != null) { ThreadedDataFetcher fetcher = new ThreadedDataFetcher( new Uri("http://www.childcancer.org.nz" + htmlNode.Descendants("img").FirstOrDefault().Attributes["src"].Value), "Resources/" + imagePath); //add the downloadFile method to a thread var th = new Thread(fetcher.downloadFile); //add the thread to the threadList threadList.Add(th); th.Start(); } else { imagePath = "images/logoCCF.png"; } } xml.Descendants("ImageList").FirstOrDefault().Add( new XElement("Image", new XAttribute("node", nNode), imagePath)); nNode++; } //write all text to filesure xml.WriteXml("Resources/xml/" + pageType + ".xml"); //join all the threads to make they have all finished foreach (var t in threadList) { t.Join(); } } catch (Exception) { //If an Exception happens while getting data, return existing data, or Error data (create if it does not exist) if (File.Exists(path)) return GetContentFromFile(path); if (!File.Exists("Resources/xml/Error.xml")) { var xml = new XDocument( new XElement("PageModel", new XAttribute("id", "Error"), new XElement("TextList", new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "title"), "Content could not be received"), new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "info"), "Content could not be retrieved from the web. Please connect to the internet and try again."), new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "date")), new XElement("Text", new XAttribute("node", "1"), new XAttribute("type", "place")) ), new XElement("ImageList"), new XElement("LinksList") ) ); xml.WriteXml("Resources/xml/Error.xml"); } return GetContentFromFile("Resources/xml/Error.xml"); } } return GetContentFromFile(path); }
public static void ResolveWithStandardMergetool( Repository repository, string fullConflictPath, XDocument baseContent, XDocument localContent, XDocument incomingContent, Logger logger, string conflict ) { // Run the standard mergetool to deal with any remaining issues. var basePath = fullConflictPath + "_base"; var localPath = fullConflictPath + "_local"; var incomingPath = fullConflictPath + "_incoming"; baseContent.WriteXml( basePath ); localContent.WriteXml( localPath ); incomingContent.WriteXml( incomingPath ); if ( RunStandardMergetool( repository, basePath, localPath, incomingPath, fullConflictPath ) == 0 ) { // The merge tool reports that the conflict was resolved logger.Info( "Resolved " + fullConflictPath + " using standard merge tool" ); File.Delete( fullConflictPath ); File.Move( localPath, fullConflictPath ); repository.Stage( conflict ); } else { logger.Info( "Did not resolve " + fullConflictPath ); throw new OperationCanceledException(); } File.Delete( basePath ); File.Delete( incomingPath ); }