private IElement LoadCountryMapPage(string countryUrl)
{
string htmlText = new HttpRequestAndLoad().Load(countryUrl);
var htmlParser = new HtmlParser();
return(htmlParser.Parse(htmlText).DocumentElement);
}
private void LoadIndexPage()
{
string htmlText = new HttpRequestAndLoad().Load(rootUrl);
var htmlParser = new HtmlParser();
htmlDocument = htmlParser.Parse(htmlText);
}
static void Main(string[] args)
{
logger.Info("Logger started");
var pageUrl = "https://en.wikipedia.org/wiki/Gallery_of_sovereign_state_flags";
var httpDownload = new HttpRequestAndLoad();
string httpText = httpDownload.Load(pageUrl);
var httpParser = new HtmlParser();
var element = httpParser.Parse(httpText).DocumentElement;
var flagsHref = element.SelectNodes(@"//*[@id='mw-content-text']/div/table//table//tr[1]/td[1]/a/img");
// Async Timer
using (var timer = new LogWrapTimer())
{
var downloadQueue = new DownloadFileQueue()
{
Extension = ".jpg",
AsyncMode = false
};
downloadQueue.Setup("Flags");
foreach (IElement flagHref in flagsHref)
{
var urlRelative = flagHref.GetAttribute("src");
var urlAbsolute = new Uri(new Uri(pageUrl), urlRelative);
downloadQueue.AddTask(urlAbsolute.AbsoluteUri);
}
downloadQueue.Wait();
}
}
/// <summary>
/// Download the resposne and parse into xml document
/// </summary>
/// <param name="api"></param>
/// <returns></returns>
protected virtual XmlParserWrapper RequestXmlResponse(string api)
{
logger.Info($"Request web resource from API '{api}'");
var config = new HttpRequestConfiguration();
var httpRequest = new HttpRequestAndLoad(config);
var responseXml = httpRequest.Load(api);
return(new XmlParserWrapper(responseXml, "wb"));
}
private void Process(string url, string fileName)
{
var httpText = new HttpRequestAndLoad().Load(url);
var rootElement = new HtmlParser().Parse(httpText).DocumentElement;
var scriptTags = rootElement.SelectNodes("script");
var aHrefTags = rootElement.SelectNodes("a[@href]");
var imageTags = rootElement.SelectNodes("img[@src]");
var linksTags = rootElement.SelectNodes("link");
var objectTags = rootElement.SelectNodes("object"); // Svg
}
/// <summary>
/// Download the resposne and parse into xml document
/// </summary>
/// <param name="api"></param>
/// <returns></returns>
protected virtual XmlDocument RequestXmlResponse(string api)
{
logger.Info($"Request web resource from API '{api}'");
var config = new HttpRequestConfiguration();
var httpRequest = new HttpRequestAndLoad(config);
var responseXml = httpRequest.Load(api);
var document = new XmlDocument();
document.LoadXml(responseXml);
namespaceManager = new XmlNamespaceManager(document.NameTable);
namespaceManager.AddNamespace("wb", document.DocumentElement.NamespaceURI);
return(document);
}