public void Save()
{
var body = doc.DocumentNode.SelectSingleNode("//body");
body.InnerHtml = "";
currentUnit.Write(body);
var stream = File.CreateText(fileList[currentUnitName]);
doc.Save(stream);
}
public static string Inject(string htmlText)
{
//https://stackoverflow.com/questions/9520932/how-do-i-use-html-agility-pack-to-edit-an-html-snippet
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(htmlText);
foreach (var node in doc.DocumentNode.Descendants().Where(n => n.NodeType == HtmlAgilityPack.HtmlNodeType.Element))
{
var classAttr = node.Attributes["class"];
if (classAttr != null)
{
classAttr.Value = $"{classAttr.Value} {cssClassMarker}";
}
else
{
node.SetAttributeValue("class", cssClassMarker);
}
node.Attributes.Add("lineNumber", node.Line.ToString());
node.Attributes.Add("column", node.LinePosition.ToString());
}
using (StringWriter writer = new StringWriter())
{
doc.Save(writer);
var result = writer.ToString();
return(result);
}
}
public string FormatArticleContent(string content)
{
//var baseUri = new Uri(publicSiteUrl);
var baseUri = new Uri(_appSettings.Value.PublicSiteBaseUrl);
var tags = new[] { "img", "iframe", "video", "audio" };
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(content);
foreach (var node in doc.DocumentNode.Descendants().Where(n => tags.Contains(n.Name)))
{
if (node.HasAttributes)
{
var attr = node.Attributes["src"];
if (attr != null)
{
attr.Value = GetAbsoluteUrl(baseUri, attr.Value);
}
}
}
using (var writer = new StringWriter())
{
doc.Save(writer);
var formattedContent = writer.ToString();
return(formattedContent);
}
}
public void Process(Crawler crawler, PropertyBag propertyBag)
{
if (propertyBag["HtmlDoc"].Value == null)
{
return;
}
string gender = "gender";
string age_min = "age_min";
string age_max = "age_max";
string start = "start";
gender += "=" + CrawlUtil.getQueryValueFromUrl(propertyBag.ResponseUri.AbsoluteUri, gender);
age_min += "=" + CrawlUtil.getQueryValueFromUrl(propertyBag.ResponseUri.AbsoluteUri, age_min);
age_max += "=" + CrawlUtil.getQueryValueFromUrl(propertyBag.ResponseUri.AbsoluteUri, age_max);
start += "=" + CrawlUtil.getQueryValueFromUrl(propertyBag.ResponseUri.AbsoluteUri, start);
ReviewCrawler.MainForm.appendLineToLog(propertyBag.Title);
ReviewCrawler.MainForm.appendLineToLog(gender);
ReviewCrawler.MainForm.appendLineToLog(age_min);
ReviewCrawler.MainForm.appendLineToLog(age_max);
ReviewCrawler.MainForm.appendLineToLog(start);
HtmlAgilityPack.HtmlDocument doc = propertyBag["HtmlDoc"].Value as HtmlAgilityPack.HtmlDocument;
doc.Save("HtmlDump/"
+ CrawlUtil.SanitiseFileName(CrawlUtil.getMovieNameFromTitle(HttpUtility.HtmlDecode(propertyBag.Title)))
+ "#" + gender
+ "#" + age_min
+ "#" + age_max
+ "#" + start
+ ".html");
}
public static string RelativeToAbsoluteUrls(string html, string siteUrl)
{
StringWriter writer = new StringWriter();
string baseUrl = siteUrl;
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
foreach (var link in doc.DocumentNode.Descendants("link"))
{
link.Attributes["href"].Value = new Uri(new Uri(baseUrl), link.Attributes["href"].Value).AbsoluteUri;
}
foreach (var img in doc.DocumentNode.Descendants("img"))
{
img.Attributes["src"].Value = new Uri(new Uri(baseUrl), img.Attributes["src"].Value).AbsoluteUri;
}
foreach (var a in doc.DocumentNode.Descendants("a"))
{
a.Attributes["href"].Value = new Uri(new Uri(baseUrl), a.Attributes["href"].Value).AbsoluteUri;
}
doc.Save(writer);
string newHtml = writer.ToString();
return(newHtml);
}
private void frmSettings_Load(object sender, EventArgs e)
{
Thread.CurrentThread.CurrentUICulture = new CultureInfo("en");
var svgPath = @"C:\Users\Pc\Documents\TBI Chart.svg";
HtmlAgilityPack.HtmlDocument svg = new HtmlAgilityPack.HtmlDocument();
svg.Load(svgPath);
var text = svg.DocumentNode.Descendants("text");
//var rect = svg.DocumentNode.Descendants("rect");
var max = 66.2;
var basic = 24.2;
var good = 45.2;
double height = svg.GetElementbyId("bar_current").GetAttributeValue("height", 0);
double top = svg.GetElementbyId("bar_current").GetAttributeValue("y", 0);
text.ElementAt(6).InnerHtml = text.ElementAt(0).InnerHtml = $"{max}".ToNumber();
text.ElementAt(5).InnerHtml = text.ElementAt(1).InnerHtml = $"{max/2}".ToNumber();
svg.GetElementbyId("bar_basic_as").SetAttributeValue("height", $"{basic * height / max}".ToNumber());
svg.GetElementbyId("bar_basic_as").SetAttributeValue("y", $"{(top + height) - (basic * height / max)}".ToNumber());
svg.GetElementbyId("bar_good_as").SetAttributeValue("height", $"{good * height / max}".ToNumber());
svg.GetElementbyId("bar_good_as").SetAttributeValue("y", $"{(top + height) - (good * height / max)}".ToNumber());
var savePath = $@"C:\Users\Pc\Documents\temp\{Guid.NewGuid()}.svg";
svg.Save(savePath);
webBrowser1.Navigate(svgPath);
}
/// <summary>
/// See interface docs.
/// </summary>
/// <param name="html"></param>
/// <param name="encoding"></param>
public string Html(string html, Encoding encoding)
{
var result = html;
var document = new HtmlAgilityPack.HtmlDocument()
{
OptionCheckSyntax = false,
OptionDefaultStreamEncoding = encoding,
};
document.LoadHtml(html);
if (Html(document))
{
using (var stream = new MemoryStream()) {
document.Save(stream);
stream.Position = 0;
using (var streamReader = new StreamReader(stream, encoding, true)) {
result = streamReader.ReadToEnd();
}
}
}
return(result);
}
private static String ParseSaveAndFixImages(string contents, string dirPath)
{
contents = System.Web.HttpUtility.HtmlDecode(contents);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(new StringReader(contents));
var nav = doc.CreateNavigator();
var strExpression = "//img";
HtmlAgilityPack.HtmlNodeCollection imgTags = doc.DocumentNode.SelectNodes(strExpression);
if (imgTags != null)
{
foreach (HtmlAgilityPack.HtmlNode tag in imgTags)
{
if (tag.Attributes["src"] != null)
{
String imgPath = tag.Attributes["src"].Value;
tag.Attributes["src"].Value = GetAndSaveImage(imgPath, dirPath);
}
}
}
string finalContents = null;
using (StringWriter sw = new StringWriter())
{
doc.Save(sw);
finalContents = sw.ToString();
}
return(finalContents);
}
List<Show> Grab(GrabParametersBase p)
{
var shows = new List<Show>();
try
{
var param = (GrabParameters)p;
var wr = WebRequest.Create(string.Format(urlFormat, (int)param.ChannelId));
_logger.WriteEntry(string.Format("Grabbing Channel {0} ...", param.ChannelId), LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(res.GetResponseStream());
doc.OptionOutputAsXml = true;
var writer = new StringWriter();
doc.Save(writer);
var xml = XDocument.Load(new StringReader(writer.ToString()));
FillShows(xml, shows);
for (int i = shows.Count - 1; i >= 0; i--)
{
var show = shows[i];
show.Channel = param.ChannelId.ToString();
if (i == shows.Count - 1)
show.EndTime = show.StartTime.AddHours(12);// usually 3-4 days from now , not that important
else
show.EndTime = shows[i + 1].StartTime;
}
}
catch (Exception ex)
{
_logger.WriteEntry(ex.Message, LogType.Error);
}
_logger.WriteEntry(string.Format("Found {0} Shows", shows.Count), LogType.Info);
return shows;
}
/// <summary>
/// Update <img src> url
/// </summary>
private async Task <string> SetImgUrls(string htmlBody, string localGetImgRelativeURI)
{
await Task.CompletedTask;
var htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(htmlBody);
var imgItems = htmlDoc.DocumentNode.SelectNodes("//img")?.ToList();
if (imgItems != null)
{
foreach (var imgItem in imgItems)
{
string urlSrc = imgItem.Attributes["src"].Value;
if (urlSrc != null)
{
if (!urlSrc.ToLower().StartsWith("http"))
{
string newImgSrc = localGetImgRelativeURI + Convert.ToBase64String(Encoding.UTF8.GetBytes(urlSrc));
imgItem.Attributes["src"].Value = newImgSrc;
}
}
}
}
var sw = new StringWriter();
htmlDoc.Save(sw);
return(sw.ToString());
}
/// <summary>
/// Load HTML DOM from given <paramref name="stream"/>.
/// </summary>
private bool loadHTML(Context ctx, TextReader stream, string filename, int options = 0)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
// setup HTML parser
htmlDoc.OptionOutputAsXml = true;
//htmlDoc.OptionOutputOriginalCase = true; // NOTE: we need lower-cased names because of XPath queries
//htmlDoc.OptionFixNestedTags = true;
htmlDoc.OptionCheckSyntax = false;
htmlDoc.OptionUseIdAttribute = false; // only needed when XPath navigator is used on htmlDoc
htmlDoc.OptionWriteEmptyNodes = true;
// load HTML (from string or a stream)
htmlDoc.Load(stream);
CheckHtmlErrors(ctx, htmlDoc, filename);
// save to string as XML
using (var sw = new StringWriter())
{
htmlDoc.Save(sw);
// load as XML
return(loadXMLInternal(ctx, sw.ToString(), 0, true));
}
}
private static void TranformHtml(IDocumentBuildContext context, string transformed, string relativeModelPath, StreamWriter outputWriter)
{
// Update HREF and XREF
HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(transformed);
var xrefLinkNodes = html.DocumentNode.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefExceptions = new List <CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref/@href");
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Constants.DefaultLanguage);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, relativeModelPath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, relativeModelPath);
}
}
html.Save(outputWriter);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
public static string GetOutputHTML(this HtmlAgilityPack.HtmlDocument source)
{
StringWriter stream = new StringWriter();
source.Save(stream);
return(stream.ToString());
}
/// <summary>
/// Load HTML DOM from given <paramref name="stream"/>.
/// </summary>
private object loadHTML(TextReader stream, string filename)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
// setup HTML parser
htmlDoc.OptionOutputAsXml = true;
//htmlDoc.OptionOutputOriginalCase = true; // NOTE: we need lower-cased names because of XPath queries
//htmlDoc.OptionFixNestedTags = true;
htmlDoc.OptionCheckSyntax = false;
htmlDoc.OptionWriteEmptyNodes = true;
// load HTML (from string or a stream)
htmlDoc.Load(stream);
CheckHtmlErrors(htmlDoc, filename);
// save to string as XML
using (StringWriter sw = new StringWriter())
{
htmlDoc.Save(sw);
// load as XML
return(this.loadXMLInternal(sw.ToString(), 0, true));
}
}
/// <summary>
/// See interface docs.
/// </summary>
/// <param name="environment"></param>
/// <param name="textContent"></param>
public void ManipulateTextResponse(IDictionary <string, object> environment, TextContent textContent)
{
var context = OwinContext.Create(environment);
var pathAndFile = context.RequestPathFlattened;
var allInjectors = _HtmlContentInjectors;
var injectors = allInjectors.Where(r =>
!String.IsNullOrEmpty(r.Element) &&
r.Content != null &&
(r.PathAndFile == null || r.PathAndFile.Equals(pathAndFile, StringComparison.OrdinalIgnoreCase))
).ToArray();
if (injectors.Length > 0)
{
var document = new HtmlAgilityPack.HtmlDocument()
{
OptionCheckSyntax = false,
OptionDefaultStreamEncoding = textContent.Encoding,
};
document.LoadHtml(textContent.Content);
var modified = false;
foreach (var injector in injectors.OrderByDescending(r => r.Priority))
{
var elements = document.DocumentNode.Descendants(injector.Element);
var element = injector.AtStart ? elements.FirstOrDefault() : elements.LastOrDefault();
var content = element == null ? null : injector.Content();
if (element != null && !String.IsNullOrEmpty(content))
{
var subDocument = new HtmlAgilityPack.HtmlDocument()
{
OptionCheckSyntax = false,
};
subDocument.LoadHtml(injector.Content());
if (injector.AtStart)
{
element.PrependChild(subDocument.DocumentNode);
}
else
{
element.AppendChild(subDocument.DocumentNode);
}
modified = true;
}
}
if (modified)
{
using (var stream = new MemoryStream()) {
document.Save(stream);
stream.Position = 0;
using (var streamReader = new StreamReader(stream, textContent.Encoding, true)) {
textContent.Content = streamReader.ReadToEnd();
}
}
}
}
}
public static string ToHTML(this HtmlAgilityPack.HtmlDocument Document)
{
using (var Stream = new MemoryStream())
{
Document.Save(Stream);
return(Document.Encoding.GetString(Stream.ToArray()));
}
}
private static void TranformHtml(IDocumentBuildContext context, string transformed, string relativeModelPath, string outputPath)
{
// Update HREF and XREF
HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(transformed);
var xrefExceptions = new List <CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref/@href");
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Language);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, relativeModelPath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, relativeModelPath);
}
}
// Save with extension changed
var subDirectory = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(subDirectory) && !Directory.Exists(subDirectory))
{
Directory.CreateDirectory(subDirectory);
}
html.Save(outputPath, Encoding.UTF8);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
static FineAntsCore.Statement ConvertHSBCHTMLFileToFineAnts(FileInfo fileInfo)
{
HtmlAgilityPack.HtmlDocument brokenDocument = new HtmlAgilityPack.HtmlDocument();
brokenDocument.Load(fileInfo.FullName);
brokenDocument.OptionOutputAsXml = true;
string fixedXmlFileName = fileInfo.FullName + ".fixed.xml";
brokenDocument.Save(fixedXmlFileName);
XmlDocument document = new XmlDocument();
document.Load(fixedXmlFileName);
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(document.NameTable);
namespaceManager.AddNamespace("d", "http://www.w3.org/1999/xhtml");
XmlNode closingBalanceNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[last()]/d:td[6]/d:p", namespaceManager);
XmlNode closingBalanceSignNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[last()]/d:td[7]/d:p", namespaceManager);
int closingBalance = moneyInPenceFromString(closingBalanceNode.InnerText.Trim());
if (closingBalanceSignNode.InnerText.Trim() == "D") closingBalance = -closingBalance;
XmlNode endDateNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:div[@class='extPibRow hsbcRow']/d:div[@class='hsbcPadding']/d:div[@class='hsbcTextRight']", namespaceManager);
string endDateString = HtmlAgilityPack.HtmlEntity.DeEntitize(endDateNode.InnerText).Trim();
System.Globalization.CultureInfo provider = System.Globalization.CultureInfo.InvariantCulture;
DateTime endDate = DateTime.ParseExact(endDateString, "dd MMM yyyy", provider);
XmlNode startDateNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[1]/d:td[1]/d:p", namespaceManager);
string startDateString = HtmlAgilityPack.HtmlEntity.DeEntitize(startDateNode.InnerText).Trim();
DateTime startDate = dateFromDateStringFixedUsingUpperBoundDate(startDateString, endDate.AddDays(-1)).AddDays(1);
List<FineAntsCore.Transaction> transactions = new List<FineAntsCore.Transaction>();
XmlNodeList transactionNodes = document.SelectNodes("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[position()>1 and position()<last()]", namespaceManager);
foreach (XmlNode node in transactionNodes)
{
XmlNode dateNode = node.SelectSingleNode("d:td[1]/d:p", namespaceManager);
XmlNode typeNode = node.SelectSingleNode("d:td[2]/d:p", namespaceManager);
XmlNode nameNode = node.SelectSingleNode("d:td[3]/d:p", namespaceManager);
XmlNode moneyOutNode = node.SelectSingleNode("d:td[4]/d:p", namespaceManager);
XmlNode moneyInNode = node.SelectSingleNode("d:td[5]/d:p", namespaceManager);
string date = HtmlAgilityPack.HtmlEntity.DeEntitize(dateNode.InnerText).Trim();
string name = HtmlAgilityPack.HtmlEntity.DeEntitize(getInnerTextIgnoringLinks(nameNode));
string moneyIn = HtmlAgilityPack.HtmlEntity.DeEntitize(moneyInNode.InnerText).Trim();
string moneyOut = HtmlAgilityPack.HtmlEntity.DeEntitize(moneyOutNode.InnerText).Trim();
int money = moneyIn == "" ? -moneyInPenceFromString(moneyOut) : moneyInPenceFromString(moneyIn);
transactions.Add(new FineAntsCore.Transaction(money, dateFromDateStringFixedUsingUpperBoundDate(date, endDate), name, ""));
}
// remove the temporary fixed file
System.IO.File.Delete(fixedXmlFileName);
FineAntsCore.Statement statement = new FineAntsCore.Statement(transactions, startDate, endDate, closingBalance);
return statement;
}
private string SetTitles(string value)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(value);
HtmlAgilityPack.HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//img[@src]");
if (nodes == null || nodes.Count == 0)
{
return(value);
}
string id = null; // non-null indicates need to update field value after looping
foreach (HtmlAgilityPack.HtmlNode node in nodes)
{
string src = node.GetAttributeValue("src", String.Empty);
if (src == null)
{
continue;
}
Match match = this.regex.Match(src);
if (match.Success)
{
id = match.Groups[1].Value;
Sitecore.Data.ID guid = Sitecore.Data.ID.Parse(id);
Sitecore.Data.Items.Item item = Sitecore.Context.Database.GetItem(guid);
if (item == null)
{
continue;
}
string title = String.Format(
"{0} [{1}, {2}]",
item.Name,
item["extension"].ToUpper(),
this.FormatBytes(Int32.Parse(item["size"])));
node.SetAttributeValue("title", title);
}
}
if (id == null)
{
return(value);
}
StringBuilder sb = new StringBuilder();
StringWriter sw = new StringWriter(sb);
doc.Save(sw);
sw.Flush();
return(sb.ToString());
}
string[] ExtractFormAndHiddenControls(Response response)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(response.Body);
var tempxml = new StringBuilder();
var tsw = new StringWriter(tempxml);
htmlDoc.OptionOutputAsXml = true;
htmlDoc.Save(tsw);
var doc = new XmlDocument();
doc.LoadXml(tempxml.ToString());
XmlNamespaceManager nsmgr = new XmlNamespaceManager(doc.NameTable);
nsmgr.AddNamespace("html", HTML_NAMESPACE);
XmlNode formNode = doc.SelectSingleNode("//html:form", nsmgr);
if (formNode == null)
{
throw new ArgumentException("Form was not found in document: " + response.Body);
}
string actionUrl = formNode.Attributes ["action"].Value;
XmlNode method = formNode.Attributes ["method"];
var data = new List <string> ();
string name, value;
foreach (XmlNode inputNode in doc.SelectNodes("//html:input[@type='hidden']", nsmgr))
{
name = inputNode.Attributes["name"].Value;
if (String.IsNullOrEmpty(name))
{
continue;
}
XmlAttribute attr = inputNode.Attributes["value"];
if (attr != null)
{
value = attr.Value;
}
else
{
value = String.Empty;
}
data.Add(name);
data.Add(value);
}
return(data.ToArray());
}
private void ExtractFormAndHiddenControls (Response response, string formId)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument ();
htmlDoc.LoadHtml (response.Body);
StringBuilder tempxml = new StringBuilder ();
StringWriter tsw = new StringWriter (tempxml);
htmlDoc.OptionOutputAsXml = true;
htmlDoc.Save (tsw);
XmlDocument doc = new XmlDocument ();
doc.LoadXml (tempxml.ToString ());
const string HTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
XmlNamespaceManager nsmgr = new XmlNamespaceManager (doc.NameTable);
nsmgr.AddNamespace ("html", HTML_NAMESPACE);
#if USE_CORRECT_FORMID
XmlNode formNode = doc.SelectSingleNode ("//html:form[@name='" + formId + "']", nsmgr);
#else
XmlNode formNode = doc.SelectSingleNode ("//html:form", nsmgr);
#endif
if (formNode == null)
throw new ArgumentException ("Form with id='" + formId +
"' was not found in document: " + response.Body);
string actionUrl = formNode.Attributes["action"].Value;
if (actionUrl != null && actionUrl != string.Empty)
base.Url = actionUrl;
XmlNode method = formNode.Attributes["method"];
if (method != null && "POST" == method.Value.ToUpper(CultureInfo.InvariantCulture))
base.IsPost = true;
else
base.IsPost = false;
#if USE_CORRECT_FORMID
foreach (XmlNode inputNode in formNode.SelectNodes ("//html:input", nsmgr))
#else
foreach (XmlNode inputNode in doc.SelectNodes ("//html:input[@type='hidden']", nsmgr))
#endif
{
BaseControl bc = new BaseControl ();
bc.Name = inputNode.Attributes["name"].Value;
if (bc.Name == null || bc.Name == string.Empty)
continue;
if (inputNode.Attributes["value"] != null)
bc.Value = inputNode.Attributes["value"].Value;
else
bc.Value = "";
Controls[bc.Name] = bc;
}
}
public static XElement HtmlToXElement(string html)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.OptionOutputAsXml = true;
doc.LoadHtml(html);
using (StringWriter writer = new StringWriter())
{
doc.Save(writer);
using (StringReader reader = new StringReader(writer.ToString()))
{
return(XElement.Load(reader));
}
}
}
private void TransformHtml(IDocumentBuildContext context, string html, string sourceFilePath, string destFilePath, StreamWriter outputWriter)
{
// Update href and xref
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(html);
var xrefExceptions = TransformHtmlCore(context, sourceFilePath, destFilePath, document);
document.Save(outputWriter);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
private string HtmltoXml(string html)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
StringBuilder fixedxml = new StringBuilder();
StringWriter sw = new StringWriter(fixedxml);
try
{
StringBuilder tempxml = new StringBuilder();
StringWriter tsw = new StringWriter(tempxml);
doc.OptionOutputAsXml = true;
doc.Save(tsw);
// fix style attribute
// the reason is that style attribute name-value pairs come in different order
// in .NET and GH
// Here I will sort the values of style attribute
XmlDocument tempDoc = new XmlDocument();
tempDoc.LoadXml(tempxml.ToString());
XmlNodeList allNodes = tempDoc.SelectNodes("//*");
foreach (XmlNode n in allNodes)
{
if (n.Attributes["style"] != null)
{
string att = n.Attributes["style"].Value;
string [] style = att.Trim(new char[] { ' ', ';' }).Split(';');
for (int styleIndex = 0; styleIndex < style.Length; styleIndex++)
{
style[styleIndex] = FixStyleNameValue(style[styleIndex]);
}
Array.Sort(style);
n.Attributes["style"].Value = string.Join(";", style);
}
}
tempDoc.Save(sw);
}
catch (Exception)
{
Console.WriteLine("Error parsing html response...");
Console.WriteLine("Test case aborted");
return("<TestCaseAborted></TestCaseAborted>");
}
return(fixedxml.ToString());
}
/// <summary>
/// When overridden in a derived class, this validates both the request and response.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
public virtual void Validate(Object sender, ValidationEventArgs e)
{
_context.Outcome = WebTestOutcome.NotExecuted;
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(((WebTestRequest)e.WebTestItem).HttpResponseBody);
document.OptionOutputAsXml = true;
using (StringWriter writer = new StringWriter())
{
document.Save(writer);
_document = XDocument.Parse(writer.GetStringBuilder().ToString());
}
}
/// <summary>
/// Returns xhtml string from html document
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
public static string HtmlToXml(string html)
{
StringBuilder stringbuild = new StringBuilder();
StringWriter stringwriter = new StringWriter(stringbuild);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
doc.OptionOutputAsXml = true;
doc.OptionCheckSyntax = true;
doc.OptionFixNestedTags = true;
doc.Save(stringwriter);
return(stringbuild.ToString());
}
/// <summary>
/// Creates OneNote page
/// </summary>
/// <param name="document"></param>
/// <returns></returns>
private static Page OpenOrCreatePage(Document document)
{
Console.WriteLine("Loading pages ...");
string title = "Microsoft Graph API example";
var pageFactory = new GraphPageFactory();
var allPages = pageFactory.GetAllItems(document.Id);
var page = allPages.FirstOrDefault(p => p.Title.Equals(title));
if (page != null)
{
return(pageFactory.GetItem(page.Id));
}
// create sample html page document and add some fields
var sampleDoc = new HtmlAgilityPack.HtmlDocument();
sampleDoc.LoadHtml(Resources.SamplePageHtml);
var body = sampleDoc.DocumentNode.SelectSingleNode("//body");
var textField = sampleDoc.CreateElement("div");
textField.SetAttributeValue("id", "dynamicElement");
textField.InnerHtml = "Dynamically added DIV element #";
body.AppendChild(textField);
//save html document as HTML
string htmlContent;
using (var htmlStream = new MemoryStream())
{
sampleDoc.Save(htmlStream, Encoding.UTF8);
htmlStream.Position = 0;
byte[] buffer = new byte[htmlStream.Length];
htmlStream.Read(buffer, 0, buffer.Length);
htmlContent = Encoding.UTF8.GetString(buffer);
}
//MemoryStream stream = ReadImage();
//page = pageFactory.AddItem(new Page { Content = htmlContent, StreamContent = stream, Title = title }, section.Id);
Console.WriteLine("Created page 'Microsoft Graph API example'...");
return(page);
}
public void Test_Save_Does_Not_Stackoverflow_Even_If_The_Dom_Is_Deep()
{
var deep = String.Join("",
Enumerable.Repeat("<div><span>", 1000)
.Concat(Enumerable.Repeat("</span></div>", 1000))
);
CrashTest(() =>
{
var doc = new HtmlAgilityPack.HtmlDocument();
var writer = new StringWriter();
doc.LoadHtml(deep);
doc.Save(writer);
});
}
public void Add(string name)
{
doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(Properties.Resources.EmptyTest);
var title = doc.DocumentNode.SelectSingleNode("//head/title");
title.InnerHtml = name;
string filename = CreateFileName();
fileList.Add(name, filename);
themes.Add(name);
var stream = File.CreateText(filename);
doc.Save(stream);
}
private static void TranformHtml(DocumentBuildContext context, string transformed, string relativeModelPath, string outputPath)
{
// Update HREF and XREF
var internalXref = context.XRefSpecMap;
var externalXref = context.ExternalXRefSpec;
HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(transformed);
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateSrc(link, context.FileMap, s => UpdateFilePath(s, relativeModelPath));
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
// xref is generated by docfx, and is lower-cased
if (link.Name == "xref")
{
UpdateXref(link, internalXref, externalXref, s => UpdateFilePath(s, relativeModelPath), Language);
}
else
{
UpdateHref(link, context.FileMap, s => UpdateFilePath(s, relativeModelPath));
}
}
}
// Save with extension changed
var subDirectory = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(subDirectory) && !Directory.Exists(subDirectory))
{
Directory.CreateDirectory(subDirectory);
}
html.Save(outputPath, Encoding.UTF8);
}
/// <summary>
/// Parse HTML file and find all local contents to download.
/// </summary>
/// <param name="pnum">Problem number</param>
/// <param name="replace">True, if you want to replace old files.</param>
/// <returns>List of files to download</returns>
public static List <DownloadTask> ProcessHtmlContent(long pnum, bool replace)
{
try
{
string external = string.Format("http://uva.onlinejudge.org/external/{0}/", pnum / 100);
string filepath = LocalDirectory.GetProblemHtml(pnum);
if (!File.Exists(filepath))
{
return(new List <DownloadTask>());
}
List <string> urls = new List <string>();
List <DownloadTask> tasks = new List <DownloadTask>();
HtmlAgilityPack.HtmlDocument htdoc = new HtmlAgilityPack.HtmlDocument();
htdoc.Load(filepath);
DFS(htdoc.DocumentNode, urls);
htdoc.Save(filepath);
foreach (string str in urls)
{
string url = str.StartsWith("./") ? str.Remove(0, 2) : str;
while (url.StartsWith("/"))
{
url = url.Remove(0, 1);
}
string file = url.Replace('/', Path.DirectorySeparatorChar);
file = LocalDirectory.GetProblemContent(pnum, file);
if (replace || LocalDirectory.GetFileSize(file) < 10)
{
tasks.Add(new DownloadTask(external + url, file, pnum));
}
}
urls.Clear();
return(tasks);
}
catch (Exception ex)
{
Logger.Add(ex.Message, "Internet");
return(new List <DownloadTask>());
}
}
public string HtmltoXml(string html) //throws XmlException
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html.Trim(new char[] { '\r', '\n', ' ' })); // bug in HtmlAgilityPack
StringBuilder fixedxml = new StringBuilder();
StringWriter sw = new StringWriter(fixedxml);
StringBuilder tempxml = new StringBuilder();
StringWriter tsw = new StringWriter(tempxml);
doc.OptionOutputAsXml = true;
doc.Save(tsw);
// fix style attribute
// the reason is that style attribute name-value pairs come in different order
// in .NET and GH
// Here I will sort the values of style attribute
XmlDocument tempDoc = new XmlDocument();
tempDoc.LoadXml(tempxml.ToString());
XmlNodeList allNodes = tempDoc.SelectNodes("//*");
foreach (XmlNode n in allNodes)
{
if (n.Attributes["style"] != null)
{
string att = n.Attributes["style"].Value;
string[] style = att.Trim(new char[] { ' ', ';' }).Split(';');
for (int styleIndex = 0; styleIndex < style.Length; styleIndex++)
{
style[styleIndex] = FixStyleNameValue(style[styleIndex]);
}
Array.Sort(style);
n.Attributes["style"].Value = string.Join(";", style);
}
}
tempDoc.Save(sw);
return(fixedxml.ToString());
}
/// <summary>
/// 将port端口加入html文件的id=port控件
/// </summary>
/// <param name="htmlDirPath"></param>
/// <param name="htmlFileName"></param>
/// <returns></returns>
public static string fillPortInHtml(string htmlDirPath, string htmlFileName)
{
var htmlDoc = new HtmlAgilityPack.HtmlDocument();
var fileName = htmlFileName.Substring(0, htmlFileName.LastIndexOf('.'));
using (var htmlFileR = File.OpenRead(htmlDirPath + htmlFileName)) {
htmlDoc.Load(htmlFileR, Encoding.UTF8);
var portInput = htmlDoc.DocumentNode.SelectSingleNode("//*[@id='port']");
if (portInput != null)
{
portInput.SetAttributeValue("value", TCPHelper.port.ToString());
}
}
var newFileFullPath = htmlDirPath + "/" + fileName + "_fill.html";
File.Delete(newFileFullPath);
using (var htmlFileW = File.OpenWrite(newFileFullPath)) {
htmlDoc.Save(htmlFileW);
}
return(newFileFullPath);
}
private static void TranformHtml(DocumentBuildContext context, string transformed, string relativeModelPath, string outputPath)
{
// Update HREF and XREF
var internalXref = context.XRefSpecMap;
var externalXref = context.ExternalXRefSpec;
HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(transformed);
var xrefExceptions = new List<CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref/@href");
if (xrefNodes != null)
{
foreach(var xref in xrefNodes)
{
try
{
UpdateXref(xref, internalXref, externalXref, Language);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context.FileMap, relativeModelPath);
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context.FileMap, relativeModelPath);
}
}
// Save with extension changed
var subDirectory = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(subDirectory) && !Directory.Exists(subDirectory)) Directory.CreateDirectory(subDirectory);
html.Save(outputPath, Encoding.UTF8);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
public string HtmltoXml (string html) //throws XmlException
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument ();
doc.LoadHtml (html.Trim (new char[] { '\r', '\n', ' ' })); // bug in HtmlAgilityPack
StringBuilder fixedxml = new StringBuilder ();
StringWriter sw = new StringWriter (fixedxml);
StringBuilder tempxml = new StringBuilder ();
StringWriter tsw = new StringWriter (tempxml);
doc.OptionOutputAsXml = true;
doc.Save (tsw);
// fix style attribute
// the reason is that style attribute name-value pairs come in different order
// in .NET and GH
// Here I will sort the values of style attribute
XmlDocument tempDoc = new XmlDocument ();
tempDoc.LoadXml (tempxml.ToString ());
XmlNodeList allNodes = tempDoc.SelectNodes ("//*");
foreach (XmlNode n in allNodes) {
if (n.Attributes["style"] != null) {
string att = n.Attributes["style"].Value;
string[] style = att.Trim (new char[] { ' ', ';' }).Split (';');
for (int styleIndex = 0; styleIndex < style.Length; styleIndex++) {
style[styleIndex] = FixStyleNameValue (style[styleIndex]);
}
Array.Sort (style);
n.Attributes["style"].Value = string.Join (";", style);
}
}
tempDoc.Save (sw);
return fixedxml.ToString ();
}
/// <summary>
/// Load HTML DOM from given <paramref name="stream"/>.
/// </summary>
private object loadHTML(TextReader stream)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
// setup HTML parser
htmlDoc.OptionOutputAsXml = true;
//htmlDoc.OptionOutputOriginalCase = true; // NOTE: we need lower-cased names because of XPath queries
//htmlDoc.OptionFixNestedTags = true;
htmlDoc.OptionCheckSyntax = false;
htmlDoc.OptionWriteEmptyNodes = true;
// load HTML (from string or a stream)
htmlDoc.Load(stream);
CheckHtmlErrors(htmlDoc);
// save to string as XML
using (StringWriter sw = new StringWriter())
{
htmlDoc.Save(sw);
// load as XML
try
{
return loadXML(this, sw.ToString(), 0);
}
finally
{
this._isHtmlDocument = true;
}
}
}
private void TranformHtml(DocumentBuildContext context, string transformed, string relativeModelPath, string outputPath)
{
// Update HREF and XREF
HtmlAgilityPack.HtmlDocument html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(transformed);
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
foreach (var link in srcNodes)
{
UpdateSrc(link, context.FileMap, s => UpdateFilePath(s, relativeModelPath));
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
foreach (var link in hrefNodes)
{
// xref is generated by docfx, and is lower-cased
if (link.Name == "xref")
{
UpdateXref(link, context.XRefSpecMap, context.ExternalXRefSpec, s => UpdateFilePath(s, relativeModelPath), Language);
}
else
{
UpdateHref(link, context.FileMap, s => UpdateFilePath(s, relativeModelPath));
}
}
// Save with extension changed
var subDirectory = Path.GetDirectoryName(outputPath);
if (!string.IsNullOrEmpty(subDirectory) && !Directory.Exists(subDirectory)) Directory.CreateDirectory(subDirectory);
html.Save(outputPath, Encoding.UTF8);
}
/// <summary>
/// Load HTML DOM from given <paramref name="stream"/>.
/// </summary>
private object loadHTML(TextReader stream, string filename)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
// setup HTML parser
htmlDoc.OptionOutputAsXml = true;
//htmlDoc.OptionOutputOriginalCase = true; // NOTE: we need lower-cased names because of XPath queries
//htmlDoc.OptionFixNestedTags = true;
htmlDoc.OptionCheckSyntax = false;
htmlDoc.OptionUseIdAttribute = false; // only needed when XPath navigator is used on htmlDoc
htmlDoc.OptionWriteEmptyNodes = true;
// load HTML (from string or a stream)
htmlDoc.Load(stream);
CheckHtmlErrors(htmlDoc, filename);
// save to string as XML
using (StringWriter sw = new StringWriter())
{
htmlDoc.Save(sw);
// load as XML
return this.loadXMLInternal(sw.ToString(), 0, true);
}
}
public async Task LoadPost() {
if (Status == NetworkStatus.NotReachable) {//true ||
ShowErrorMessage (Settings.MSG_NETWORK_NOT_REACHABLE);
return;
}
IsLoading = true;
RequestPost req = new RequestPost();
req.Id = Post.Id;
ResponsePost detailPost;
try{
detailPost = await Service.GetPost (req);
Post = detailPost.Post;
if (Post.Kento_vote != null) {
IsLikedThisPost = Post.Kento_vote.Vote_status == 1;
Like_count = Post.Kento_vote.Vote_up_total;
}
Title = Post.FirstCategoryName;//category name - home post
} catch (Exception e){
Debug.WriteLine (e);
ShowErrorMessage (Settings.MSG_NETWORK_COMMON, e);
return;
}
var doc = new HtmlAgilityPack.HtmlDocument();
//remove ulike info
try {
doc.LoadHtml(detailPost.Post.Content);
foreach(var item in doc.DocumentNode.ChildNodes)
{
if (item.Id.StartsWith ("kento-vote")) {
item.InnerHtml = string.Empty;
Debug.WriteLine ("Empty ULike Info: " + item.OuterHtml);
}
//remove follow & unfollow content
if (item.Attributes["class"] != null && item.Attributes["class"].Value.StartsWith("wpw-fp-follow-post-wrapper"))
item.InnerHtml = string.Empty;
}
var stringBuilder = new System.Text.StringBuilder();
doc.Save (new System.IO.StringWriter(stringBuilder));
Post.Content = stringBuilder.ToString();
} catch (Exception e) {
ShowErrorMessage (Settings.MSG_NETWORK_COMMON, e);
}
//Repair comment content to plain text
try {
Comment_count = Post.Comment_count;
foreach (Comment comment in Post.Comments) {
doc.LoadHtml(comment.Content);
foreach(var item in doc.DocumentNode.ChildNodes)// "//div" is a xpath which means select div nodes that are anywhere in the html
{
if (item.Id.StartsWith ("wp-ulike-comment-")) {
item.InnerHtml = string.Empty;
Debug.WriteLine (item.OuterHtml);
}
}
comment.Content = System.Net.WebUtility.HtmlDecode (doc.DocumentNode.InnerText);
Comments.Add (comment);
}
} catch (Exception e) {
Debug.WriteLine ("[CommentPage-LoadComments] {0}", e);
}
string featured_img = Post.FullImage; //Post.IconSource==null ? "" : (Post.Thumbnail_images.Full.Url ?? Post.IconSource) ;
string timeAgo = new TimeAgoValueConverter ().Convert (Post.Date, null, null, null).ToString();
// Html = NewsTemplates.DETAIL_TEMPLATE_HEADER + String.Format(NewsTemplates.DETAIL_TEMPLATE_BODY, Post.Title, timeAgo, featured_img ,Post.Content, Post.FirstCategoryName);
Html = String.Format(NewsTemplates.DETAIL_TEMPLATE, Post.Title, timeAgo, featured_img ,Post.Content, Post.FirstCategoryName);
//test video player
//Html = string.Format(NewsTemplates.DETAIL_VIDEO_PLAYER, "http://techslides.com/demos/sample-videos/small.mp4");
RaisePropertyChanged ("Comment_count");
RaisePropertyChanged ("Comments");
IsLoading = false;
}
/// <summary>
/// When overridden in a derived class, this validates both the request and response.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
public virtual void Validate(Object sender, ValidationEventArgs e)
{
_context.Outcome = WebTestOutcome.NotExecuted;
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(((WebTestRequest)e.WebTestItem).HttpResponseBody);
document.OptionOutputAsXml = true;
using (StringWriter writer = new StringWriter())
{
document.Save(writer);
_document = XDocument.Parse(writer.GetStringBuilder().ToString());
}
}
private string HtmltoXml(string html)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
StringBuilder fixedxml = new StringBuilder();
StringWriter sw = new StringWriter(fixedxml);
try
{
StringBuilder tempxml = new StringBuilder();
StringWriter tsw = new StringWriter(tempxml);
doc.OptionOutputAsXml = true;
doc.Save(tsw);
// fix style attribute
// the reason is that style attribute name-value pairs come in different order
// in .NET and GH
// Here I will sort the values of style attribute
XmlDocument tempDoc = new XmlDocument();
tempDoc.LoadXml(tempxml.ToString());
XmlNodeList allNodes = tempDoc.SelectNodes("//*");
foreach (XmlNode n in allNodes)
{
if (n.Attributes["style"] != null)
{
string att = n.Attributes["style"].Value;
string [] style = att.Trim(new char[]{' ', ';'}).Split(';');
for (int styleIndex=0; styleIndex<style.Length; styleIndex++)
{
style[styleIndex] = FixStyleNameValue(style[styleIndex]);
}
Array.Sort(style);
n.Attributes["style"].Value = string.Join(";", style);
}
}
tempDoc.Save(sw);
}
catch (Exception)
{
Console.WriteLine("Error parsing html response...");
Console.WriteLine("Test case aborted");
return "<TestCaseAborted></TestCaseAborted>";
}
return fixedxml.ToString();
}
private static void TransformHtml(IDocumentBuildContext context, string html, string relativeModelPath, StreamWriter outputWriter)
{
// Update href and xref
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(html);
var xrefExceptions = TransformHtmlCore(context, relativeModelPath, document);
document.Save(outputWriter);
if (xrefExceptions.Count > 0)
{
throw new AggregateException(xrefExceptions);
}
}
public void editPost(string postid, string title, string body)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(body);
HtmlAgilityPack.HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//img");
if (nodes != null)
foreach (HtmlAgilityPack.HtmlNode node in nodes)
{
//获取html源地址
HtmlAgilityPack.HtmlAttribute attr = node.Attributes["src"];
JoeBlogs.MediaObject mediaObj = new JoeBlogs.MediaObject();
mediaObj.Name = "test";
mediaObj.Type = "image/jpeg";
FileStream file = new FileStream(attr.Value, FileMode.Open);
long size = file.Length;
byte[] stream = new byte[size];
file.Read(stream, 0, (int)size);
file.Close();
mediaObj.Bits = stream;
JoeBlogs.MediaObjectInfo mediainfo = metablog.NewMediaObject(mediaObj);
attr.Value = mediainfo.Url;
}
JoeBlogs.Post post = new JoeBlogs.Post();
post.Title = title;
StringWriter writer = new StringWriter();
if (nodes != null)
{
doc.Save(writer);
post.Body = writer.ToString();
}
else
{
post.Body = body;
}
metablog.EditPost(postid, post, true);
}
/// <summary>
/// Generates Image from the specified Website
/// </summary>
protected void Preview_Gen_Click(object sender, EventArgs e)
{
string url = txtUrl.Text;
uri = CreateUri(url);
string domain = uri.Host;
domain = Regex.Replace(domain, @"^(?:http(?:s)?://)?(?:www(?:[0-9]+)?\.)?", string.Empty, RegexOptions.IgnoreCase);
var date = DateTime.Now.Year.ToString() + DateTime.Now.Month.ToString() + DateTime.Now.Day.ToString() + "_" + DateTime.Now.Hour.ToString() + DateTime.Now.Minute.ToString() + DateTime.Now.Second.ToString();
var filename = domain + "_" + date + ".html";
/**********************************
* Method WebClient.DownloadString
*
***********************************/
domain = "http://" + domain;
string webpage = GetHTML(url);
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(webpage);
var head = doc.DocumentNode.SelectSingleNode("//head");
var baseTag = HtmlAgilityPack.HtmlNode.CreateNode("<base href=\"" + domain + "\">");
var fontAwesomeRes = HtmlAgilityPack.HtmlNode.CreateNode("<link rel=\"stylesheet\" href=\"https://maxcdn.bootstrapcdn.com/font-awesome/4.4.0/css/font-awesome.min.css\">");
head.PrependChild(baseTag);
head.AppendChild(fontAwesomeRes);
doc = AssignAsbsoluteUri(doc, "//img", "src");
doc = AssignAsbsoluteUri(doc, "//link", "href");
string path = Server.MapPath("Content/Images/Screenshots");
var filepath = path + "\\" + filename;
doc.Save(filepath);//webpageurlNext
iframeLoader.Src = "Content/Images/Screenshots/" + filename;
iframeLoader.Visible = true;
iframeVisible = true;
//StatusText.Text = "Wait until this message disappear";
//ContentLoaded.Visible = true;
}
private static String ParseSaveAndFixImages(string contents, string dirPath)
{
contents = System.Web.HttpUtility.HtmlDecode(contents);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(new StringReader(contents));
var nav = doc.CreateNavigator();
var strExpression = "//img";
HtmlAgilityPack.HtmlNodeCollection imgTags = doc.DocumentNode.SelectNodes(strExpression);
if (imgTags != null)
{
foreach (HtmlAgilityPack.HtmlNode tag in imgTags)
{
if (tag.Attributes["src"] != null)
{
String imgPath = tag.Attributes["src"].Value;
tag.Attributes["src"].Value = GetAndSaveImage(imgPath, dirPath);
}
}
}
string finalContents = null;
using (StringWriter sw = new StringWriter())
{
doc.Save(sw);
finalContents = sw.ToString();
}
return finalContents;
}
string[] ExtractFormAndHiddenControls (Response response)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument ();
htmlDoc.LoadHtml (response.Body);
var tempxml = new StringBuilder ();
var tsw = new StringWriter (tempxml);
htmlDoc.OptionOutputAsXml = true;
htmlDoc.Save (tsw);
var doc = new XmlDocument ();
doc.LoadXml (tempxml.ToString ());
XmlNamespaceManager nsmgr = new XmlNamespaceManager (doc.NameTable);
nsmgr.AddNamespace ("html", HTML_NAMESPACE);
XmlNode formNode = doc.SelectSingleNode ("//html:form", nsmgr);
if (formNode == null)
throw new ArgumentException ("Form was not found in document: " + response.Body);
string actionUrl = formNode.Attributes ["action"].Value;
XmlNode method = formNode.Attributes ["method"];
var data = new List <string> ();
string name, value;
foreach (XmlNode inputNode in doc.SelectNodes ("//html:input[@type='hidden']", nsmgr)) {
name = inputNode.Attributes["name"].Value;
if (String.IsNullOrEmpty (name))
continue;
XmlAttribute attr = inputNode.Attributes["value"];
if (attr != null)
value = attr.Value;
else
value = String.Empty;
data.Add (name);
data.Add (value);
}
return data.ToArray ();
}
public Transaction[] GetTransactionsFromAccountHistory(Stream stream)
{
StreamReader reader = new StreamReader(stream);
// From here: http://stackoverflow.com/questions/12822680/xmldocument-failed-to-load-xhtml-string-because-of-error-reference-to-undeclare
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(reader.ReadToEnd());
doc.OptionOutputAsXml = true;
StringWriter writer = new StringWriter();
doc.Save(writer);
XDocument root = XDocument.Load(new StringReader(writer.ToString()));
XPathNavigator navigator = root.CreateNavigator();
List<Transaction> resultTransactions = new List<Transaction>();
// Pullled from http://stackoverflow.com/questions/2524804/how-do-i-use-xpath-with-a-default-namespace-with-no-prefix
var nameTable = new NameTable();
var nsMgr = new XmlNamespaceManager(nameTable);
nsMgr.AddNamespace("xhtml", "http://www.w3.org/1999/xhtml");
var transactionTable = navigator.SelectSingleNode("span/body/table[1]/tr[4]/td/table[2]");
var transactions = transactionTable.Select("tr");
bool first = true;
foreach (XPathNavigator t in transactions)
{
if (first)
{
first = false;
continue;
}
var cellSelect = t.Select("td");
var cells = new List<string>();
cells.Add(cellSelect.Current.Value.Trim());
while (cellSelect.MoveNext())
{
cells.Add(cellSelect.Current.Value.Trim());
}
if (cells.Count == 8 && (accountsToFilter.Contains(cells[4])))
{
TransactionType tType = TransactionType.Buy;
if (cells[3].Contains("Short-Term Earnings") || cells[3] == "Dividend" || cells[3] == "Earnings")
{
tType = TransactionType.ReinvestDividend;
}
else if (cells[3] == "Contributions")
{
tType = TransactionType.Buy;
}
else if (cells[3] == "ADP/ACP Refunds")
{
tType = TransactionType.Sell;
}
else
{
continue;
}
resultTransactions.Add(new Transaction()
{
FundName = cells[2],
Price = double.Parse(cells[5], System.Globalization.NumberStyles.Any),
Amount = double.Parse(cells[6]),
Total = double.Parse(cells[7], System.Globalization.NumberStyles.Any),
Date = DateTime.Parse(cells[1]),
Type = tType
});
}
}
return resultTransactions.ToArray();
}
/// <summary>
/// Parse HTML file and find all local contents to download.
/// </summary>
/// <param name="pnum">Problem number</param>
/// <param name="replace">True, if you want to replace old files.</param>
/// <returns>List of files to download</returns>
public static List<DownloadTask> ProcessHtmlContent(long pnum, bool replace)
{
try
{
string external = string.Format("http://uva.onlinejudge.org/external/{0}/", pnum / 100);
string filepath = LocalDirectory.GetProblemHtml(pnum);
if (!File.Exists(filepath)) return new List<DownloadTask>();
List<string> urls = new List<string>();
List<DownloadTask> tasks = new List<DownloadTask>();
HtmlAgilityPack.HtmlDocument htdoc = new HtmlAgilityPack.HtmlDocument();
htdoc.Load(filepath);
GetAllImageFiles(htdoc.DocumentNode, urls);
htdoc.Save(filepath);
foreach (string str in urls)
{
string url = str.StartsWith("./") ? str.Remove(0, 2) : str;
while (url.StartsWith("/")) url = url.Remove(0, 1);
string file = url.Replace('/', Path.DirectorySeparatorChar);
file = LocalDirectory.GetProblemContent(pnum, file);
if (replace || LocalDirectory.GetFileSize(file) < 10)
{
tasks.Add(new DownloadTask(external + url, file, pnum));
}
}
urls.Clear();
return tasks;
}
catch (Exception ex)
{
Logger.Add(ex.Message, "Internet");
return new List<DownloadTask>();
}
}
//---------------------------
public int NStartProcessingAuthentification(int jobID)
{
int status = 0;
//Пишем последнюю страницу в файл
//Создается каталог для контроля считанной странички
/*DirectoryInfo _directoryInfo = new DirectoryInfo(Path.GetDirectoryName(Application.ExecutablePath) + "\\OUT");
if (!_directoryInfo.Exists)
{
_directoryInfo.Create();
}*/
HTTPClient client = new HTTPClient(_dbConnectionString, "MySession", _ipProxy, _portProxy, _streamNumber); ;
bool nextPage = true;
string sHTML = "";
string address = "";
string message = "";
HtmlDocument document = new HtmlDocument();
if (!_webAuthentification) // Аунтификация
{
HttpWebResponse httpWebResponse = client.Request(_webPreAuthentificationAddress); //("http://yarsk24.ru");
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
httpWebResponse.Close();
httpWebResponse = client.Request_Post(_webAuthentificationAddress, _webConnectionString); //("http://yarsk24.ru/index/sub/", "user=zero2001&password=2001&rem=1&a=2&ajax=1&rnd=038&_tp_=xml");
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Stream stream = httpWebResponse.GetResponseStream();
using (StreamReader reader = new StreamReader(stream, this.Encoding)) //System.Text.Encoding.GetEncoding(1251))) / System.Text.Encoding.UTF8
{
sHTML = reader.ReadToEnd();
}
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
httpWebResponse.Close();
_webAuthentification = true;
}
else
{
address = _webAuthentificationAddress;
message = httpWebResponse.StatusCode.ToString();
status = 8;
}
}
else
{
address = _webAuthentificationAddress;
message = httpWebResponse.StatusCode.ToString();
status = 7;
}
}
else
{
address = _webPreAuthentificationAddress;
message = httpWebResponse.StatusCode.ToString();
status = 6;
}
}
if (status != 0)
{
DataDBService ddbs = new DataDBService();
ddbs.WriteErrorMessage(_dbConnectionString, _streamNumber, !string.IsNullOrEmpty(address) ? address : _webAuthentificationAddress, "Ошибка при прохождении аутентификации: " + message);
}
if (_webAuthentification) // Запрос страницы с количеством объявлений
{
HttpWebResponse httpWebResponse = client.Request(_startAddress);
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Stream stream = httpWebResponse.GetResponseStream();
using (StreamReader reader = new StreamReader(stream, this.Encoding))
{
sHTML = reader.ReadToEnd();
document = new HtmlDocument();
document.LoadHtml(sHTML);
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
httpWebResponse.Close();
document.Save(_directoryInfo + "\\LastPage" + _streamNumber.ToString() + ".html");
}
else
{
status = 4;
nextPage = false;
}
}
}
else
{
status = 10;
nextPage = false;
}
}
if (_webAuthentification && nextPage) // Цикл по страницам
{
int countPagesTotal = GetPageCount(document, _startAddress);
if (countPagesTotal > 0)
{
for (int i = 0; i < countPagesTotal; i++)
{
status = 1;
if (i + 1 >= _firstPage)
{
List<bool> valids;
List<string> addresses;
List<string> stickers;
//Получаем страницу для обработки
string currentAddress = GetLoadedPage(_startAddress, i);
if (_lastAddress != currentAddress)
{
_lastAddress = currentAddress;
// Запрос страницы с количеством объявлений
document = null;
HttpWebResponse httpWebResponse = client.Request(_lastAddress);
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Stream stream = httpWebResponse.GetResponseStream();
using (StreamReader reader = new StreamReader(stream, this.Encoding))
{
sHTML = reader.ReadToEnd();
document = new HtmlDocument();
document.LoadHtml(sHTML);
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
httpWebResponse.Close();
else { }
}
}
if (document != null)
{
document.Save(_directoryInfo + "\\LastPage" + _streamNumber.ToString() + ".html");
//Получаем список интересных объявлений
valids = GetValidObjects(document);
//Получаем список ссылок на странице
addresses = GetAddressList(document, valids);
//Получаем список стикеров на странице
stickers = GetStickers(document, valids);
//------------------------------------
if (valids != null && (addresses != null || stickers != null))
{
System.Threading.Thread.Sleep(new Random().Next(1000, 3000));
string hostName = Dns.GetHostName();
if (InsertIntoAddressesUpload(addresses, stickers, jobID, hostName))
{
bool bb = false;
while (!bb)
{
if (IUDAdressesMask(hostName, jobID))
{
nextPage = NSetJobState(jobID, i + 1, countPagesTotal, addresses.Count, jobID, document);
if (!nextPage)
{
DataDBExtention ddbe = new DataDBExtention();
ddbe.NSetStatistics(jobID, _dbConnectionString, hostName, _streamNumber);
}
//if (valids == null || addresses == null || stickers == null)
//{
// status = 3;
// break;
//}
bb = true;
}
else
System.Threading.Thread.Sleep(new Random().Next(100, 300));
//else
//{
// nextPage = NSetJobState(jobID, i + 1, countPagesTotal, 0, jobID, document);
// //если что-то, то все равно идем дальше
// ////может брейк постаивть. например, если таймаут вышел и запрос вывалился
// ////или оставить как есть и припоследующих запусках оно само заберет неотданные ссылки
//}
}
}
//тут надо отписываться, что он спарсил и сколько ссылок
}
else
{
status = 3;
break;
}
// Досрочный выход из цикла по страницам
// Достигнуто заданное количество страниц
if (i + 1 >= _firstPage - 1 + _pageCount || !nextPage)
{
break;
}
// Исчерпано время (в часах)
DateTime dateTime = DateTime.Now;
if ((dateTime.Year - _dateTimeBegin.Year) * 365 * 24
+ (dateTime.DayOfYear - _dateTimeBegin.DayOfYear) * 24
+ dateTime.Hour - _dateTimeBegin.Hour
> _timeInHours)
{
status = 2;
break;
}
//----------------------------------------
// //Обрабатываем каждую ссылку и выбираем необходимую информацию
// List<DirtyApartments> apartments = CardProcessingAuthentification(addresses, stickers, client);
// if (apartments != null)
// {
// //Делегат
// if (collectedApartmentsDel != null)
// nextPage = collectedApartmentsDel(apartments, i + 1, countPagesTotal, _directoryInfo + "\\LastPage" + _streamNumber.ToString() + ".html");
// // Досрочный выход из цикла по страницам
// // Достигнуто заданное количество страниц
// if (i + 1 >= _firstPage - 1 + _pageCount || !nextPage)
// {
// break;
// }
// // Исчерпано время (в часах)
// DateTime dateTime = DateTime.Now;
// if ((dateTime.Year - _dateTimeBegin.Year) * 365 * 24
// + (dateTime.DayOfYear - _dateTimeBegin.DayOfYear) * 24
// + dateTime.Hour - _dateTimeBegin.Hour
// > _timeInHours)
// {
// status = 2;
// break;
// }
// if (valids == null || addresses == null || stickers == null)
// {
// status = 3;
// break;
// }
// }
// else
// {
// status = 16;
// break;
// }
}
else
{
status = 4;
break;
}
}
else
{
status = 5;
break;
}
}
}
}
else
status = 9;
}
else { }
return status;
}