protected override string GetPosition(HtmlAgilityPack.HtmlNode div)
{
return div.Descendants("a").Where(
d => d.Attributes.Contains("class") &&
d.Attributes["class"].Value.Contains("rua-b-vacancy-title") || d.Attributes["class"].Value.Contains("jqKeywordHighlight")
).Select(d => d.InnerText).First();
}
public static XrefDetails From(HtmlAgilityPack.HtmlNode node)
{
if (node.Name != "xref") throw new NotSupportedException("Only xref node is supported!");
var xref = new XrefDetails();
xref.Uid = node.GetAttributeValue("href", null);
var overrideName = node.InnerText;
if (!string.IsNullOrEmpty(overrideName))
{
xref.AnchorDisplayName = overrideName;
xref.PlainTextDisplayName = overrideName;
}
else
{
// If name | fullName exists, use the one from xref because spec name is different from name for generic types
// e.g. return type: IEnumerable<T>, spec name should be IEnumerable
xref.AnchorDisplayName = node.GetAttributeValue("name", null);
xref.PlainTextDisplayName = node.GetAttributeValue("fullName", null);
}
xref.Title = node.GetAttributeValue("title", null);
xref.Raw = node.GetAttributeValue("data-raw", null);
xref.ThrowIfNotResolved = node.GetAttributeValue("data-throw-if-not-resolved", false);
return xref;
}
public static ResultType_enum CheckDocResult(HtmlAgilityPack.HtmlDocument doc, out string message)
{
try
{
message = "";
if (doc == null)
{
message = "Не удается подключиться к сети";
return ResultType_enum.ErrorNetwork;
}
//Не удается отобразить эту страницу
if (doc.DocumentNode.InnerText.IndexOf("Не удается отобразить эту страницу") > -1)
{
message = "Не удается подключиться к сети";
return ResultType_enum.ErrorNetwork;
}
if (doc.DocumentNode.InnerText.IndexOf("Ведутся регламентные работы") > -1)
{
message = "Ведутся регламентские работы";
return ResultType_enum.ReglamentaWork;
}
return ResultType_enum.Done;
}
catch (Exception ex)
{
message = ex.Message + '\n' + ex.StackTrace;
return ResultType_enum.Error;
}
}
protected override string GetVacancyUrl(HtmlAgilityPack.HtmlNode div)
{
var vacancyHref = div.Descendants("a").Where(
d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("vacancy-title"))
.Select(d => d.Attributes["href"].Value).SingleOrDefault();
return BaseUrl + vacancyHref;
}
public KeyValuePair<PartsPage, IEnumerable<string>> RunActions(string content, string url, HtmlAgilityPack.HtmlDocument doc)
{
Content = content;
Url = url;
Doc = doc;
string res = null;
var count = 3;
if (IsPermessionEmail())
{
while (string.IsNullOrEmpty(res) || (res.Equals("error") && count > 0))
{
LoadCaptcha();
res = SendMessage();
count--;
}
}
else
res = "No Permission";
var typeResult = PartsPage.MessageSend;
if (res != null)
{
return new KeyValuePair<PartsPage, IEnumerable<string>>(typeResult, new List<string>()
{
res
});
}
else
return new KeyValuePair<PartsPage, IEnumerable<string>>(typeResult, null);
}
private static string AttrValue(HtmlAgilityPack.HtmlNode htmlNode, string name)
{
var attr = htmlNode.Attributes[name];
return attr != null
? attr.Value
: null;
}
/// <summary>
///
/// </summary>
/// <param name="node"></param>
/// <param name="element"></param>
/// <param name="attribute"></param>
/// <param name="valuePortion"></param>
/// <returns></returns>
public static HtmlAgilityPack.HtmlNode GetNode(HtmlAgilityPack.HtmlNode node,
string element,
string attribute,
string valuePortion,
bool tryNull = false)
{
if(tryNull)
{
try
{
var _nodes = node.Descendants(element)
.Where(d => d.Attributes.Contains(attribute) && d.Attributes[attribute].Value.Contains(valuePortion));
if(_nodes != null && _nodes.Count() > 0)
{
var _node = _nodes.ToArray()[0];//.SingleOrDefault();
return _node;
}
}
catch { }
}
else
{
var _nodes = node.Descendants(element)
.Where(d => d.Attributes.Contains(attribute) && d.Attributes[attribute].Value.Contains(valuePortion));
if(_nodes != null && _nodes.Count() > 0)
{
var _node = _nodes.ToArray()[0];//.SingleOrDefault();
return _node;
}
}
return null;
}
public List<ListPicker> ConvertSelectFields(HtmlAgilityPack.HtmlNodeCollection nodes)
{
if (nodes == null)
return null;
var ret = new List<ListPicker>();
foreach (var item in nodes)
{
ListPicker foo = new ListPicker();
foo.Name = item.Attributes["Name"].Value;
foo.HorizontalAlignment = HorizontalAlignment.Stretch;
var options = item.SelectNodes(".//option");
var listboxitems = new List<KeyValuePair<string, string>>();
foreach (var option in options)
{
var bar = new KeyValuePair<string, string>(HttpUtility.HtmlDecode(option.NextSibling.OuterHtml), option.Attributes["value"].Value);
listboxitems.Add(bar);
}
foo.ItemsSource = listboxitems;
foo.FullModeItemTemplate = (DataTemplate)Application.Current.Resources["SearchListBoxItemTemplate"];
foo.ItemTemplate = (DataTemplate)Application.Current.Resources["SearchListBoxItemTemplateSelected"];
foo.BorderThickness = new Thickness(2.0);
foo.SelectedIndex = 0;
ret.Add(foo);
}
return ret;
}
public List<EDAdvancedTextbox> ConvertInputFields(HtmlAgilityPack.HtmlNodeCollection nodes)
{
if (nodes == null)
return null;
var ret = new List<EDAdvancedTextbox>();
foreach (var item in nodes)
{
EDAdvancedTextbox foo = new EDAdvancedTextbox();
foo.Name = item.Attributes["Name"].Value;
try
{
foo.DefaultText = item.Attributes["Title"].Value;
}
catch (NullReferenceException)
{
foo.DefaultText = foo.Name;
}
foo.HorizontalAlignment = HorizontalAlignment.Stretch;
foo.BorderBrush = new SolidColorBrush(Colors.Black);
foo.BorderThickness = new Thickness(2.0);
ret.Add(foo);
}
return ret;
}
private void ValidateTable(ValidationEventArgs e, HtmlAgilityPack.HtmlNode table)
{
var columns = table.SelectNodes("//th");
if (columns == null || columns.Count == 0)
{
Fail(e, "Could not locate any columns in the table!");
}
else
{
int columnIndex = FindColumnIndexByName(columns, ColumnName);
if (columnIndex == -1)
{
Fail(e, String.Format("Could not find a column named '{0}'!", ColumnName));
}
else
{
bool foundTheValue = DoesValueExistInColumn(table, columnIndex, ExpectedValue);
if (foundTheValue == true)
{
Pass(e, "Found the column value!");
}
else
{
Fail(e, String.Format("Could not find a cell with the value '{0}'", ExpectedValue));
}
}
}
}
public static void AddColumnRowToTable(HtmlAgilityPack.HtmlNode table, string s1, string s2, string s3)
{
HtmlAttribute attr = table.OwnerDocument.CreateAttribute("style", "border:1px solid black;border-collapse:collapse;");
HtmlNode row = table.OwnerDocument.CreateElement("tr");
row.Attributes.Add(table.OwnerDocument.CreateAttribute("valign", "top"));
HtmlNode td1 = table.OwnerDocument.CreateElement("td");
HtmlNode td2 = table.OwnerDocument.CreateElement("td");
HtmlAttribute rightJustify = table.OwnerDocument.CreateAttribute("align", "right");
row.Attributes.Add(attr);
td1.Attributes.Add(attr);
td2.Attributes.Add(attr);
td1.Attributes.Add(rightJustify);
td2.Attributes.Add(rightJustify);
td1.InnerHtml = s1;
td2.InnerHtml = s2;
row.ChildNodes.Add(td1);
row.ChildNodes.Add(td2);
if (string.IsNullOrEmpty(s3)==false)
{
HtmlNode td3 = table.OwnerDocument.CreateElement("td");
td3.Attributes.Add(attr);
td3.Attributes.Add(rightJustify);
td3.InnerHtml = s3;
row.ChildNodes.Add(td3);
}
table.ChildNodes.Add(row);
}
private static Gear ParseGear(HtmlNode gearNode, ProfileParseConfig config)
{
var gear = new Gear();
var gearArray = new[] { gearNode };
var gearImage = config.GearImageGetter(gearArray).Single();
var gearImageUris = ParseImage(gearImage, config);
gear.ImageUri = gearImageUris.Item1;
gear.RetinaImageUri = gearImageUris.Item2;
var gearPowerMainSvg = config.GearPowerMainSvgGetter(gearArray).Single();
gear.GearPowerMainSvgUri = ParseImageUriFromStyle(gearPowerMainSvg, config);
var gearPowerSub = config.GearPowerSubGetter(gearArray).Single();
var gearPowerSubArray = new[] { gearPowerSub };
var gearPowerSub1Svg = config.GearPowerSub1Getter(gearPowerSubArray).Single();
gear.GearPowerSub1SvgUri = ParseImageUriFromStyle(gearPowerSub1Svg, config);
try
{
var gearPowerSub2Svg = config.GearPowerSub2Getter(gearPowerSubArray).Single();
gear.GearPowerSub2SvgUri = ParseImageUriFromStyle(gearPowerSub2Svg, config);
var gearPowerSub3Svg = config.GearPowerSub3Getter(gearPowerSubArray).Single();
gear.GearPowerSub3SvgUri = ParseImageUriFromStyle(gearPowerSub3Svg, config);
}
catch (ArgumentOutOfRangeException) { }
return gear;
}
public int[] GetLinesFromCommand(HtmlAgilityPack.HtmlNode htmlNode, string commandName)
{
if (htmlNode == null)
{
return null;
}
var text = htmlNode.InnerText;
text = text.Replace("<!--", string.Empty).Replace("-->", string.Empty);
var command = text.Split(';').Where(s => s.Trim().StartsWith(commandName)).FirstOrDefault();
if (command == null)
{
return null;
}
try
{
var arguments = command.Split(':')[1];
var lineRanges = arguments.Split(',');
var lineSet = new System.Collections.Generic.SortedSet<int>();
foreach (var range in lineRanges)
{
this.ParseRange(lineSet, range);
}
return lineSet.ToArray();
}
catch
{
return null;
}
}
public override string GrabStoryVariables(HtmlAgilityPack.HtmlDocument doc)
{
string[] addressParts = storyURL.Split('/');
try
{
storyChapters = (ushort)(doc.DocumentNode.SelectNodes("//div[@id='chapterList']//li").Count() - 1);
storyTitle = addressParts[3].Replace("_", " ");
try { authorName = doc.DocumentNode.SelectSingleNode("//h2[@class='textCenter']/a").InnerText; }
catch { authorName = doc.DocumentNode.SelectSingleNode("//h1[@class='textCenter']/a").InnerText; }
authorLink = "<a href=\"http://" + addressParts[2] + "\">" + authorName + "</a>";
HtmlNodeCollection summaryNodes = doc.DocumentNode.SelectNodes("//div[@id='chapterList']//li[1]/p");
foreach (HtmlNode node in summaryNodes)
{
if (!(node.InnerText.Contains("Word count:") || node.InnerText.Contains("Also available as:")))
storySummary += node.InnerText;
}
string[] splitHeader = doc.DocumentNode.SelectSingleNode("//div[@id='chapterList']//li[1]/p[1]").InnerText.Split(' ');
//Grabs individual variables from story, then splits header up for further parsing.
for (int i = 0; i < splitHeader.Length; i++)
{//Looks for keywords in header, and if found, assigns them and following section to appropriate story variable.
if (splitHeader[i].Contains("Word") && splitHeader[i + 1].Contains("count"))
storyWords = "Words: " + splitHeader[i + 2];
else if (splitHeader[i].Contains("Status:"))
{
if (splitHeader[i + 1].Contains("Complete"))
storyStatus = "Complete";
else
storyStatus = "In Progress";
}
}
splitHeader = doc.DocumentNode.SelectSingleNode("//div[@id='chapterList']//li[2]").InnerText.Split(' ');
for (int i = 0; i < splitHeader.Length; i++)
if (splitHeader[i].Contains("Uploaded"))
storyPublished = "Published: " + splitHeader[i + 2] + " " + splitHeader[i + 3] + " " + splitHeader[i + 4];
splitHeader = doc.DocumentNode.SelectSingleNode("//div[@id='chapterList']//li[" + (storyChapters + 1) + "]").InnerText.Split(' ');
for (int i = 0; i < splitHeader.Length; i++)
if (splitHeader[i].Contains("Uploaded"))
storyUpdated = "Updated: " + splitHeader[i + 2] + " " + splitHeader[i + 3] + " " + splitHeader[i + 4];
for (int i = 2; i <= storyChapters + 1; i++)
{
addressParts = doc.DocumentNode.SelectSingleNode("//div[@id='chapterList']//li[" + i + "]/a[1]").Attributes["href"].Value.Split('/');
addressParts[2] = addressParts[2].Replace("__", " - ");
addressParts[2] = addressParts[2].Replace("_", " ");
chapterTitles.Add(addressParts[2]);
}
storyBody = "(//li[@class='chapterDisplay'])";
return "Story grab success.";
}
catch
{ return "Story grab failed! Please verify URL provided is for a story."; }
}
protected override string GetVacancyBody(HtmlAgilityPack.HtmlDocument htmlDocument)
{
var nodes = htmlDocument.DocumentNode.Descendants("div").Where( r => r.Attributes.Contains("class") && r.Attributes["class"].Value.Equals("description"));
var body = nodes.Aggregate("", (current, node) => current + node.InnerText);
return body;
}
/// <summary>
/// 执行所有 HTML 检查
/// </summary>
/// <param name="context"></param>
/// <returns>HTML 检查结果。</returns>
public static IList<ValidationResult> Validate(this IEnumerable<IValidation> validations, HtmlAgilityPack.HtmlDocument document, HtmlStaticizeContext context)
{
if (document == null)
{
throw new ArgumentNullException("document");
}
if (context == null)
{
throw new ArgumentNullException("context");
}
if (validations == null)
{
return null;
}
var validationResult = new List<ValidationResult>();
foreach (var vd in validations)
{
var errorMessage = vd.Validate(document, context);
if (errorMessage != null && errorMessage.Length > 0)
{
validationResult.Add(new ValidationResult
{
Uri = context.Uri,
ValidationType = vd.Type,
Name = vd.Name,
Message = errorMessage,
});
}
}
return validationResult;
}
protected override string[] GetSrcAttributes(HtmlAgilityPack.HtmlNodeCollection nodes)
{
var srcAttributes = (from node in nodes
where !String.IsNullOrWhiteSpace(node.GetAttributeValue("href", null))
select node.GetAttributeValue("href", null)).ToArray();
return srcAttributes;
}
public void addComponent(HtmlAgilityPack.HtmlNode row)
{
RowInterpreter interpreter = new RowInterpreter(row);
if (!interpreter.isCancelled())
{
if (interpreter.isNewCourse())
{
if (course != null && section != null)
{
course.addSection(section);
courseList.Add(course);
}
// new course
course = new Course();
course.addCourseID(interpreter.getCourseID());
course.addSectionCode(interpreter.getSectionCode());
course.addTitle(interpreter.getTitle());
// new section AND add it to course
section = new Section();
section.addSectionID(interpreter.getSectionID());
section.addTime(interpreter.getTime());
section.addlocation(interpreter.getLocation());
section.addInstructor(interpreter.getInstructor());
}
else
{
if (interpreter.hasSectionID())
{
course.addSection(section);
// new section
section = new Section();
section.addSectionID(interpreter.getSectionID());
section.addTime(interpreter.getTime());
section.addlocation(interpreter.getLocation());
section.addInstructor(interpreter.getInstructor());
}
else
{
// same section more info
section.addTime(interpreter.getTime());
section.addlocation(interpreter.getLocation());
section.addInstructor(interpreter.getInstructor());
}
}
}
else
{
/*
// cancelled (Implies old course ended)
if (course != null && section != null)
{
course.addSection(section);
courseList.Add(course);
}
*/
}
}
public override List<Index> GetIndex(HtmlAgilityPack.HtmlDocument html, string url, uint level, string path, object userData)
{
var books = new List<Index>();
var links = html.DocumentNode.SelectNodes("//body/div[@class='z']/div[@class='z']/a");
if (links == null || links.Count / 3 <= 0 || links.Count % 3 != 0)
return null;
for (int i = 0; i < links.Count / 3; ++i)
{
var j = i * 3;
var title = links[j].InnerText;
var target = links[j + 1].Attributes["href"].Value;
//var uri = new Uri(target, UriKind.RelativeOrAbsolute);
var cid = Path.GetFileNameWithoutExtension(target);
var index = new Index(level) { name = XTrim(title), url = GetTargetUrl(cid) };
books.Add(index);
File.WriteAllText(Path.ChangeExtension(Path.Combine(path, index.name), "url"), String.Format("[InternetShortcut]{0}URL={1}", Environment.NewLine, GetPlayUrl(cid)));
}
return books;
}
public static HtmlAgilityPack.HtmlNode CreateReportTableNode(HtmlAgilityPack.HtmlNode node)
{
HtmlNode table = node.OwnerDocument.CreateElement("table");
HtmlAttribute attr = table.OwnerDocument.CreateAttribute("style", "border:1px solid black;border-collapse:collapse;");
table.Attributes.Add(attr);
return table;
}
private bool EndOfRoster(HtmlAgilityPack.HtmlDocument htmlDoc)
{
var nothingFound = htmlDoc.DocumentNode.SelectNodes("//div[@id='roster']/table/tbody/tr[@class='no-results']");
if (nothingFound != null)
return true;
return false;
}
public string GetTextFromSiblings(HtmlAgilityPack.HtmlNode node, HtmlAgilityPack.HtmlNode parentNode, ref bool foundParent)
{
var ret = new StringBuilder();
if (node != null)
{
HtmlAgilityPack.HtmlNode sibling = null;
do
{
sibling = sibling != null ? sibling.PreviousSibling : node.PreviousSibling;
if (sibling == parentNode)
{
foundParent = true;
}
if (sibling != null && sibling != parentNode)
{
var siblingInnerText = sibling.InnerText;
if (!string.IsNullOrWhiteSpace(siblingInnerText))
{
var text = HtmlEntity.DeEntitize(siblingInnerText).Trim();
ret.Append(text);
ret.Append(" ");
}
}
}
while (sibling != null);
}
return ret.ToString().Trim();
}
//functions taking data and checking that exists from website
public string part1(HtmlAgilityPack.HtmlDocument htmlDoc, string newdata)
{
if(htmlDoc.DocumentNode.SelectNodes("//title") != null)
foreach (var title in htmlDoc.DocumentNode.SelectNodes("//title"))
{
string titles = title.InnerText;
newdata += titles;
}
if(htmlDoc.DocumentNode.SelectNodes("//h1") != null)
foreach (var h1 in htmlDoc.DocumentNode.SelectNodes("//h1"))
{
string h1s = h1.InnerText;
newdata += h1s;
}
if(htmlDoc.DocumentNode.SelectNodes("//h2") != null)
foreach (var h2 in htmlDoc.DocumentNode.SelectNodes("//h2"))
{
string h2s = h2.InnerText;
newdata += h2s;
}
return newdata;
}
public Property(HtmlAgilityPack.HtmlNode node, HtmlAgilityPack.HtmlAttribute itemscopeAttr)
{
this.node = node;
this.itemscopeAttr = itemscopeAttr;
nodeType = XmlNodeType.Element;
}
public string Validate(HtmlAgilityPack.HtmlDocument document, HtmlStaticizeContext status)
{
//得到当前页面的目录
string documentDir = status.Uri.GetFileDirectory();
var errorMessage = new StringBuilder();
var htmlNode = document.DocumentNode;
var nodes = htmlNode.SelectNodes("//a");
if (nodes == null || nodes.Count == 0)
{
return null;
}
foreach (var aNode in nodes)
{
string href = aNode.GetAttributeValue("href", null);
if (string.IsNullOrWhiteSpace(href) || href[0] == '#')
{
continue;
}
//如果href是相对当前页面来说的：<a href="1.html" />
if (!href[0].IsDirectorySeparator())
{
href = documentDir + href;
}
Uri uri;
if (!Uri.TryCreate(href, UriKind.RelativeOrAbsolute, out uri))
{
continue;
}
//这里可以增加对站内域名的判断
if (uri.IsAbsoluteUri && !string.IsNullOrEmpty(uri.Host))
{
continue;
}
string local = fileReslover.ResloveLocalPath(uri);
if (string.IsNullOrEmpty(local))
{
continue;
}
string localPath = System.IO.Path.Combine(searchDirectory, local);
bool isExisting;
if (!files.TryGetValue(localPath, out isExisting))
{
isExisting = System.IO.File.Exists(localPath);
try
{
files.Add(localPath, isExisting);
}
catch (Exception)
{
}
}
if (!isExisting)
{
errorMessage.AppendFormat("本地不存在链接 \"{0}\" 所指向的文件 \"{1}\"。", uri.ToString(), localPath);
}
}
return errorMessage.Length == 0 ? null : errorMessage.ToString();
}
private HtmlAgilityPack.HtmlNode GetSongNodeList(HtmlAgilityPack.HtmlNode body)
{
var contentNode = helper.GetElementWithAttribute(body, "div", "class", "section");
//var songList = contentNode.ChildNodes;// helper.GetElementWithAttribute(contentNode, "p", "class", "row");
return contentNode;
//var song = helper.GetElementWithAttribute(songList, "li", "class", "song-wrap");
//return helper.GetElementWithAttribute(song, "span", "class", "main");
}
/// <summary>
/// 执行验证。
/// </summary>
/// <param name="document">被验证的 HtmlDocument</param>
/// <returns>验证通过则返回true。</returns>
public string Validate(HtmlAgilityPack.HtmlDocument document, HtmlStaticizeContext status)
{
if (document == null)
{
throw new ArgumentNullException("document");
}
return documentValidation.Invoke(document) ? null : errorMessage;
}
public KeyValuePair<PartsPage, IEnumerable<string>> RunActions(string content, string url, HtmlAgilityPack.HtmlDocument doc)
{
var result = AvitoHelpFulMethod.GetParamFromNodes(doc, "Предмет одежды");
if (result != null)
return new KeyValuePair<PartsPage, IEnumerable<string>>(PartsPage.DressSubject, new List<string> { result });
else
return new KeyValuePair<PartsPage, IEnumerable<string>>(PartsPage.DressSubject, null);
}
private static IEnumerable<HtmlAgilityPack.HtmlNode> Traverse(HtmlAgilityPack.HtmlNode node)
{
yield return node;
foreach (var child in node.ChildNodes)
foreach (var n in Traverse(child))
yield return n;
}
public FreeElementAdaptor( HtmlNodeFactory factory, AP.HtmlNode node )
: base( node )
{
if ( node.ParentNode != null )
throw new InvalidOperationException();
_factory = factory;
}
