public ParentNodeWrapper(RootNode rootNode, IParentNodeImplementor node, IParentNode parent)
: base(rootNode, node)
{
_contextRoot = this;
_parentNode = node;
_parent = parent;
}
private ParentNodeWrapper(RootNode rootNode, IParentNode contextRoot, IParentNodeImplementor node, IParentNode parent)
: base(rootNode, node)
{
_contextRoot = contextRoot;
_parentNode = node;
_parent = parent;
}
public override object Parse(IParentNode node, Type valueType) {
var ti = valueType.GetTypeInfo();
if (!ti.IsPrimitive && !valueType.Equals(typeof(string))) {
throw new NotSupportedException("HtmlValueQueryAttribute 只支持基元类型");
}
var ele = node.QuerySelector(this.Selector);
return this.Parse(ele, valueType);
}
private void SetupRootNodes()
{
var rootNode = new RootNode();
var dataXml = XmlNode.Parse("<data><note desc=\"1\" body=\"One!\"/><note desc=\"2\" body=\"Two?\"/></data>");
var template = XDocument.Load(@"..\..\templates\noteviewer.xml");
var templateXml = XmlNode.Parse(template.ToString());
rootNode.Register<IParentNodeImplementor>(new AggregateNode(dataXml, templateXml));
dataNode = rootNode.Nodes("data").First();
templateNode = rootNode.Nodes().Skip(1).First();
}
public override object Parse(IParentNode node, Type valueType) {
var ti = valueType.GetTypeInfo();
if (!typeof(IEnumerable).GetTypeInfo().IsAssignableFrom(ti)) {
throw new NotSupportedException("HtmlCollectionQueryAttribute 只支持非基元/非字符串的集合类型");
}
var type = ti.GenericTypeArguments[0];
if (type.GetTypeInfo().IsPrimitive || type.Equals(typeof(string))) {
throw new NotSupportedException("HtmlCollectionQueryAttribute 只支持非基元/非字符串的集合类型");
}
//object parser
var value = this.ParseSub(type, node);
return JsonConvert.DeserializeObject(JsonConvert.SerializeObject(value), valueType);
}
private async Task ParseRelease(ICollection <ReleaseInfo> releases, IParentNode row, TorznabQuery query,
bool matchWords)
{
var anchor = row.QuerySelector("a");
var commentsLink = anchor.GetAttribute("href");
var title = anchor.TextContent.Trim();
var cat = commentsLink.Split('/')[3];
var categories = MapTrackerCatToNewznab(cat);
var publishStr = row.QuerySelectorAll("td")[2].TextContent.Trim();
var publishDate = TryToParseDate(publishStr, DateTime.Now);
var sizeStr = row.QuerySelectorAll("td")[3].TextContent.Trim();
// return results only for requested categories
if (query.Categories.Any() && !query.Categories.Contains(categories.First()))
{
return;
}
// match the words in the query with the titles
if (matchWords && !CheckTitleMatchWords(query.SearchTerm, title))
{
return;
}
// parsing is different for each category
if (cat == DivxTotalCategories.Series)
{
await ParseSeriesRelease(releases, query, commentsLink, cat, publishDate);
}
else if (query.Episode == null) // if it's scene series, we don't return other categories
{
if (cat == DivxTotalCategories.Peliculas || cat == DivxTotalCategories.PeliculasHd ||
cat == DivxTotalCategories.Peliculas3D || cat == DivxTotalCategories.PeliculasDvdr)
{
ParseMovieRelease(releases, query, title, commentsLink, cat, publishDate, sizeStr);
}
else
{
var size = TryToParseSize(sizeStr, DivxTotalFizeSizes.Otros);
GenerateRelease(releases, title, commentsLink, commentsLink, cat, publishDate, size);
}
}
}
/// <summary>
///
/// </summary>
/// <param name="document">the html page</param>
/// <param name="spell">An almost empty spell</param>
/// <returns>A filled spell</returns>
private static Spell ParseSpellFromDocument(IParentNode document, Spell spell)
{
spell.Name = document.QuerySelector(".pagetitle").TextContent;
var div = document.QuerySelector("#frmPrint");
div.RemoveChild(div.LastChild); // rm script
var html = div.InnerHtml.Split("<br><br>");
var parser = new HtmlParser();
var content = html
.FirstOrDefault()?.Trim()
.Split("<br>", StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Substring(x.IndexOf("</b> ") + 5))
.Select(x => parser.Parse(x).DocumentElement.TextContent)
.ToArray();
var firstline = content[0].Split(" ; Niveau ", StringSplitOptions.RemoveEmptyEntries);
spell.School = firstline[0].Contains(" ") ? firstline[0].Remove(firstline[0].IndexOf(' ')) : firstline[0];
spell.SchoolFull = firstline[0];
//spell.Level = firstline[1]
// .Split (", ", StringSplitOptions.RemoveEmptyEntries)
// .Select (x => x.Split (' ')) // no space sometime Bard6
// .ToDictionary (x => x[0], x => int.Parse (x[1]));
spell.CastingTime = content[1];
spell.Components = content[2];
spell.Range = content[3];
spell.Target = Find(content, "Cibles ", "Cible "); // should be line 4
spell.Effect = Find(content, "Effet "); // should be line 4
spell.Zone = Find(content, "Zone d'effet "); // should be line 4
spell.Duration = content[5];
var lastline = content[6].Split(" ; Résistance à la magie ", StringSplitOptions.RemoveEmptyEntries);
spell.SavingThrow = lastline[0];
spell.SpellResistance = lastline[1];
spell.Description = string
.Concat(html?.Skip(1).Select(x => parser.Parse(x).DocumentElement.TextContent)).Trim();
return(spell);
}
private IEnumerable<object> ParseSub(Type t, IParentNode node) {
var values = new List<object>();
//var ttt = typeof(IList<>).MakeGenericType(t);
var eles = node.QuerySelectorAll(this.Selector);
var ps = t.GetRuntimeProperties();
foreach (var ele in eles) {
var o = Activator.CreateInstance(t);
var obj = Convert.ChangeType(o, t);
foreach (var p in ps) {
var value = p.Parse(ele);
p.SetValue(obj, value);
}
values.Add(obj);
}
return values;
}
private static (IElement Element, IHtmlItem HtmlItem) GetFirstOfDefaultNode(IParentNode element,
IEnumerable <IHtmlItem> attributes)
{
IElement node = null;
IHtmlItem htmlItem = null;
foreach (var attribute in attributes)
{
node = element.QuerySelector(attribute.Selector);
if (node == null || !string.IsNullOrEmpty(attribute.Attr) && !node.HasAttribute(attribute.Attr))
{
continue;
}
htmlItem = attribute;
break;
}
return(node, htmlItem);
}
private static (IHtmlCollection <IElement> Elements, IHtmlItem HtmlItem) GetFirstOfDefaultNodes(
IParentNode element, IEnumerable <IHtmlItem> attributes)
{
IHtmlCollection <IElement> node = null;
IHtmlItem htmlItem = null;
foreach (var attribute in attributes)
{
node = element.QuerySelectorAll(attribute.Selector);
if (node == null || !node.Any())
{
continue;
}
htmlItem = attribute;
break;
}
return(node, htmlItem);
}
/// <summary>
/// The CountryResults.
/// </summary>
/// <param name="document">The document<see cref="IParentNode"/>.</param>
/// <returns>The <see cref="List{CountryResult}"/>.</returns>
private static List <CountryResult> CountryResults(IParentNode document)
{
var result = new List <CountryResult>();
var countriesTableCells = document.QuerySelectorAll(CountriesTodayTbodyTr).Where(c => !c.ClassList.Any(c => c == "total_row_world") || !c.ClassList.Any(c => c == "row_continent"));
//var totalColumns = 10;
const int countryColIndex = 0;
const int casesColIndex = 1;
const int todayCasesColIndex = 2;
const int deathsColIndex = 3;
const int todayDeathsColIndex = 4;
const int curedColIndex = 5;
const int activeColIndex = 6;
const int criticalColIndex = 7;
const int casesPerOneMillionColIndex = 8;
const int deathsPerOneMillionColIndex = 9;
// const int firstCaseColIndex = 10;
const int totalTestColIndex = 10;
const int testPerOneMillionColIndex = 11;
result.AddRange(countriesTableCells.Select(row => (row, cells: row.QuerySelectorAll("td")))
.Select(t => new { t, country = GetCountry(countryColIndex, t.cells) })
.Where(t => t.country != "Total:")
.Select(t => new CountryResult
{
Country = t.country,
Cases = t.t.cells[casesColIndex].TextContent.ToInt(),
Active = t.t.cells[activeColIndex].TextContent.ToInt(),
CasesPerOneMillion = t.t.cells[casesPerOneMillionColIndex].TextContent.ToDecimal(),
Critical = t.t.cells[criticalColIndex].TextContent.ToInt(),
Deaths = t.t.cells[deathsColIndex].TextContent.ToInt(),
DeathsPerOneMillion = t.t.cells[deathsPerOneMillionColIndex].TextContent.ToDecimal(),
Recovered = t.t.cells[curedColIndex].TextContent.ToInt(),
TodayCases = t.t.cells[todayCasesColIndex].TextContent.ToInt(),
TodayDeaths = t.t.cells[todayDeathsColIndex].TextContent.ToInt(),
// FirstCase = t.t.cells[firstCaseColIndex]?.TextContent?.Trim(),
TotalTest = t.t.cells[totalTestColIndex]?.TextContent?.ToInt(),
TestPerOneMillion = t.t.cells[testPerOneMillionColIndex]?.TextContent?.ToInt()
}));
return(result);
}
private async Task <ApiImageItem[]> GetPhotos(IParentNode detailCardDocument)
{
var result = new List <ApiImageItem>();
const string photoQuery = "[data-fancybox=gallery]";
var cardPhotosElements = detailCardDocument.QuerySelectorAll(photoQuery);
foreach (var cardPhotoElement in cardPhotosElements)
{
var link = MireaApiEndpoints.NewsBaseAddress + cardPhotoElement.Attributes["href"]?.Value;
var name = link?.Split('/').Last();
var photoData = await _httpClient.GetByteArrayAsync(link);
var newFile = new ApiImageItem(link, photoData, name);
result.Add(newFile);
}
return(result.ToArray());
}
public void SetParent(IParentNode parentNode)
{
if (!JNet.IsServer)
{
Debug.LogError("Not on server, should not call SetParent.");
return;
}
if (parentNode == null)
{
ParentNodeID = 0;
ParentNetID = 0;
transform.SetParent(null, true);
}
else
{
ParentNodeID = parentNode.GetNodeID();
ParentNetID = parentNode.GetNetObject()?.NetID ?? 0;
transform.SetParent(parentNode.GetTransform(), true);
}
}
private static object DeserializeObject(IParentNode element, Type type)
{
if (!type.IsClass)
{
return(null);
}
var properties =
type.GetProperties()
.Where(item => item.CanWrite && item.CanRead);
var instance = CreateInstance(type);
foreach (var propertyInfo in properties)
{
var isHtmlItem = Attribute.IsDefined(propertyInfo, typeof(HtmlItemAttribute));
var isHtmlMultiItems = Attribute.IsDefined(propertyInfo, typeof(HtmlMultiItemsAttribute));
if (!isHtmlMultiItems && !isHtmlItem)
{
continue;
}
object propertyValue = null;
if (isHtmlItem)
{
propertyValue = DeserializeHtmlItem(propertyInfo, element);
}
else if (isHtmlMultiItems)
{
propertyValue = DeserializeHtmlMultiItems(propertyInfo, element);
}
if (propertyValue != null)
{
propertyInfo.SetValue(instance, propertyValue);
}
}
return(instance);
}
/// <summary>
/// Получает из тега tr набор PrimaryLesson для урока по группам, или один PrimaryLesson для урока всем классом
/// </summary>
/// <param name="element">Тег tr с набором уроков в td</param>
/// <returns>Объект Lesson для данного номера урока</returns>
private Lesson GetLessonOfClass(IParentNode element)
{
switch (element.Children[1].GetAttribute("class"))
{
case "yok":
return(null);
case "cv":
return(new Lesson(GetOneGroupLessonOfClass(element.Children[1])));
default:
var lesson = new Lesson();
foreach (var item in element.Children)
{
if (IsPrimaryLessonOfClass(item))
{
lesson.AddOneGroupLesson(GetOneGroupLessonOfClass(item));
}
}
return(lesson);
}
}
public override object Parse(IParentNode node, Type valueType) {
var ti = valueType.GetTypeInfo();
if (!typeof(IEnumerable).GetTypeInfo().IsAssignableFrom(ti)) {
throw new NotSupportedException("HtmlValueCollectionParserAttribute 只支持基元类型与string的集合类型");
}
var type = ti.GenericTypeArguments[0];
if (!type.GetTypeInfo().IsPrimitive && !type.Equals(typeof(string))) {
throw new NotSupportedException("HtmlValueCollectionParserAttribute 只支持基元类型与string的集合类型");
}
var values = new List<object>();
var eles = node.QuerySelectorAll(this.Selector);
foreach (var ele in eles) {
var value = base.Parse(ele, type);
values.Add(value);
}
return JsonConvert.DeserializeObject(JsonConvert.SerializeObject(values), valueType);
}
private static IElement GetPriceElement(IParentNode document)
{
foreach (var htmlId in HtmlIdsToCheck)
{
var idElement = document.QuerySelectorAll($"#{htmlId}").FirstOrDefault();
if (idElement != null)
{
return(idElement);
}
}
foreach (var htmlClass in HtmlClassesToCheck)
{
var classElement = document.QuerySelectorAll($".{htmlClass}").FirstOrDefault();
if (classElement != null)
{
return(classElement);
}
}
return(null);
}
private static object DeserializeHtmlItem(PropertyInfo propertyInfo, IParentNode element)
{
var attributes = propertyInfo.GetCustomAttributes <HtmlItemAttribute>().Cast <IHtmlItem>().ToList();
if (attributes?.Any() != true)
{
throw new NullReferenceException();
}
var tuple = GetFirstOfDefaultNode(element, attributes);
if (tuple.Element == null || tuple.HtmlItem == null)
{
return(null);
}
var converter = CheckForConverter(propertyInfo);
var targetValue = GetTargetValue(tuple.HtmlItem, tuple.Element, propertyInfo.PropertyType);
if (converter != null)
{
targetValue = converter.ReadHtml(tuple.Element, propertyInfo.PropertyType, targetValue);
}
return(targetValue);
}
private static string GetDescription(IParentNode document)
{
var description = document.QuerySelector("#event-description");
var bPageShare = description.QuerySelector(".b-page-share");
var prmplace = description.QuerySelector(".b-prmplace-media");
var note = description.QuerySelector(".note");
if (bPageShare != null)
{
description.RemoveElement(bPageShare);
}
if (prmplace != null)
{
description.RemoveElement(prmplace);
}
if (note != null)
{
description.RemoveElement(note);
}
return(description.InnerHtml);
}
private static List <Mirror> ListMirrorsInNode(IParentNode parentNode)
{
List <Mirror> mirrorList = new List <Mirror>();
void AddMirrorsFromElement(IElement element)
{
if (element is IHtmlAnchorElement a && a.Text != "JDownloader2")
{
mirrorList.Add(new Mirror
{
MirrorName = Regex.Replace(a.Text, "filehoster(s?): ", "", RegexOptions.IgnoreCase),
MirrorUrl = a.Href
});
}
}
IElement mirrorListElement = parentNode.QuerySelector(".entry-content")
.Children.First(e => e.Text().Contains("download mirrors", StringComparison.CurrentCultureIgnoreCase))
.NextElementSibling;
foreach (IElement element in mirrorListElement.Children)
{
if (element.LocalName == "li")
{
foreach (IElement child in element.Children)
{
AddMirrorsFromElement(child);
}
}
else
{
AddMirrorsFromElement(element);
}
}
return(mirrorList);
}
private static EntityListController BuildEntitySelector(INode rootNode, Func<IParentNode, IAccessor<string>> fnGetText, RootNode actualRootNode, IParentNode dynamicData)
{
var entityList = rootNode.Get<IEntityList<IUIEntityRow>>();
var selector = new EntitySelector {Rows = entityList.Rows, SelectedRow = entityList.Rows.First()};
var rowSelector = dynamicData.Nodes("rowSelector").First();
rowSelector.Register(selector);
var textOutput = dynamicData.Nodes("textOutput").First().Get<IAccessor<string>>();
var nodeMessage = new NodeMessage
{
Target = textOutput,
MessagePredicate = (message => message.Method.Name == "set_SelectedRow"),
TargetDelegate = (Func<IInvocation, Action<IAccessor<string>>>)
(row => accessor => accessor.Value = fnGetText(((IEntityRow)row.Arguments.First()).Context).Value)
//TODO: 1. This seems a little complicated...
//Somebody, or something in the xml needs to clue the framework in to how to do this wiring.
};
actualRootNode.InstallHook(nodeMessage);
var _entityListController = new EntityListController {EntityList = rowSelector.Get<IEntitySelector>()};
_entityListController.Beginning();
return _entityListController;
}
private static IMenuItem CreateMenuItem(IParentNode element, IMenuCategory category, string description, decimal price)
{
string itemNumber = category.Name;
string itemName = GetItemName(element);
int indexOfSpace = itemName.IndexOf(' ');
if (itemName[0].IsDigit() && indexOfSpace > 0)
{
// We probably have a menu number, so let's extract it
itemNumber = itemName.Substring(0, indexOfSpace);
if (itemNumber.EndsWith("."))
{
itemNumber = itemNumber.Substring(0, itemNumber.Length - 1);
}
itemName = itemName.Substring(indexOfSpace + 1);
}
IMenuItem item = new MenuItem {
Number = itemNumber.Trim(), Name = itemName.Trim(), Category = category, Description = description, Price = price
};
return(item);
}
private static Dictionary <IElement, List <ICssStyleRule> > GetElementsWithStyles(IEnumerable <ICssStyleSheet> stylesToInline, IParentNode document)
{
var elementsWithStyles = new Dictionary <IElement, List <ICssStyleRule> >();
foreach (var style in stylesToInline.SelectMany(x => x.Children).OfType <ICssStyleRule>())
{
var elementsForSelector = document.QuerySelectorAll(style.Selector.Text);
foreach (var el in elementsForSelector)
{
var existing = elementsWithStyles.ContainsKey(el) ? elementsWithStyles[el] : new List <ICssStyleRule>();
existing.Add(style);
elementsWithStyles[el] = existing;
}
}
return(elementsWithStyles);
}
private static void BuildNoteList(IParentNode rootNode, Func<IParentNode, IEnumerable<IParentNode>> fnGetNotes, Func<IParentNode, IAccessor<string>> fnGetDesc, IContainer container, out EntityList entityList)
{
var notesContext = fnGetNotes(rootNode);
var rows = notesContext.Take(2).Select(node => BuildEntityRow(node, fnGetDesc)).ToList();
entityList = new EntityList {Parent = container, Rows = rows};
rootNode.Register(entityList);
}
public abstract object Parse(IParentNode node, Type valueType);
public static void ProcessSvGCss(IParentNode items, SpecData specificationData)
{
var properties = BuildSvgPropertyList(items);
specificationData.Interfaces.AddRange(GenerateInterfacesForCss(properties, specificationData));
}
private static IEnumerable<Property> BuildSvgPropertyList(IParentNode table)
{
var properties = new List<Property>();
var t = table.QuerySelector("thead").QuerySelectorAll("th").Select(b => b.TextContent.Trim()).ToList();
foreach (var element in table.QuerySelector("tbody").QuerySelectorAll("tr"))
{
var p = new Property();
for (var i = 0; i < t.Count; i++)
{
var f = element.QuerySelectorAll("th, td")[i].TextContent.Trim();
p.GetType().GetProperty(FixPropertyName(t[i])).SetValue(p, f, null);
}
AddSingleOrMultipleProperties(p, properties);
}
return properties;
}
public override object Parse(IParentNode node, Type valueType) {
throw new NotImplementedException();
}
public static IHtmlElement HasElement(string selector, IParentNode document)
{
var element = Assert.Single(document.QuerySelectorAll(selector));
return(Assert.IsAssignableFrom <IHtmlElement>(element));
}
//Проверка, есть ли на поисковой странице ссылки
bool CheckYaPage(IParentNode htmlDocument) => htmlDocument.QuerySelector(".serp-item")?.InnerHtml != null;
private static string GetDownloadLink(IParentNode dom) =>
dom.QuerySelector($"a[href*=\"{DownloadLink}\"]")?.GetAttribute("href");
private static Post ParsePostInternal(IParentNode postContainer, uint threadId, string opId = null)
{
var result = new Post();
var elemPost = postContainer.Children.SingleOrDefault(x => x.ClassList.Contains("post"));
var elemPostInfo = elemPost?.Children.SingleOrDefault(x => x.ClassName == "postInfo");
var elemSubject = elemPostInfo?.Children
.SingleOrDefault(x =>
x.LocalName == "span" &&
x.ClassName == "subject");
result.Subject = elemSubject?.InnerHtml;
var elemNameBlock = elemPostInfo?.Children
.SingleOrDefault(x =>
x.LocalName == "span" &&
x.ClassName == "nameBlock");
result.Email = elemNameBlock?.Children.FirstOrDefault()?.InnerHtml;
var id = elemNameBlock?.Children.ElementAtOrDefault(1)?.InnerHtml;
result.Id = id?.Substring(5, id.Length - 6);
var elemDate = elemPostInfo?.Children
.SingleOrDefault(x =>
x.LocalName == "span" &&
x.HasAttribute("data-raw"));
var dateRawStr = elemDate?.GetAttribute("data-raw");
if (elemDate != null && !string.IsNullOrEmpty(dateRawStr) && long.TryParse(dateRawStr, out var dateRaw))
{
result.Date = DateTimeOffset.FromUnixTimeSeconds(dateRaw).UtcDateTime;
}
var elemPostNum = elemPostInfo?.Children
.SingleOrDefault(x =>
x.LocalName == "span" &&
x.ClassName == "postNum");
var elemPostQuote = elemPostNum?.Children.ElementAtOrDefault(1);
if (elemPostQuote != null && !string.IsNullOrEmpty(elemPostQuote.InnerHtml) &&
uint.TryParse(elemPostQuote.InnerHtml, out var postNum))
{
result.Number = postNum;
}
var elemPostMessage = elemPost?.Children
.SingleOrDefault(x =>
x.LocalName == "blockquote" &&
x.ClassList.Contains("postMessage"));
if (elemPostMessage != null)
{
var quoteLinks = elemPostMessage.Children
.Where(x =>
x.LocalName == "a" &&
x.ClassName == "quotelink");
foreach (var quoteLink in quoteLinks)
{
var href = quoteLink.GetAttribute("href");
quoteLink.SetAttribute("href", href.Substring(href.LastIndexOf("#", StringComparison.Ordinal)));
}
var images = elemPostMessage.Children
.Where(x =>
x.LocalName == "img" &&
x.HasAttribute("src"));
foreach (var img in images)
{
img.SetAttribute("src", "../Resources/" + img.GetAttribute("src").Trim('/'));
}
}
result.MessageBody = elemPostMessage?.InnerHtml;
if (elemPostMessage != null)
{
var backup = elemPostMessage.InnerHtml;
elemPostMessage.InnerHtml = elemPostMessage.InnerHtml
.Replace("<br>", "\n")
.Trim('\n');
result.MessageText = elemPostMessage.Text().Trim().Replace(" ", " ").Replace(" \n", "\n");
elemPostMessage.InnerHtml = backup;
}
var elemFile = elemPost?.Children
.SingleOrDefault(x => x.LocalName == "div" &&
x.ClassName == "file");
var elemFileLink = elemFile?
.Children.FirstOrDefault()?
.Children.FirstOrDefault()?
.Children.SingleOrDefault(x =>
x.LocalName == "a" &&
x.HasAttribute("href"));
if (elemFileLink != null)
{
if (elemFileLink.HasAttribute("download"))
{
var fileName = elemFileLink.GetAttribute("download");
var fileUrl = SanitizeRelativeSrcUrl(elemFileLink.GetAttribute("href"), 5);
var elemFileThumbImg = elemFile.Children.ElementAtOrDefault(1)?
.Children.SingleOrDefault(x =>
x.LocalName == "img" &&
x.HasAttribute("src"));
var fileThumbUrl = SanitizeRelativeSrcUrl(elemFileThumbImg?.GetAttribute("src"), 5);
if (fileThumbUrl != null)
{
result.File = new Post.PostFile
{
FileName = fileName,
FileUrl = fileUrl,
FileThumbUrl = fileThumbUrl
};
}
}
else if (elemFileLink.InnerHtml.Contains("Spoiler"))
{
var fileName = elemFileLink.GetAttribute("href");
fileName = fileName.Substring(fileName.LastIndexOf('/') + 1);
var fileUrl = SanitizeRelativeSrcUrl(elemFileLink.GetAttribute("href"), 5);
result.File = new Post.PostFile
{
FileName = fileName,
FileUrl = fileUrl,
FileThumbUrl = "SPOILER"
};
}
}
else
{
var elemEmbed = elemFile?.Children.ElementAtOrDefault(1)?
.Children.SingleOrDefault(x =>
x.LocalName == "iframe" &&
x.HasAttribute("src"));
result.EmbedUrl = elemEmbed?.GetAttribute("src");
}
if (opId == null || result.Id == opId)
{
result.Name = "OP";
result.NameColor = Color.Red;
}
else
{
var(name, color) = UidHighlighter.GetHighlightForPost(threadId, result.Id);
result.Name = name;
result.NameColor = color;
}
return(result);
}
private static IAccessor<string> BuildTextDisplay(IParentNode rootNode, IContainer container)
{
var textOutput = rootNode.Nodes("textOutput").First();
textOutput.Register<IAccessor<string>>(new TextDisplay {Parent = container});
return textOutput.Get<IAccessor<string>>();
}
private void ProcessAttribute(IParentNode data, IParentNode parent, INode node, object newObj)
{
var propToSet = newObj.GetType().GetProperties().SingleOrDefault(prop => prop.Name.ToLower() == node.Name.ToLower());
var get = node.Get<IAccessor<string>>();
if (node.Name == "id")
{
_nodeLookups[get.Value] = parent;
return;
}
if (get.Value[0] != '[' && get.Value[0] != '{')
{
if (propToSet != null)
{
propToSet.SetValue(newObj, get.Value, null);
}
return;
}
var path = Decoder.FromPath(get.Value);
//if both local and data are set, this means that each of the children in Local gets each instance of Data
if (path.Local != null && path.Data != null)
{
//TODO: Get all of this expression processing into a separate class.
//Might even wrap some of the functionality in a localized class
var fnData = path.Data.Compile();
var dataRows = fnData.DynamicInvoke(data);
var fnLocal = path.Local.Compile();
var localRows = ((IEnumerable)fnLocal.DynamicInvoke(parent)).Cast<IParentNode>();
var localRow = localRows.Single();
var isEnumerable = typeof (System.Collections.IEnumerable).IsAssignableFrom(dataRows.GetType());
var propIsEnumerable = typeof (System.Collections.IEnumerable).IsAssignableFrom(propToSet.PropertyType);
if (propIsEnumerable && isEnumerable)
{
//I should cast here... cause I can
var innerType = propToSet.PropertyType.GetGenericArguments().First();
var e = ((System.Collections.IEnumerable) dataRows).Cast<IParentNode>();
foreach (var row in e)
{
ProcessTemplateElem(row, localRow);
//call back to ProcessTemplateElem?
//ultimately need to create an object of type innerType, with the context of row
int debug2 = 0;
}
}
if (!isEnumerable)
{
}
}
if (path.Local != null)
{
var fnLocal = path.Local.Compile();
var localRows = ((IEnumerable)fnLocal.DynamicInvoke(parent)).Cast<IParentNode>();
var dataRows = localRows.Select(localRow => ProcessTemplateElem(data, localRow)).ToList();
var e = dataRows.Cast<IUIInitialize>().ToList();
node.Register(e);
var child = node.Get<IEnumerable<IUIInitialize>>();
propToSet.SetValue(newObj, child, null);
int debug = 0;
//now set this to propToSet
}
return;
}
private object ProcessTemplateElem(IParentNode dataNode, IParentNode templateNode)
{
//create an object based on the template node, and insert it in the template node
var nodeTypes = typeof (DynamicTemplate)
.Assembly
.GetTypes()
.Where(type => type.Name.ToLower() == templateNode.Name.ToLower());
if (nodeTypes.Count() != 1) throw new Exception("Found " + nodeTypes.Count() + " of type " + templateNode.Name);
var newType = nodeTypes.Single();
var newObj = newType.GetConstructor(Type.EmptyTypes).Invoke(new object[] {});
templateNode.Attributes().Do(attr => ProcessAttribute(dataNode, templateNode, attr, newObj));
//TODO: now do child elements of the templateNode
//each element in the template will have data registered to it,
//even if the parent never explicitly calls it.
//A later optimization step could cull this out
return newObj;
}
public static T Qs<T>(this IParentNode b, string selector) where T : class, IElement => b.QuerySelector(selector) as T;
public static string QsAttr(this IParentNode b, string selector, string attribute) => b.QuerySelector(selector)?.GetAttribute(attribute);
public string ParseName(IParentNode node)
{
return(node.QuerySelector("div.product_header h1").TextContent);
}
public static IElement El(this IParentNode b, string selector) => b.QuerySelector(selector);
public static IHtmlFormElement HasForm(string selector, IParentNode document)
{
var form = Assert.Single(document.QuerySelectorAll(selector));
return(Assert.IsAssignableFrom <IHtmlFormElement>(form));
}
//TODO: 2. $$$ Replace this method with the automatic template reading above
private static IUIEntityRow BuildEntityRow(IParentNode node, Func<IParentNode, IAccessor<string>> fnGetDesc)
{
IEntityRow row = new EntityRow {Columns = new[] {fnGetDesc(node)}, Context = node};
node.Register(row);
return node.Get<IUIEntityRow>();
}
public static IElement QuerySelectorOrThrow(this IParentNode parent, string selector) =>
parent.QuerySelector(selector) ?? throw new InvalidOperationException($"Can't find any element matching selector '{selector}'.");
public static IEnumerable <T> Els <T>(this IParentNode b, string selector) where T : class, IElement => b.QuerySelectorAll(selector).Cast <T>();
public static IEnumerable <IElement> Els(this IParentNode b, string selector) => b.QuerySelectorAll(selector);
/// <summary>
/// Parse Countries Data.
/// </summary>
/// <param name="document">The document<see cref="IParentNode"/>.</param>
/// <returns>The <see cref="List{CountryResult}" />.</returns>
private static List <CountryResult> ParseCountriesData(IParentNode document) => CountryResults(document);
private void DoSanitize(IHtmlDocument dom, IParentNode context, string baseUrl = "")
{
// remove non-whitelisted tags
foreach (var tag in context.QuerySelectorAll("*").Where(t => !IsAllowedTag(t)).ToList())
{
RemoveTag(tag, RemoveReason.NotAllowedTag);
}
SanitizeStyleSheets(dom, baseUrl);
// cleanup attributes
foreach (var tag in context.QuerySelectorAll("*").ToList())
{
// remove non-whitelisted attributes
foreach (var attribute in tag.Attributes.Where(a => !IsAllowedAttribute(a)).ToList())
{
RemoveAttribute(tag, attribute, RemoveReason.NotAllowedAttribute);
}
// sanitize URLs in URL-marked attributes
foreach (var attribute in tag.Attributes.Where(IsUriAttribute).ToList())
{
var url = SanitizeUrl(attribute.Value, baseUrl);
if (url == null)
{
RemoveAttribute(tag, attribute, RemoveReason.NotAllowedUrlValue);
}
else
{
tag.SetAttribute(attribute.Name, url);
}
}
// sanitize the style attribute
var oldStyleEmpty = string.IsNullOrEmpty(tag.GetAttribute("style"));
SanitizeStyle(tag, baseUrl);
var checkClasses = AllowedCssClasses != null;
var allowedTags = AllowedCssClasses?.ToArray() ?? new string[0];
// sanitize the value of the attributes
foreach (var attribute in tag.Attributes.ToList())
{
// The '& Javascript include' is a possible method to execute Javascript and can lead to XSS.
// (see https://www.owasp.org/index.php/XSS_Filter_Evasion_Cheat_Sheet#.26_JavaScript_includes)
if (attribute.Value.Contains("&{"))
{
RemoveAttribute(tag, attribute, RemoveReason.NotAllowedValue);
}
else
{
if (checkClasses && attribute.Name == "class")
{
var removedClasses = tag.ClassList.Except(allowedTags).ToArray();
foreach (var removedClass in removedClasses)
{
RemoveCssClass(tag, removedClass, RemoveReason.NotAllowedCssClass);
}
if (!tag.ClassList.Any())
{
RemoveAttribute(tag, attribute, RemoveReason.ClassAttributeEmpty);
}
}
else if (!oldStyleEmpty && attribute.Name == "style" && string.IsNullOrEmpty(attribute.Value))
{
RemoveAttribute(tag, attribute, RemoveReason.StyleAttributeEmpty);
}
}
}
}
RemoveComments(context as INode);
DoPostProcess(dom, context as INode);
}
/// <summary>
/// Проверяет является ли содержимое тега tr уроком
/// </summary>
/// <param name="tr">Тег tr</param>
/// <returns>true - в tr уроки, иначе - false</returns>
private bool IsLessonOfClass(IParentNode tr)
{
var x = tr.FirstElementChild.NodeName;
return(x != "TH");
}
