public void GetStringContent_WithInlineEditingAttribute_TextElementProperlyCreated()
{
// Arrange
var htmlProcessor = new HtmlProcessor();
var dummyWidgetModel = new DummyWidgetModel { EditableContent = this.dummyContent, NonEditableContent = this.dummyContent };
var fieldName = "DummyWidget";
var type = "LongText";
// Act
string inlineeditingAwareContent = htmlProcessor.GetStringContent(dummyWidgetModel, "EditableContent");
// Assert
using (var parser = new HtmlParser(inlineeditingAwareContent))
{
HtmlChunk chunk = parser.ParseNext();
Assert.IsNotNull(chunk);
// checks if the HTML tag is of type div and if it has the required attributes
Assert.IsTrue(chunk.TagName.Equals(this.htmlWrapperTag, StringComparison.Ordinal), "There is no wrapper div appended to the property representation.");
Assert.IsTrue(chunk.HasAttribute(this.fieldAttribute), "The field attribute is not appended correctly.");
Assert.IsTrue(chunk.HasAttribute(this.fieldTypeAttribute), "The field type attribute is not appended correctly.");
// checks if the required attributes has proper values assigned to them
Assert.AreEqual(fieldName, chunk.GetParamValue(this.fieldAttribute), "The value of the field attribute is not correct.");
Assert.AreEqual(type, chunk.GetParamValue(this.fieldTypeAttribute), "The value of the fieldType attribute is not correct.");
this.AssertContentAndCloseTag(parser);
}
}
protected WebSearchCounter(
HtmlParser htmlParser,
PageCrawler crawler)
{
this.htmlParser = htmlParser;
this.crawler = crawler;
}
public void HtmlProcessor_CreateInlineEditingRegion_IsDummyContentwrappedIntoInlineEditingRegion()
{
//Arrange: create dummy data which will be set to the related attributes inside the region div tag
TextWriter writer = new StringWriter();
string providerName = "dummyProvider";
string type = "dummyType";
var id = Guid.NewGuid();
string dummyContent = "Lorem ipsum dolor sit amet, consectetuer adipiscing elit";
var providerAttribute = "data-sf-provider";
var typeAttribute = "data-sf-type";
var idAttribute = "data-sf-id";
//Act: create the HTML region
var htmlProcessor = new HtmlProcessor();
using (htmlProcessor.CreateInlineEditingRegion(writer, providerName, type, id))
{
writer.WriteLine(dummyContent);
}
var outPut = writer.ToString();
//Assert: Parses the generated by the htmlTransformationProxy HTML checks if the HTML content is properly wrapped into a div tag
//which has the required by the InlineEditing attributes
//and these attributes has a proper data assigned
using (HtmlParser parser = new HtmlParser(outPut))
{
var chunk = parser.ParseNext();
Assert.IsNotNull(chunk);
//checks if the HTML tag is of type div and if it has the required attributes
Assert.IsTrue(chunk.TagName.Equals("div", StringComparison.InvariantCultureIgnoreCase));
Assert.IsTrue(chunk.HasAttribute(idAttribute));
Assert.IsTrue(chunk.HasAttribute(providerAttribute));
Assert.IsTrue(chunk.HasAttribute(typeAttribute));
//checks if the required attributes has proper values assigned to them
Assert.AreEqual<string>(providerName, chunk.GetParamValue(providerAttribute));
Assert.AreEqual<string>(type, chunk.GetParamValue(typeAttribute));
Assert.AreEqual<string>(id.ToString(), chunk.GetParamValue(idAttribute));
string content = null;
HtmlChunk nextChunk = null;
while ((nextChunk = parser.ParseNext()) != null)
{
chunk = nextChunk;
if (nextChunk.Type == HtmlChunkType.Text)
{
content = nextChunk.GenerateHtml();
}
}
//checks if the region inner content is what it should be
Assert.IsTrue(content.StartsWith(dummyContent, StringComparison.InvariantCultureIgnoreCase));
//checks if the region is properly closed
Assert.IsTrue(chunk.TagName.Equals("div", StringComparison.InvariantCultureIgnoreCase));
Assert.IsTrue(chunk.Type == HtmlChunkType.CloseTag);
}
}
public void MarksExtraTableHeaderAsError()
{
var actual = new ArrayList {new Name("joe", "smith")};
Parse table = new HtmlParser().Parse("<table><tr><td><table><tr><td>first</td><td>last</td><td>address</td></tr><tr><td>joe</td><td>smith</td><td></td></tr></table></td></tr></table>");
var fixture = new Fixture {Processor = new Service.Service()};
fixture.CellOperation.Check(null, new TypedValue(actual), table.Parts.Parts);
Assert.AreEqual("<td><table><tr><td>first</td><td>last</td><td class=\"error\">address<hr /><pre><div class=\"fit_stacktrace\">fitlibrary.exception.FitFailureException: Column 'address' not used.</div></pre></td></tr><tr><td class=\"pass\">joe</td><td class=\"pass\">smith</td><td></td></tr></table></td>", table.Parts.Parts.ToString());
}
public void MarksSameTableCellAsRight()
{
var actual = new ArrayList {new Name("joe", "smith")};
Parse table = new HtmlParser().Parse("<table><tr><td><table><tr><td>first</td><td>last</td></tr><tr><td>joe</td><td>smith</td></tr></table></td></tr></table>");
var fixture = new Fixture {Processor = new Service.Service()};
fixture.CellOperation.Check(null, new TypedValue(actual), table.Parts.Parts);
Assert.AreEqual("<td><table><tr><td>first</td><td>last</td></tr><tr><td class=\"pass\">joe</td><td class=\"pass\">smith</td></tr></table></td>", table.Parts.Parts.ToString());
}
public void RepeatKeywordWithNoPreviousValue()
{
Parse rows = new HtmlParser().Parse("<table><tr><td>xxx</td><td>yy</td></tr><tr><td>aaa</td><td>bb</td><td>ditto</td></tr></table>");
ArrayList results = ProcessRows(rows.Parts);
Assert.AreEqual(3, results.Count, "count");
Assert.AreEqual("aaa", results[0], "first");
Assert.AreEqual("bb", results[1], "second");
Assert.AreEqual("ditto", results[2], "third");
}
public void SingleRowReturnsEachCell()
{
Parse rows = new HtmlParser().Parse("<table><tr><td>aaa</td><td>bb</td><td>c</td></tr></table>");
ArrayList results = ProcessRows(rows.Parts);
Assert.AreEqual(3, results.Count, "count");
Assert.AreEqual("aaa", results[0], "first");
Assert.AreEqual("bb", results[1], "second");
Assert.AreEqual("c", results[2], "third");
}
public void CellIsMadeWithEmbeddedTable()
{
var service = new Service.Service();
Parse table =
new HtmlParser().Parse("<table><tr><td>11</td><td>12</td></tr><tr><td>21</td><td>22</td></tr></table>");
var cell = (Parse) service.Compose(new ParseTable(table));
Assert.AreEqual("\n<td>\n<table>\n<tr>\n<td><span class=\"fit_grey\">11</span></td>\n<td><span class=\"fit_grey\">12</span></td></tr>" +
"\n<tr>\n<td><span class=\"fit_grey\">21</span></td>\n<td><span class=\"fit_grey\">22</span></td></tr></table></td>", cell.ToString());
}
public void NameKeywordAssignsASymbol()
{
var fixture = new TestDoFixture { Processor = new CellProcessorBase() };
var keywords = new FlowKeywords(fixture);
Parse table = new HtmlParser().Parse("<table><tr><td>name</td><td>symbol</td><td>stuff</td></tr></table>");
keywords.Name(table.Parts.Parts);
Assert.AreEqual("some stuff", fixture.NamedFixture("symbol"));
Assert.AreEqual("some stuff", fixture.Processor.Load(new Symbol("symbol")).Instance);
}
public void TagCloseTest() {
var str = @"</p> Test </p><img src='http://www.baidu.com'><a href=''>a";
var parser = new HtmlParser();
using (var doc = parser.Parse(str)) {
var fmt = new PrettyMarkupFormatter();
var html = doc.ToHtml(HtmlMarkupFormatter.Instance);
var html2 = doc.ToHtml(fmt);
var html3 = doc.DocumentElement.ToHtml(fmt);
}
}
public static string GetAttributeValue(this string target, string htmlTag, string attribute)
{
HtmlParser parser = new HtmlParser(target);
HtmlTag tag;
if (parser.ParseNext(htmlTag, out tag))
{
return tag.Attributes[attribute];
}
return Null.NullString;
}
public static string SourceToHtml(string source, HighlighterBase highlighter)
{
HtmlParser parser = new HtmlParser();
highlighter.Parser = parser;
source = Regex.Replace(source, "<%--Start exclude--%>.*?<%--End exclude--%>", "", RegexOptions.Singleline);
string s = highlighter.Parse(source);
return string.Concat("<pre style='margin: 0px;font-size:14px;'>", s, "</pre>");
}
public void MethodIsInvokedForEachRow()
{
Parse table =
new HtmlParser().Parse("<table><tr><td></td></tr><tr><td>method</td></tr><tr><td>value</td></tr></table>");
var cellOperation = new Mock<CellOperation>();
var setUp = new SetUpFixture {CellOperation = cellOperation.Object};
setUp.DoTable(table);
cellOperation.Verify(o => o.TryInvoke(setUp,
It.Is<Tree<Cell>>(c => c.Branches[0].Value.Text == "method"),
It.Is<Tree<Cell>>(c => c.Branches[0].Value.Text == "value"),
It.Is<Tree<Cell>>(c => c.Value.Text == "value")));
}
/// <summary>
/// Extracts the components.
/// </summary>
/// <param name="fileStream">The file stream.</param>
/// <returns></returns>
public static IList<string> ExtractComponents(Stream fileStream)
{
var candidateComponents = new HashSet<string>();
using (var reader = new StreamReader(fileStream))
{
using (HtmlParser parser = new HtmlParser(reader.ReadToEnd()))
{
HtmlChunk chunk = null;
parser.SetChunkHashMode(false);
parser.AutoExtractBetweenTagsOnly = false;
parser.CompressWhiteSpaceBeforeTag = false;
parser.KeepRawHTML = true;
parser.AutoKeepComments = false;
while ((chunk = parser.ParseNext()) != null)
{
if (chunk.Type == HtmlChunkType.OpenTag)
{
//// Angular directives can be tag name (E)
var tagName = chunk.TagName.ToLower();
//// The parser can't handle tag names containing '-'
if (string.IsNullOrEmpty(tagName))
{
var match = Regex.Match(chunk.Html, @"<\W*([a-zA-Z_-]+)").Groups[1];
if (match.Success)
tagName = match.Value.ToLower();
}
candidateComponents.Add(tagName);
for (int i = 0; i < chunk.Attributes.Length; i++)
{
//// The html parser has no more attributes
if (chunk.Values[i] == null)
break;
//// Angular directives can be class attribute value (C)
if (chunk.Attributes[i].ToLower() == "class")
candidateComponents.Add(chunk.Values[i].ToLower());
else
//// Angular directives can be attribute name (A)
candidateComponents.Add(chunk.Attributes[i].ToLower());
}
}
}
}
}
candidateComponents.IntersectWith(ComponentsDependencyResolver.AvailableComponents.Value.Keys);
return candidateComponents.Select(key => ComponentsDependencyResolver.AvailableComponents.Value[key]).ToList();
}
public void SetStyleAttributeAfterPageLoadWithInvalidColor()
{
var html = "<Div style=\"background-color: http://www.codeplex.com?url=<SCRIPT>a=/XSS/alert(a.source)</SCRIPT>\">";
var parser = new HtmlParser(new Configuration().WithCss());
var dom = parser.Parse(html);
var div = (IHtmlElement)dom.QuerySelector("div");
var n = div.Style.Length;
// hang occurs only if this line is executed prior to setting the attribute
// hang occurs when executing next line
div.SetAttribute("style", "background-color: http://www.codeplex.com?url=<SCRIPT>a=/XSS/alert(a.source)</SCRIPT>");
Assert.AreEqual(div.Style.BackgroundColor, "");
}
public void MethodIsInvokedForEachRow()
{
Parse table =
new HtmlParser().Parse("<table><tr><td></td></tr><tr><td>method</td></tr><tr><td>value</td></tr></table>");
var cellOperation = new Mock<CellOperation>();
var processor = new Mock<CellProcessor>();
var constraint = new ConstraintFixture {CellOperation = cellOperation.Object, Processor = processor.Object};
processor.Setup(p => p.TestStatus).Returns(new TestStatus());
cellOperation.Setup(o => o.TryInvoke(constraint,
It.Is<Tree<Cell>>(c => c.Branches[0].Value.Text == "method"),
It.Is<Tree<Cell>>(c => c.Branches[0].Value.Text == "value"),
It.Is<Tree<Cell>>(c => c.Value.Text == "value")))
.Returns(new TypedValue(true));
constraint.DoTable(table);
}
public string Parse(string zenSyntax, ZenType type)
{
switch (type)
{
case ZenType.CSS:
CssParser cssParser = new CssParser();
return cssParser.Parse(zenSyntax.Trim());
case ZenType.HTML:
HtmlParser htmlParser = new HtmlParser();
return htmlParser.Parse(zenSyntax.Trim());
}
return null;
}
public void ConvertFeedItemListToRSSFeedTest()
{
// arrange
string uriPrefix = string.Empty;
HtmlParser target = new HtmlParser(uriPrefix);
// act
RssFeed actual = target.ConvertFeedItemListToRSSFeed(target.HtmlParserFeedItems.ToList());
// arrange
Assert.IsNotNull(actual);
Assert.AreEqual("RSSCode", actual.FeedCode);
Assert.AreEqual("LocationName", actual.LocationName);
Assert.AreEqual(@"<a href='https://www.windowsazurestatus.com/windowsazure/support/status/RSSLink'>rss</a>", actual.RSSLink);
Assert.AreEqual("ServiceName", actual.ServiceName);
}
public ActionResult GetSourceFile(string file)
{
string path = this.HttpContext.Server.MapPath(file);
string examplesRoot = this.HttpContext.Server.MapPath(ExamplesModel.ApplicationRoot + "/Areas/");
var fi = new FileInfo(path);
if (!path.StartsWith(examplesRoot, true, CultureInfo.CurrentCulture))
{
return new HttpStatusCodeResult((int)HttpStatusCode.BadRequest);
}
HighlighterBase hb = null;
switch (fi.Extension.ToLowerInvariant())
{
case ".aspx":
case ".ascx":
case ".cshtml":
case ".master":
hb = new ASPXHighlighter();
break;
case ".cs":
hb = new CSharpHighlighter();
break;
case ".xml":
case ".xsl":
hb = new XMLHighlighter();
break;
case ".js":
hb = new JavaScriptHighlighter();
break;
case ".css":
hb = new CSSHighlighter();
break;
default:
return Content(System.IO.File.ReadAllText(fi.FullName), "text/plain");
}
HtmlParser parser = new HtmlParser();
hb.Parser = parser;
string source = Regex.Replace(System.IO.File.ReadAllText(fi.FullName), "<%--Start exclude--%>.*?<%--End exclude--%>", "", RegexOptions.Singleline);
source = hb.Parse(source);
return this.Content(string.Concat("<pre style='margin: 0px;font-size:14px;'>", source, "</pre>"), "text/html");
}
/// <summary>
/// подготавливает строку к хешированию
/// </summary>
/// <param name="data">входной набор символов</param>
/// <returns>StringBuilder or null</returns>
private StringBuilder CleanString(string data)
{
if (string.IsNullOrEmpty(data)) return null;
HtmlParser parser = new HtmlParser();
var result = new StringBuilder(data.ToLower());
parser.ClearContent(result);
for (int i = 0; i < this.keywords.Length; i++)
{
if (this.keywords[i] != "")
{
while (result.Contains(this.keywords[i]) >= 0)
result.Replace(this.keywords[i], string.Empty);
}
}
parser.ClearContent(result);//еще раз зачищаем строку
return result;
}
/// <summary>
/// Makes sure that the system containers are runat="server" so the layout declaration can be used as a proper container.
/// </summary>
/// <param name="targetTemplate">The template.</param>
/// <param name="ensureSfColsWrapper">if set to <c>true</c> ensures sf_cols containers exists in the template.</param>
protected virtual string ProcessLayoutString(string targetTemplate, bool ensureSfColsWrapper)
{
using (var parser = new HtmlParser(targetTemplate))
{
parser.SetChunkHashMode(false);
parser.AutoExtractBetweenTagsOnly = false;
parser.CompressWhiteSpaceBeforeTag = false;
parser.KeepRawHTML = true;
bool hasSfCols = false;
var output = new StringBuilder();
HtmlChunk chunk;
while ((chunk = parser.ParseNext()) != null)
{
var modified = false;
if (chunk.Type == HtmlChunkType.OpenTag)
{
var cssClass = this.GetAttributeValue(chunk, "class");
if (cssClass != null)
{
var classes = cssClass.Split(new char[] { ' ' });
var chunkHasSfCols = classes.Contains("sf_cols", StringComparer.Ordinal);
hasSfCols = hasSfCols || chunkHasSfCols;
if (chunkHasSfCols ||
classes.Contains("sf_colsIn", StringComparer.Ordinal) ||
classes.Contains("sf_colsOut", StringComparer.Ordinal))
{
chunk.SetAttribute("runat", "server");
modified = true;
}
}
}
output.Append(modified ? chunk.GenerateHtml() : chunk.Html);
}
if (!hasSfCols && ensureSfColsWrapper)
{
return "<div runat=\"server\" class=\"sf_cols\">" + output.ToString() + "</div>";
}
return output.ToString();
}
}
public string CleanString2(string data)
{
if (string.IsNullOrEmpty(data)) return null;
HtmlParser parser = new HtmlParser();
//var result = new StringBuilder(data.ToLower());
data = data.ToLower();
var tmpData = parser.ClearContent(data).Split(' ').ToList<string>();
for (int i = 0; i < this.keywords.Length; i++)
{
if (this.keywords[i] != "")
{
int j = 0;
while (j < tmpData.Count())
{
if (this.keywords[i].Length == 1 && !Char.IsLetter(this.keywords[i][0]))
{
tmpData[j] = tmpData[j].Replace(this.keywords[i], " ");
if(tmpData[j].Length==0)
tmpData.RemoveAt(j);
j++;
}
else if (tmpData[j] == this.keywords[i])
{
tmpData.RemoveAt(j);
}
else { j++; }
}
}
}
StringBuilder sb = new StringBuilder(100);
for (int i = 0; i < tmpData.Count(); i++)
{
sb.Append(tmpData[i] + " ");
}
string result = parser.ClearContent(sb.ToString().Trim());//еще раз зачищаем строку
return result;
}
public static List<string> FileSearch(Uri uriObj1)
{
List<string> returnString = new List<string>();
HtmlTag tag;
CookieAwareWebClient client = new CookieAwareWebClient();
client.CookieContainer = Web.CC;
string html = client.DownloadString(uriObj1);
HtmlParser parse = new HtmlParser(html);
string returnedValue = "";
string value1;
while (parse.ParseNext("a", out tag))
{
if (tag.Attributes.TryGetValue("href", out value1))
{
returnedValue = value1;
if (searchKeyword(returnedValue, "http://10.1.1.242/moodle/") && searchKeyword(returnedValue, ".pdf"))
{
returnString.Add(returnedValue);
}
if (searchKeyword(returnedValue, "http://10.1.1.242/moodle/") && searchKeyword(returnedValue, ".ppt"))
{
returnString.Add(returnedValue);
}
if (searchKeyword(returnedValue, "http://10.1.1.242/moodle/") && searchKeyword(returnedValue, ".doc"))
{
returnString.Add(returnedValue);
}
if (searchKeyword(returnedValue, "http://10.1.1.242/moodle/") && searchKeyword(returnedValue, ".docx"))
{
returnString.Add(returnedValue);
}
if (searchKeyword(returnedValue, "http://10.1.1.242/moodle/") && searchKeyword(returnedValue, ".dwg"))
{
returnString.Add(returnedValue);
}
}
//Console.WriteLine("Exiting filesearch...........");
}
return returnString;
}
public void CreateInlineEditingRegion_DummyContent_IsWrappedIntoInlineEditingRegion()
{
// Arrange: create dummy data which will be set to the related attributes inside the region div tag
TextWriter writer = new StringWriter(System.Globalization.CultureInfo.InvariantCulture);
var providerName = "dummyProvider";
var type = "dummyType";
Guid id = Guid.NewGuid();
// Act: create the CreateInlineEditingRegion
var htmlProcessor = new HtmlProcessor(isInlineEditing: true);
using (htmlProcessor.CreateInlineEditingRegion(writer, providerName, type, id))
{
writer.WriteLine(this.dummyContent);
}
string outPut = writer.ToString();
// Assert: Parses the generated by the htmlTransformationProxy HTML checks if the HTML content is properly wrapped into a div tag
// which has the required by the InlineEditing attributes
// and these attributes has a proper data assigned
using (var parser = new HtmlParser(outPut))
{
HtmlChunk chunk = parser.ParseNext();
Assert.IsNotNull(chunk);
// checks if the HTML tag is of type div and if it has the required attributes
Assert.IsTrue(chunk.TagName.Equals(this.htmlWrapperTag, StringComparison.Ordinal));
Assert.IsTrue(chunk.HasAttribute(this.idAttribute), "The id of the item is not appended as attribute correctly.");
Assert.IsTrue(chunk.HasAttribute(this.providerAttribute), "The provider is not appended as attribute correctly.");
Assert.IsTrue(chunk.HasAttribute(this.typeAttribute), "The id type the item is not appended as attribute correctly.");
// checks if the required attributes has proper values assigned to them
Assert.AreEqual(providerName, chunk.GetParamValue(this.providerAttribute), "The value of the provider attribute is not correct.");
Assert.AreEqual(type, chunk.GetParamValue(this.typeAttribute), "The value of the provider attribute is not correct.");
Assert.AreEqual(id.ToString(), chunk.GetParamValue(this.idAttribute), "The value of the id attribute is not correct.");
this.AssertContentAndCloseTag(parser);
}
}
public void GetAttributeValue_GetTheValueOfTheClassAttribute_VerifyTheMethodReturnsTheProperAttributeValue()
{
//Arrange: Initialize the GridControl, create a fake HTML template with attributes
var layoutControl = new DummyGridControl();
string expectedAttributeValue = "sf_colsOut";
string actualAttributeValue = string.Empty;
string attributeName = "class";
var template = string.Format(@"<div {0}=""{1}"" runat=""server""></div>", attributeName, expectedAttributeValue);
//Act: parse the HTML template and then get the value of the class attribute
using (HtmlParser parser = new HtmlParser(template))
{
parser.SetChunkHashMode(false);
parser.AutoExtractBetweenTagsOnly = false;
parser.CompressWhiteSpaceBeforeTag = false;
parser.KeepRawHTML = true;
HtmlChunk chunk = parser.ParseNext();
actualAttributeValue = layoutControl.PublicGetAttributeValue(chunk, attributeName);
}
//Assert: Verify the GetAttributeValue of the GridControl class is returning the correct attribute value
Assert.AreEqual(expectedAttributeValue, actualAttributeValue, "The attribute value returned by the GetAttributeValue method is not correct.");
}
// End Of Function Home PAge Search
public static List<string> FolderSearch(Uri uriObj1)
{
List<string> returnString = new List<string>();
HtmlTag tag;
CookieAwareWebClient client = new CookieAwareWebClient();
client.CookieContainer = Web.CC;
string html = client.DownloadString(uriObj1);
HtmlParser parse = new HtmlParser(html);
string returnedValue = "";
string value1;
while (parse.ParseNext("a", out tag))
{
if (tag.Attributes.TryGetValue("href", out value1))
{
returnedValue = value1;
if((searchKeyword(returnedValue, "http://10.1.1.242/moodle/mod/folder/")&&(returnedValue!="http://10.1.1.242/moodle/course/index.php")))
{
returnString.Add(returnedValue);
}
}
}
return returnString;
}
private void AddContent(HtmlParser htmlParser, string htmlContent)
{
ElementState elementState = htmlParser.CurrentElementState;
htmlContent = replaceMultipleWhiteSpacesWithSingleWhitespaceRegex.Replace(htmlContent, " ");
string decodedHtml = HtmlParser.UrlDecode(htmlContent);
switch (elementState.TypeName)
{
case "a":
{
elementsUnderConstruction.Push(new FlowLayoutWidget());
elementsUnderConstruction.Peek().Name = "a";
if (decodedHtml != null && decodedHtml != "")
{
Button linkButton = linkButtonFactory.Generate(decodedHtml.Replace("\r\n", "\n"));
StyledTypeFace styled = new StyledTypeFace(LiberationSansFont.Instance, elementState.PointSize);
double descentInPixels = styled.DescentInPixels;
linkButton.OriginRelativeParent = new VectorMath.Vector2(linkButton.OriginRelativeParent.x, linkButton.OriginRelativeParent.y + descentInPixels);
linkButton.Click += (sender, mouseEvent) =>
{
MatterControlApplication.Instance.LaunchBrowser(elementState.Href);
};
elementsUnderConstruction.Peek().AddChild(linkButton);
}
}
break;
case "h1":
case "p":
{
elementsUnderConstruction.Push(new FlowLayoutWidget());
elementsUnderConstruction.Peek().Name = "p";
elementsUnderConstruction.Peek().HAnchor = HAnchor.ParentLeftRight;
if (decodedHtml != null && decodedHtml != "")
{
WrappingTextWidget content = new WrappingTextWidget(decodedHtml, pointSize: elementState.PointSize, textColor: ActiveTheme.Instance.PrimaryTextColor);
//content.VAnchor = VAnchor.ParentTop;
elementsUnderConstruction.Peek().AddChild(content);
}
}
break;
case "div":
{
elementsUnderConstruction.Push(new FlowLayoutWidget());
elementsUnderConstruction.Peek().Name = "div";
if (decodedHtml != null && decodedHtml != "")
{
TextWidget content = new TextWidget(decodedHtml, pointSize: elementState.PointSize, textColor: ActiveTheme.Instance.PrimaryTextColor);
elementsUnderConstruction.Peek().AddChild(content);
}
}
break;
case "!DOCTYPE":
break;
case "body":
break;
case "img":
{
ImageBuffer image = new ImageBuffer(elementState.SizeFixed.x, elementState.SizeFixed.y, 32, new BlenderBGRA());
ImageWidget_AsyncLoadOnDraw imageWidget = new ImageWidget_AsyncLoadOnDraw(image, elementState.src);
// put the image into the widget when it is done downloading.
if (elementsUnderConstruction.Peek().Name == "a")
{
Button linkButton = new Button(0, 0, imageWidget);
linkButton.Cursor = Cursors.Hand;
linkButton.Click += (sender, mouseEvent) =>
{
MatterControlApplication.Instance.LaunchBrowser(elementState.Href);
};
elementsUnderConstruction.Peek().AddChild(linkButton);
}
else
{
elementsUnderConstruction.Peek().AddChild(imageWidget);
}
}
break;
case "input":
break;
case "table":
break;
case "td":
case "span":
GuiWidget widgetToAdd;
if (elementState.Classes.Contains("translate"))
{
decodedHtml = decodedHtml.Localize();
}
if (elementState.Classes.Contains("toUpper"))
{
decodedHtml = decodedHtml.ToUpper();
}
if (elementState.Classes.Contains("versionNumber"))
{
decodedHtml = VersionInfo.Instance.ReleaseVersion;
}
if (elementState.Classes.Contains("buildNumber"))
{
decodedHtml = VersionInfo.Instance.BuildVersion;
}
Button createdButton = null;
if (elementState.Classes.Contains("centeredButton"))
{
createdButton = textImageButtonFactory.Generate(decodedHtml);
widgetToAdd = createdButton;
}
else if (elementState.Classes.Contains("linkButton"))
{
double oldFontSize = linkButtonFactory.fontSize;
linkButtonFactory.fontSize = elementState.PointSize;
createdButton = linkButtonFactory.Generate(decodedHtml);
StyledTypeFace styled = new StyledTypeFace(LiberationSansFont.Instance, elementState.PointSize);
double descentInPixels = styled.DescentInPixels;
createdButton.OriginRelativeParent = new VectorMath.Vector2(createdButton.OriginRelativeParent.x, createdButton.OriginRelativeParent.y + descentInPixels);
widgetToAdd = createdButton;
linkButtonFactory.fontSize = oldFontSize;
}
else
{
TextWidget content = new TextWidget(decodedHtml, pointSize: elementState.PointSize, textColor: ActiveTheme.Instance.PrimaryTextColor);
widgetToAdd = content;
}
if (createdButton != null)
{
if (elementState.Id == "sendFeedback")
{
createdButton.Click += (s, e) => ContactFormWindow.Open();
}
else if (elementState.Id == "clearCache")
{
createdButton.Click += (s, e) => AboutWidget.DeleteCacheData(0);
}
}
if (elementState.VerticalAlignment == ElementState.VerticalAlignType.top)
{
widgetToAdd.VAnchor = VAnchor.ParentTop;
}
elementsUnderConstruction.Peek().AddChild(widgetToAdd);
break;
case "tr":
elementsUnderConstruction.Push(new FlowLayoutWidget());
elementsUnderConstruction.Peek().Name = "tr";
if (elementState.SizePercent.y == 100)
{
elementsUnderConstruction.Peek().VAnchor = VAnchor.ParentBottomTop;
}
if (elementState.Alignment == ElementState.AlignType.center)
{
elementsUnderConstruction.Peek().HAnchor |= HAnchor.ParentCenter;
}
break;
default:
throw new NotImplementedException("Don't know what to do with '{0}'".FormatWith(elementState.TypeName));
}
}
/// <summary>
/// URLからHTMLを取得し、指定した文字列を含むURLがあるかどうか調べる
/// </summary>
/// <param name="url">URL</param>
/// <param name="currentDepth">現在の深度（何回目の再帰呼び出しか）</param>
private void SearchLinks(string url, int currentDepth)
{
// 探索済みにする
crawledUrlList.Add(url);
// 最大深度を超えたらリターンする
if (currentDepth > limitDepth)
{
return;
}
// 最初に指定されたURLに文字列があるかチェック
if (url.Contains(searchString))
{
isFound = true;
foundUrl = url;
return;
}
WebClient client = new WebClient();
string htmlString;
try
{
// 指定したURLから、HTMLデータを文字列として取得する
htmlString = client.DownloadString(url);
}
catch (WebException ex)
{
// 指定したURLにはアクセスできなかったのでリターンする
return;
}
// 取得したHtmlの文字列をパースする
var parser = new HtmlParser();
var document = parser.Parse(htmlString);
List <string> linkList = new List <string>();
string hrefValue = string.Empty;
Url baseUrl = new Url(url);
// ページ内のリンクをすべて取得する
foreach (var item in document.QuerySelectorAll("a"))
{
hrefValue = item.Attributes["href"]?.Value;
// href属性が存在していなかったら追加しない
if (hrefValue == null)
{
continue;
}
// すでに追加済みなら追加しない
if (linkList.Contains(hrefValue))
{
continue;
}
// すでに調査済みなら追加しない
if (crawledUrlList.Contains(new Url(baseUrl, hrefValue).ToString()))
{
continue;
}
linkList.Add(item.Attributes["href"].Value);
}
// リンクが無いなら、リターンする。
if (!linkList.Any())
{
return;
}
// 取得したリンクのURL内に、目的の文字列が含まれているか
//（ searchStringが日本語の場合を考慮して、Urlインスタンスを作成し、URLエンコードしている）
string str = linkList.Where(x => x.Contains(new Url(searchString).ToString())).FirstOrDefault();
// 一致するものが見つかる
if (str != null)
{
// 経路スタックにurlをpushする
urlPathStack.Push(url);
// 検索結果を保存
foundUrl = new Url(baseUrl, str).ToString();
isFound = true;
return;
}
else
{
// 一致するものが見つからなかったので、Link先を辿っていく
string subLink;
foreach (var link in linkList)
{
subLink = new Url(baseUrl, link).ToString();
// すでに探索済み
if (crawledUrlList.Contains(subLink))
{
continue;
}
// urlを変更して再帰呼び出し
SearchLinks(new Url(baseUrl, subLink).ToString(), (currentDepth + 1));
// 見つかったらスタックに入れて終了
if (isFound)
{
urlPathStack.Push(url);
return;
}
}
}
}
protected void Update()
{
_buttonActions.Clear();
_contentContainer.RemoveAllChildren ();
_pos=0;
_line=null;
_textNotRendered=null;
_parsingY=0;
_charCount=0;
_lines=new List<List<LinePiece>>();
bool lineSplit=false; //set to true when, for example, a style is changed and requires the line to be split in several pieces
HtmlTag tag;
HtmlParser parse = new HtmlParser(_text);
while (parse.ParseNext("*", true, out tag)) {
//Debug.Log ("found tag = ["+_text.Substring (tag.startPos, tag.endPos - tag.startPos)+"] tag.TrailingSlash="+tag.TrailingSlash+" tag.Name="+tag.Name);
Dictionary<string, string> styleToPush=null;
bool styleToPop=false;
if (!tag.Closing) {
if (!tag.TrailingSlash) {
if (tag.Name.Equals("style")) {
//Push new style, break line
styleToPush=new Dictionary<string, string>(_stylesStack.Last());
//_stylesStack.Add(style);
foreach (KeyValuePair<string, string> pair in tag.Attributes)
{
string defaultValue;
styleToPush.TryGetValue(pair.Key,out defaultValue);
if (defaultValue!=null) {
if (!defaultValue.Equals(pair.Value)) {
styleToPush.Remove(pair.Key);
styleToPush.Add(pair.Key,pair.Value);
lineSplit=true;
}
} else {
styleToPush.Add(pair.Key,pair.Value);
lineSplit=true;
}
}
}
}
} else {
if (tag.Name.Equals("style")) {
styleToPop=true;
//_stylesStack.Pop();
lineSplit=true;
}
}
Render(tag.startPos,lineSplit);
lineSplit=false;
if (styleToPush!=null) {
_stylesStack.Add(styleToPush);
} else {
if (styleToPop) {
_stylesStack.Pop();
}
}
if (tag.TrailingSlash) {
if (tag.Name.Equals("br")) {
//Debug.Log ("br");
if (_textNotRendered==null) _textNotRendered="";
RenderPiece(_textNotRendered); _textNotRendered=null;
RenderLine();
} else if (tag.Name.Equals("fsprite")) {
//Create a FSprite
string val=null;
StringAttributeParam("src","fsprite",tag.Attributes,ref val);
if (val!=null) {
FSprite sprite=new FSprite(val);
ApplyStyles(sprite,tag.Attributes);
RenderPiece(sprite);
}
} else if (tag.Name.Equals("fbutton")) {
//Create a FButton
string up=null;
StringAttributeParam("src","fbutton",tag.Attributes,ref up);
if (up!=null) {
string down=up;
StringAttributeParam("down","fbutton",tag.Attributes,ref down);
string over=null;
StringAttributeParam("over","fbutton",tag.Attributes,ref over);
string sound=null;
StringAttributeParam("sound","fbutton",tag.Attributes,ref sound);
FButton button=new FButton(up,down,over,sound);
ApplyStyles(button,tag.Attributes);
RenderPiece(button);
}
}
}
_pos=tag.endPos; //skipping tag
}
Render(_text.Length,lineSplit);
lineSplit=false; //useless, but that's how I do things
//Last line?
RenderPiece(_textNotRendered); _textNotRendered=null;
RenderLine();
//center the text (positio 0,0 is in the moddile of the text box)
_contentContainer.y=-_parsingY*0.5f;
_width=_maxWidth;
_height=-_parsingY;
//All visible
_startVisibleCharIdx=0;
_endVisibleCharIdx=_charCount;
}
/// <summary>
/// Parses the HTML content of the country profile data and converts them to a list of profile entities
/// </summary>
/// <param name="content">HTML content of the country profile data</param>
/// <returns></returns>
public static List <ProfileEntity> ParseProfileData(string content)
{
List <ProfileEntity> entities = new List <ProfileEntity>();
var parser = new HtmlParser();
var doc = parser.ParseDocument(content);
//Category Lists
var cat_lis = doc.QuerySelectorAll("li").Where(x =>
x.HasAttribute("id") && x.GetAttribute("id").EndsWith("-category-section-anchor"));
foreach (var cat_li in cat_lis)
{
var catElem = cat_li.Children.Where(x => x.LocalName.ToLower() == "div" && x.HasAttribute("sectiontitle")).First();
ProfileEntity Category = new ProfileEntity()
{
EntityType = ProfileEntityType.Category,
Key = catElem.GetAttribute("sectiontitle"),
Children = new List <ProfileEntity> ()
};
var field_divs = doc.QuerySelectorAll("div").Where(x =>
x.LocalName.ToLower() == "div" && x.HasAttribute("id") &&
x.GetAttribute("id").StartsWith($"field-anchor-{Category.Key.Trim().Replace(" ", "-").ToLower()}-"));
foreach (var field_div in field_divs)
{
string fieldname = field_div.GetAttribute("id").Trim();
fieldname = fieldname.Replace($"-anchor-{Category.Key.Trim().Replace(" ", "-").ToLower()}", "");
ProfileEntity Field = new ProfileEntity()
{
EntityType = ProfileEntityType.Field,
Key = field_div.TextContent.Trim(new char[] { ':' }).Replace(":", "").Trim(),
Children = new List <ProfileEntity> ()
};
//Get field content div
var field_content_div = doc.QuerySelectorAll("div").Where(x => x.HasAttribute("id") && x.GetAttribute("id").Trim() == fieldname).First();
//Iterate through sub-fields
//--------------------------
//1.0 Check for Notes - directly for the field
var notes = field_content_div.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class").Trim() == "category_data note");
foreach (var note in notes)
{
Field.Note += note.TextContent.Replace("note:", "").Trim() + "\n";
}
//2.0 Go through subfields
foreach (var subfield in field_content_div.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class").Contains("subfield") &&
!x.GetAttribute("class").Contains("note")))
{
//2.1 Check if the sub-filed has a direct value
if (subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class").Contains("subfield")).Count() == 0)
{
Field.Value += subfield.TextContent + "\n";
}
else //3.2 Contains sub-fields
{
ProfileEntity SubField = new ProfileEntity()
{
EntityType = ProfileEntityType.SubField
};
//3.2.1 Handle historic fields
if (subfield.GetAttribute("class").Contains("historic"))
{
if (subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class").Contains("subfield-name")).Count() != 0) //Sub Field is Numeric
{
SubField.Key = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-name").First().TextContent.Replace(":", "").Trim();
}
else //Historic sub field with no key
{
SubField.Key = "*";
}
SubField.IsHistoricEntity = true;
SubField.Value = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-number").FirstOrDefault()?.TextContent ?? "";
SubField.Note = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-note").FirstOrDefault()?.TextContent ?? "";
SubField.Date = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-date").FirstOrDefault()?.TextContent ?? "";
}
//3.2.2 Handle numeric fields
else if (subfield.GetAttribute("class").Contains("numeric"))
{
if (subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class").Contains("subfield-name")).Count() != 0) //Sub Field is Numeric
{
SubField.IsNumericEntity = true;
SubField.Key = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-name").First().TextContent.Replace(":", "").Trim();
SubField.Value = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-number").FirstOrDefault()?.TextContent ?? "";
SubField.Note = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-note").FirstOrDefault()?.TextContent ?? "";
SubField.Date = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-date").FirstOrDefault()?.TextContent ?? "";
}
else //Parent field is numeric
{
Field.IsNumericEntity = true;
Field.Value = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-number").FirstOrDefault()?.TextContent ?? "";
Field.Note = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-note").FirstOrDefault()?.TextContent ?? "";
Field.Date = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-date").FirstOrDefault()?.TextContent ?? "";
continue;
}
}
//3.3.4 Handle grouped subfield case
else if (subfield.GetAttribute("class").Contains("grouped_subfield"))
{
if (subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class").Contains("subfield-name")).Count() != 0) //Sub Field is Numeric
{
SubField.IsGroupedEntity = true;
SubField.Key = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-name").First().TextContent.Replace(":", "").Trim();
SubField.Value = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-number").First().TextContent;
SubField.Note = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-note").FirstOrDefault()?.TextContent ?? "";
SubField.Date = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-date").FirstOrDefault()?.TextContent ?? "";
}
else //Parent field is grouped
{
Field.IsGroupedEntity = true;
Field.Value = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-number").First().TextContent;
Field.Note = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-note").FirstOrDefault()?.TextContent ?? "";
Field.Date = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-date").FirstOrDefault()?.TextContent ?? "";
continue;
}
}
//3.3.4 Handle text cases
else if (subfield.GetAttribute("class").Contains("text"))
{
if (subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class").Contains("subfield-name")).Count() != 0) //Is a sub filed
{
var keyNode = subfield.Children.Where(x => x.GetAttribute("class") == "subfield-name").First();
SubField.Key = keyNode.TextContent.Trim(new char[] { ':', ' ' });
subfield.RemoveChild(keyNode);
SubField.Value = subfield.TextContent.Trim();
}
else
{
string note = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-note").FirstOrDefault()?.TextContent.Trim() ?? "";
string date = subfield.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-date").FirstOrDefault()?.TextContent.Trim() ?? "";
string val = subfield.TextContent.Trim();
if (note != string.Empty)
{
val.Replace(note, "");
}
if (date != string.Empty)
{
val.Replace(date, "");
}
Field.Value += val + "\n";
Field.Note += note + "\n";
Field.Date += date + "\n";
continue;
}
}
SubField.Value = SubField.Value.Trim();
SubField.Note = SubField.Note.Trim();
SubField.Date = SubField.Date.Trim();
Field.Children.Add(SubField);
}
}
//Check for country comparison for the field
AngleSharp.Dom.IElement compareField = null;
if ((compareField = field_content_div.Children.Where(x => x.LocalName.ToLower() == "div" && !x.HasAttribute("class") && x.TextContent.ToLower().Contains("country comparison to the world")).FirstOrDefault()) != null)
{
int rank = 0;
if (int.TryParse(compareField.Children.Where(x => x.GetAttribute("class") == "category_data").FirstOrDefault()?.TextContent ?? "".Trim(), out rank))
{
Field.ComparisonRank = rank;
}
}
Field.Value = Field.Value.Trim();
Field.Note = Field.Note.Trim();
Field.Date = Field.Date.Trim();
Category.Children.Add(Field);
}
entities.Add(Category);
}
return(entities);
}
public static async Task DownloadArticleContent(Article article)
{
String html = await Downloader.GetAsync(article.Url);
article.Content = await HtmlParser.ParseArticleContentAsync(html, article.Url);
}
public static void GetStructure(string content)
{
var parser = new HtmlParser();
var doc = parser.ParseDocument(content);
var bgelements = doc.QuerySelectorAll("div").Where(x =>
x.HasAttribute("sectiontitle") ||
(x.HasAttribute("id") &&
x.GetAttribute("id").StartsWith("field-") &&
x.GetAttribute("id").Contains("anchor")));
ConsoleColor col = Console.ForegroundColor;
Console.Clear();
string currentcategory = string.Empty;
foreach (var elem in bgelements)
{
string title = elem.TextContent.Trim(new char[] { ':' }).Trim();
if (elem.HasAttribute("sectiontitle"))
{
Console.ForegroundColor = ConsoleColor.Red;
currentcategory = elem.GetAttribute("sectiontitle").Trim();
currentcategory = currentcategory.Replace(" ", "-").ToLower();
Console.WriteLine($" + {elem.GetAttribute("sectiontitle").Trim()}");
}
else if (elem.GetAttribute("id").StartsWith($"field-anchor-{currentcategory}-"))
{
Console.ForegroundColor = ConsoleColor.Yellow;
string fieldname = elem.GetAttribute("id");
fieldname = fieldname.Replace($"anchor-{currentcategory}-", string.Empty).Trim();
Console.WriteLine($" \t- {title}");
//Check for sub-fields
//var fieldContent = doc.QuerySelectorAll("div").Where(x => x.HasAttribute("id")
//&& x.GetAttribute("id") == fieldname).First().Children
//.Where(y => y.HasAttribute("class")
//&& y.GetAttribute("class").Contains("category_data")
//&& (y.GetAttribute("class").Contains("text") || y.GetAttribute("class").Contains("note")
//|| y.GetAttribute("class").Contains("numeric") || y.GetAttribute("class").Contains("historic")
//));
var fieldContent = doc.QuerySelectorAll("div").Where(x => x.HasAttribute("id") &&
x.GetAttribute("id") == fieldname).First().Children
.Where(y => y.HasAttribute("class") && y.GetAttribute("class").Contains("subfield"));
if (fieldContent.Count() != 0)
{
foreach (var item in fieldContent)
{
var subFields = item.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-name");
foreach (var subfield in subFields)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($" \t\t* {subfield.TextContent}");
}
if (item.GetAttribute("class").Contains("historic"))
{
subFields = item.Children.Where(x => x.HasAttribute("class") && x.GetAttribute("class") == "subfield-date");
foreach (var subfield in subFields)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($" \t\t* {subfield.TextContent} [historic]");
}
}
}
}
//Indentify additional notes
var noteContent = doc.QuerySelectorAll("div").Where(x => x.HasAttribute("id") &&
x.GetAttribute("id") == fieldname).First().Children
.Where(y => y.HasAttribute("class") && y.GetAttribute("class").Contains("note"));
foreach (var noteelement in noteContent)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($" \t\t* note:");
}
//Identify comparison
var compareContent = doc.QuerySelectorAll("div").Where(x => x.HasAttribute("id") &&
x.GetAttribute("id") == fieldname).First().Children
.Where(y => y.TextContent.ToLower().Contains("country comparison to the world"));
if (compareContent.Count() == 1)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($" \t\t* country comparison to the world:");
}
}
}
Console.ForegroundColor = col;
}
public HtmlXamlGenerator(string html)
{
_parser = new HtmlParser();
_html = PrepareRawHtml(html);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = "--";
var maxPages = 2; // we scrape only 2 pages for recent torrents
if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
{
searchString = Uri.EscapeUriString(query.GetQueryString());
maxPages = MaxSearchPageLimit;
}
var lastPublishDate = DateTime.Now;
for (var page = 0; page < maxPages; page++)
{
var searchUrl = string.Format(SearchUrl, page * MaxItemsPerPage, searchString);
var result = await RequestWithCookiesAsync(searchUrl, headers : _apiHeaders);
try
{
var json = JsonConvert.DeserializeObject <dynamic>(result.ContentString);
var parser = new HtmlParser();
var doc = parser.ParseDocument((string)json["contenido"]);
var rows = doc.QuerySelectorAll("div.span2");
foreach (var row in rows)
{
var title = row.QuerySelector("h2").TextContent + " - " +
row.QuerySelector("h1").TextContent;
if (!CheckTitleMatchWords(query.GetQueryString(), title))
{
continue; // skip if it doesn't contain all words
}
var poster = new Uri(row.QuerySelector("img[id=catalog]").GetAttribute("src"));
var qLink = row.QuerySelector("a");
var details = new Uri(qLink.GetAttribute("href"));
var qTooltip = parser.ParseDocument(qLink.GetAttribute("data-content"));
// we get the language from the last class tag => class="pull-right sprite idioma_5"
var languageId = qTooltip.QuerySelector("div.pull-right").GetAttribute("class").Split('_')[1];
title += $" [{_languages[languageId]}] [epub]";
var qDesc = qTooltip.QuerySelectorAll("div.row-fluid > div");
var description = $"Rev: {qDesc[0].TextContent} Páginas: {qDesc[1].TextContent} Puntación: {qDesc[2].TextContent} Likes: {qDesc[3].TextContent}";
// publish date is not available in the torrent list, but we add a relative date so we can sort
lastPublishDate = lastPublishDate.AddMinutes(-1);
var release = new ReleaseInfo
{
Title = title,
Details = details,
Link = details,
Guid = details,
PublishDate = lastPublishDate,
Poster = poster,
Description = description,
Category = new List <int> {
TorznabCatType.BooksEBook.ID
},
Size = 5242880, // 5 MB
Seeders = 1,
Peers = 2,
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1
};
releases.Add(release);
}
if (rows.Length < MaxItemsPerPage)
{
break; // this is the last page
}
}
catch (Exception ex)
{
OnParseError(result.ContentString, ex);
}
}
return(releases);
}
public void DelayedInitialize()
{
var htmlParser = new HtmlParser();
foreach (var highlighter in Highlighters)
highlighter.Parser = htmlParser;
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var qc = new List <KeyValuePair <string, string> >(); // NameValueCollection don't support c[]=30&c[]=52
if (query.IsImdbQuery)
{
qc.Add("search", query.ImdbID);
qc.Add("d", "on");
}
else
{
qc.Add("search", query.GetQueryString());
}
var catList = MapTorznabCapsToTrackers(query);
foreach (var cat in catList)
{
qc.Add("c[]", cat);
}
var searchUrl = SearchUrl + "?" + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(searchUrl);
if (!response.Content.Contains("/logout.php")) // re-login
{
await DoLogin();
response = await RequestStringWithCookiesAndRetry(searchUrl);
}
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.Content);
var rows = dom.QuerySelectorAll("div.boxContent > table > tbody > tr");
foreach (var row in rows)
{
var cells = row.QuerySelectorAll("td");
var title = row.QuerySelector("td[class='lft'] > div > a").TextContent.Trim();
var link = new Uri(SiteLink + row.QuerySelector("img[title='Download']").ParentElement.GetAttribute("href").TrimStart('/'));
var comments = new Uri(SiteLink + row.QuerySelector("td[class='lft'] > div > a").GetAttribute("href").TrimStart('/'));
var size = ReleaseInfo.GetBytes(cells[4].TextContent);
var grabs = ParseUtil.CoerceInt(cells[5].TextContent);
var seeders = ParseUtil.CoerceInt(cells[6].TextContent);
var leechers = ParseUtil.CoerceInt(cells[7].TextContent);
var pubDateStr = row.QuerySelector("span[class^='elapsedDate']").GetAttribute("title").Replace(" at", "");
var publishDate = DateTime.ParseExact(pubDateStr, "dddd, MMMM d, yyyy h:mmtt", CultureInfo.InvariantCulture);
var cat = row.QuerySelector("a[href^='?c[]=']").GetAttribute("href").Replace("?c[]=", "");
var downloadVolumeFactor = row.QuerySelector("span:contains(\"[Freeleech]\")") != null ? 0 : 1;
var release = new ReleaseInfo
{
Title = title,
Link = link,
Guid = link,
Comments = comments,
PublishDate = publishDate,
Category = MapTrackerCatToNewznab(cat),
Size = size,
Grabs = grabs,
Seeders = seeders,
Peers = seeders + leechers,
MinimumRatio = 1,
MinimumSeedTime = 172800, // 48 hours
DownloadVolumeFactor = downloadVolumeFactor,
UploadVolumeFactor = 1
};
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.Content, ex);
}
return(releases);
}
public override async Task <IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var pairs = new Dictionary <string, string> {
{ "username", configData.Username.Value },
{ "password", configData.Password.Value },
{ "g-recaptcha-response", configData.Captcha.Value }
};
if (!string.IsNullOrWhiteSpace(configData.Captcha.Cookie))
{
// Cookie was manually supplied
CookieHeader = configData.Captcha.Cookie;
try
{
var results = await PerformQuery(new TorznabQuery());
if (!results.Any())
{
throw new Exception("no results found, please report this bug");
}
IsConfigured = true;
SaveConfig();
return(IndexerConfigurationStatus.Completed);
}
catch (Exception e)
{
IsConfigured = false;
throw new Exception("Your cookie did not work: " + e.Message);
}
}
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, configData.CookieHeader.Value, true, null, LoginUrl);
await ConfigureIfOK(result.Cookies, result.Content?.Contains("logout.php") == true, () =>
{
var errorMessage = result.Content;
var parser = new HtmlParser();
var dom = parser.ParseDocument(result.Content);
var messageEl = dom.QuerySelector("#login");
if (messageEl != null)
{
foreach (var child in messageEl.QuerySelectorAll("form"))
{
child.Remove();
}
errorMessage = messageEl.TextContent.Trim();
}
if (string.IsNullOrWhiteSpace(errorMessage) && result.IsRedirect)
{
errorMessage = $"Got a redirect to {result.RedirectingTo}, please adjust your the alternative link";
}
throw new ExceptionWithConfigData(errorMessage, configData);
});
return(IndexerConfigurationStatus.RequiresTesting);
}
public YahooWebSearchCounter(
HtmlParser parser,
PageCrawler pageCrawler)
: base(parser, pageCrawler)
{
}
public ResponseDto Post(FaceDataRequestDto[] faceDto)
{
//request is array of factDto data - we are dealing with Single face id only - so take first or default
var faceInfo = faceDto.FirstOrDefault();
//determine male or female
var gender = faceInfo.faceAttributes.gender == GenericConstantData.Female ? 0 : 1;
//having specs or glasses or none
var glass = faceInfo.faceAttributes.glasses;
var glassType = GenericConstantData.NoGlasses;
if (glass.StartsWith("Reading"))
{
glassType = CategoryCode.Specs;
}
else if (glass.StartsWith("Sun"))
{
glassType = CategoryCode.SunGlass;
}
//beard??
var hasBeard = gender > 0 && faceInfo.faceAttributes.facialHair.beard >= 0.4;
//read all data from GenderData Table
using (var context = new makeoverEntities())
{
var genderData = context.GenderDatas.Where(g => g.Gender == gender);
////////// PRIORITY ///////////////////
//1 - Smile takes high priority - CODE here for smile
//2 - No smile - Male
// ---------- Has Beard?
if (gender > 0 && hasBeard)
{
genderData = genderData.Where(g => g.CategoryCode == CategoryCode.Beard);
}
// ---------- Has No Beard? - Go for glass check
if (gender > 0 && !hasBeard)
{
//3 -- check for glasses
if (glassType != GenericConstantData.NoGlasses)
{
genderData = genderData.Where(g => g.CategoryCode == glassType);
}
else
{
//4--- no glasses return watches (general category)
genderData = genderData.Where(g => g.CategoryCode == CategoryCode.General);
}
}
//2 - No smile - Female
if (gender == 0)
{
//3 --check for glasses
if (glassType != GenericConstantData.NoGlasses)
{
genderData = genderData.Where(g => g.CategoryCode == glassType);
}
else
{
//4--- no glasses return watches (general category)
genderData = genderData.Where(g => g.CategoryCode == CategoryCode.General); //TODO once smile is taken care - change to gen category
}
}
var firstMatch = genderData.FirstOrDefault();
//return dynamic response here
HtmlParser parser = new HtmlParser();
var suggestionList = parser.GetBestSellers(firstMatch.ShortUrl);
//TODO: if suggestionList is empty then return default prodlist that is working - WATCH for men, or ReadingGlass for women
return(new ResponseDto()
{
faceId = faceInfo.faceId,
prodlist = suggestionList
});
}
}
public override async Task <IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
if (useApiKey)
{
var apiKey = configData.ApiKey;
if (apiKey?.Value == null)
{
throw new Exception("Invalid API Key configured");
}
if (apiKey.Value.Length != 41)
{
throw new Exception($"Invalid API Key configured: expected length: 41, got {apiKey.Value.Length}");
}
try
{
var results = await PerformQuery(new TorznabQuery());
if (!results.Any())
{
throw new Exception("Found 0 results in the tracker");
}
IsConfigured = true;
SaveConfig();
return(IndexerConfigurationStatus.Completed);
}
catch (Exception e)
{
IsConfigured = false;
throw new Exception($"Your API Key did not work: {e.Message}");
}
}
var pairs = new Dictionary <string, string> {
{ "username", configData.Username.Value },
{ "password", configData.Password.Value },
{ "keeplogged", "1" }
};
if (!string.IsNullOrWhiteSpace(cookie))
{
// Cookie was manually supplied
CookieHeader = cookie;
try
{
var results = await PerformQuery(new TorznabQuery());
if (!results.Any())
{
throw new Exception("Found 0 results in the tracker");
}
IsConfigured = true;
SaveConfig();
return(IndexerConfigurationStatus.Completed);
}
catch (Exception e)
{
IsConfigured = false;
throw new Exception($"Your cookie did not work: {e.Message}");
}
}
var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, string.Empty, true, SiteLink);
await ConfigureIfOK(response.Cookies, response.ContentString != null && response.ContentString.Contains("logout.php"), () =>
{
var loginResultParser = new HtmlParser();
var loginResultDocument = loginResultParser.ParseDocument(response.ContentString);
var loginform = loginResultDocument.QuerySelector("#loginform");
if (loginform == null)
{
throw new ExceptionWithConfigData(response.ContentString, configData);
}
loginform.QuerySelector("table").Remove();
var errorMessage = loginform.TextContent.Replace("\n\t", " ").Trim();
throw new ExceptionWithConfigData(errorMessage, configData);
});
return(IndexerConfigurationStatus.RequiresTesting);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var queryCollection = new NameValueCollection
{
{ "active", "0" },
{ "category", string.Join(";", MapTorznabCapsToTrackers(query)) }
};
if (query.IsImdbQuery)
{
queryCollection.Add("options", "2");
queryCollection.Add("search", query.ImdbIDShort);
}
else
{
queryCollection.Add("options", "0");
queryCollection.Add("search", query.GetQueryString());
}
var response = await RequestWithCookiesAndRetryAsync(SearchUrl + queryCollection.GetQueryString());
try
{
var resultParser = new HtmlParser();
var searchResultDocument = resultParser.ParseDocument(response.ContentString);
var rows = searchResultDocument.QuerySelectorAll("table.lista > tbody > tr");
foreach (var row in rows)
{
// this tracker has horrible markup, find the result rows by looking for the style tag before each one
var prev = row.PreviousElementSibling;
if (prev == null || !string.Equals(prev.NodeName, "style", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 86400; // 24 hours
var qLink = row.Children[1].FirstElementChild;
release.Title = qLink.TextContent.Trim();
release.Details = new Uri(SiteLink + qLink.GetAttribute("href"));
release.Guid = release.Details;
var imdbLink = row.Children[1].QuerySelector("a[href*=imdb]");
if (imdbLink != null)
{
release.Imdb = ParseUtil.GetImdbID(imdbLink.GetAttribute("href").Split('/').Last());
}
var qDownload = row.Children[3].FirstElementChild;
release.Link = new Uri(SiteLink + qDownload.GetAttribute("href"));
var dateStr = row.Children[4].TextContent.Trim();
//"July 11, 2015, 13:34:09", "Today|Yesterday at 20:04:23"
release.PublishDate = DateTimeUtil.FromUnknown(dateStr);
var sizeStr = row.Children[5].TextContent;
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(row.Children[7].TextContent);
release.Peers = ParseUtil.CoerceInt(row.Children[8].TextContent) + release.Seeders;
var grabs = row.QuerySelector("td:nth-child(10)").TextContent;
grabs = grabs.Replace("---", "0");
release.Grabs = ParseUtil.CoerceInt(grabs);
if (row.QuerySelector("img[title=\"FreeLeech\"]") != null)
{
release.DownloadVolumeFactor = 0;
}
else if (row.QuerySelector("img[src=\"images/sf.png\"]") != null) // side freeleech
{
release.DownloadVolumeFactor = 0;
}
else if (row.QuerySelector("img[title=\"Half FreeLeech\"]") != null)
{
release.DownloadVolumeFactor = 0.5;
}
else
{
release.DownloadVolumeFactor = 1;
}
release.UploadVolumeFactor = 1;
var qCat = row.QuerySelector("a[href^=\"index.php?page=torrents&category=\"]");
var cat = qCat.GetAttribute("href").Split('=')[2];
release.Category = MapTrackerCatToNewznab(cat);
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(response.ContentString, ex);
}
return(releases);
}
/// <summary>
/// Processes the layout string adding the required attributes to the head tag
/// and also adding the required form tag.
/// </summary>
/// <param name="template">The template.</param>
/// <returns></returns>
public virtual string ProcessLayoutString(string template)
{
var includeFormTag = this.IsFormTagRequired();
StringBuilder outPut = new StringBuilder();
HtmlChunk chunk = null;
using (HtmlParser parser = new HtmlParser(template))
{
parser.SetChunkHashMode(false);
parser.AutoExtractBetweenTagsOnly = false;
parser.CompressWhiteSpaceBeforeTag = false;
parser.KeepRawHTML = true;
bool setTitle = true;
bool modified;
bool isOpenBodyTag;
bool isCloseBodyTag;
bool isClosedHeadTag;
while ((chunk = parser.ParseNext()) != null)
{
modified = false;
isOpenBodyTag = false;
isCloseBodyTag = false;
isClosedHeadTag = false;
if (chunk.Type == HtmlChunkType.OpenTag)
{
if (chunk.TagName.Equals("head", StringComparison.OrdinalIgnoreCase))
{
if (!chunk.HasAttribute("runat"))
{
chunk.SetAttribute("runat", "server");
modified = true;
}
}
else if (chunk.TagName.Equals("body", StringComparison.OrdinalIgnoreCase))
isOpenBodyTag = true;
else if (chunk.TagName.Equals("title", StringComparison.OrdinalIgnoreCase))
setTitle = false;
}
else if (chunk.Type == HtmlChunkType.CloseTag)
{
if (chunk.TagName.Equals("body", StringComparison.OrdinalIgnoreCase))
isCloseBodyTag = true;
if (chunk.TagName.Equals("head", StringComparison.OrdinalIgnoreCase))
isClosedHeadTag = true;
}
if (includeFormTag && isCloseBodyTag)
outPut.Append("</form>");
else if (!includeFormTag && isClosedHeadTag)
this.AppendRequiredHeaderContent(outPut, setTitle);
if (modified)
outPut.Append(chunk.GenerateHtml());
else
outPut.Append(chunk.Html);
if (includeFormTag && isOpenBodyTag)
outPut.Append("<form runat='server'>");
}
}
return outPut.ToString();
}
public async Task <IconResult> GetIconAsync(string domain)
{
var uri = new Uri($"https://{domain}");
var response = await GetAndFollowAsync(uri, 2);
if (response == null || !response.IsSuccessStatusCode)
{
Cleanup(response);
uri = new Uri($"http://{domain}");
response = await GetAndFollowAsync(uri, 2);
if (response == null || !response.IsSuccessStatusCode)
{
Cleanup(response);
uri = new Uri($"https://www.{domain}");
response = await GetAndFollowAsync(uri, 2);
}
}
if (response?.Content == null || !response.IsSuccessStatusCode)
{
Cleanup(response);
return(null);
}
var parser = new HtmlParser();
using (response)
using (var htmlStream = await response.Content.ReadAsStreamAsync())
using (var document = await parser.ParseAsync(htmlStream))
{
uri = response.RequestMessage.RequestUri;
if (document.DocumentElement == null)
{
return(null);
}
var baseUrl = "/";
var baseUrlNode = document.QuerySelector("head base[href]");
if (baseUrlNode != null)
{
var hrefAttr = baseUrlNode.Attributes["href"];
if (!string.IsNullOrWhiteSpace(hrefAttr?.Value))
{
baseUrl = hrefAttr.Value;
}
baseUrlNode = null;
hrefAttr = null;
}
var icons = new List <IconResult>();
var links = document.QuerySelectorAll("head link[href]");
if (links != null)
{
foreach (var link in links.Take(200))
{
var hrefAttr = link.Attributes["href"];
if (string.IsNullOrWhiteSpace(hrefAttr?.Value))
{
continue;
}
var relAttr = link.Attributes["rel"];
var sizesAttr = link.Attributes["sizes"];
if (relAttr != null && _iconRels.Contains(relAttr.Value.ToLower()))
{
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
}
else
{
try
{
var extension = Path.GetExtension(hrefAttr.Value);
if (_iconExtensions.Contains(extension.ToLower()))
{
icons.Add(new IconResult(hrefAttr.Value, sizesAttr?.Value));
}
}
catch (ArgumentException) { }
}
sizesAttr = null;
relAttr = null;
hrefAttr = null;
}
links = null;
}
var iconResultTasks = new List <Task>();
foreach (var icon in icons.OrderBy(i => i.Priority).Take(10))
{
Uri iconUri = null;
if (icon.Path.StartsWith("//") && Uri.TryCreate($"{GetScheme(uri)}://{icon.Path.Substring(2)}",
UriKind.Absolute, out var slashUri))
{
iconUri = slashUri;
}
else if (Uri.TryCreate(icon.Path, UriKind.Relative, out var relUri))
{
iconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", baseUrl, relUri.OriginalString);
}
else if (Uri.TryCreate(icon.Path, UriKind.Absolute, out var absUri))
{
iconUri = absUri;
}
if (iconUri != null)
{
var task = GetIconAsync(iconUri).ContinueWith(async(r) =>
{
var result = await r;
if (result != null)
{
icon.Path = iconUri.ToString();
icon.Icon = result.Icon;
}
});
iconResultTasks.Add(task);
}
}
await Task.WhenAll(iconResultTasks);
if (!icons.Any(i => i.Icon != null))
{
var faviconUri = ResolveUri($"{GetScheme(uri)}://{uri.Host}", "favicon.ico");
var result = await GetIconAsync(faviconUri);
if (result != null)
{
icons.Add(result);
}
else
{
return(null);
}
}
return(icons.Where(i => i.Icon != null).OrderBy(i => i.Priority).First());
}
}
public static async Task <MemoryStream> GetNND(string s, DiscordMessage msg)
{
try
{
var cookies = new CookieContainer();
var handler = new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
CookieContainer = cookies,
UseCookies = true,
UseDefaultCredentials = false
};
var client = new HttpClient(handler);
string loginForm = $"mail={Bot.cfg.NndConfig.Mail}&password={Bot.cfg.NndConfig.Password}&site=nicometro";
var body = new StringContent(loginForm, Encoding.UTF8, "application/x-www-form-urlencoded");
string login = "******";
body.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
await msg.ModifyAsync("Logging in");
var doLogin = await client.PostAsync(new Uri(login), body);
await msg.ModifyAsync("Getting video page");
var videoPage = await client.GetStringAsync(new Uri($"https://www.nicovideo.jp/watch/{s}"));
var parser = new HtmlParser();
var parsedDoc = await parser.ParseDocumentAsync(videoPage);
IElement flashPart = null;
var h5Part = parsedDoc.GetElementById("js-initial-watch-data");
string jsonData = "";
string songTitle = "";
string songArtist = "";
if (h5Part != null)
{
jsonData = h5Part.GetAttribute("data-api-data");
}
else
{
flashPart = parsedDoc.GetElementById("watchAPIDataContainer");
jsonData = flashPart.TextContent;
}
MemoryStream videoData = new MemoryStream();
await msg.ModifyAsync("Downloading video (this may take up to 5 min)");
if (flashPart == null)
{
var dataObject = JsonConvert.DeserializeObject <NND_Watch>(jsonData);
videoData = new MemoryStream(await client.GetByteArrayAsync(dataObject.video.smileInfo.url));
songTitle = dataObject.video.originalTitle;
songArtist = dataObject.owner?.nickname == null ? "n/a" : dataObject.owner.nickname;
}
else
{
var dataObject = JsonConvert.DeserializeObject <NND_Watch_Flash>(jsonData);
var directVideoUri = dataObject.flashvars.flvInfo.Replace("%253A%252F%252F", "://").Replace("%252F", "/").Replace("%253F", "?").Replace("%253D", "=").Split("%3D").First(x => x.StartsWith("http")).Split("%26")[0];
Console.WriteLine(directVideoUri);
songTitle = dataObject.videoDetail.title_original;
songArtist = dataObject.uploaderInfo?.nickname == null ? "n/a" : dataObject.uploaderInfo.nickname;
videoData = new MemoryStream(await client.GetByteArrayAsync(directVideoUri));
}
videoData.Position = 0;
var videoFile = File.Create($@"{s}");
videoData.CopyTo(videoFile);
videoData.Position = 0;
videoFile.Close();
await msg.ModifyAsync("Converting");
Process process = new Process();
process.StartInfo.FileName = "ffmpeg";
process.StartInfo.Arguments = $"-i {$@"{s}"} -metadata title=\"{songTitle}\" -metadata artist=\"{songArtist}\" {$@"{s}"}.mp3";
process.OutputDataReceived += (d, f) =>
{
Console.WriteLine(f.Data);
};
process.Start();
process.WaitForExit();
File.Delete($@"{s}");
MemoryStream ms = new MemoryStream(await File.ReadAllBytesAsync($@"{s}.mp3"));
File.Delete($@"{s}.mp3");
ms.Position = 0;
return(ms);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(null);
}
}
public static void ParseHTMLContent(string content)
{
var parser = new HtmlParser();
try
{
var doc = parser.ParseDocument(content);
//Reading background info
var bgelements = doc.QuerySelectorAll("div").Where(x => x.GetAttribute("id") == "field-background");
if (bgelements.Count() != 0)
{
//Console.WriteLine(bgelements.First().FirstElementChild.InnerHtml);
Console.WriteLine(bgelements.First().TextContent);
}
Console.WriteLine("----------------");
bgelements = doc.QuerySelectorAll("div").Where(x => x.GetAttribute("id") == "field-capital");
if (bgelements.Count() != 0)
{
foreach (var elem in bgelements.First().Children.Where(x => x.HasAttribute("class") &&
x.GetAttribute("class").Contains("category_data") &&
(x.GetAttribute("class").Contains("text") || x.GetAttribute("class").Contains("note"))))
{
foreach (var child in elem.Children)
{
Console.Write(child.TextContent);
}
elem.RemoveChild(elem.Children.First());
Console.WriteLine(elem.TextContent);
}
}
Console.WriteLine("------------------");
bgelements = doc.QuerySelectorAll("div").Where(x => x.GetAttribute("id") == "field-national-anthem");
if (bgelements.Count() != 0)
{
foreach (var elem in bgelements.First().Children.Where(x => x.HasAttribute("class") &&
x.GetAttribute("class").Contains("category_data") &&
(x.GetAttribute("class").Contains("text") || x.GetAttribute("class").Contains("note"))))
{
foreach (var child in elem.Children)
{
Console.Write(child.TextContent);
}
elem.RemoveChild(elem.Children.First());
Console.WriteLine(elem.TextContent);
}
}
Console.WriteLine("------------------");
bgelements = doc.QuerySelectorAll("div").Where(x => x.GetAttribute("id") == "field-country-name");
if (bgelements.Count() != 0)
{
foreach (var elem in bgelements.First().Children.Where(x => x.HasAttribute("class") &&
x.GetAttribute("class").Contains("category_data") &&
(x.GetAttribute("class").Contains("text") || x.GetAttribute("class").Contains("note"))))
{
foreach (var child in elem.Children)
{
Console.Write(child.TextContent);
}
elem.RemoveChild(elem.Children.First());
Console.WriteLine(elem.TextContent);
}
}
Console.WriteLine("------------------");
bgelements = doc.QuerySelectorAll("div").Where(x => x.GetAttribute("id") == "field-area");
if (bgelements.Count() != 0)
{
foreach (var elem in bgelements.First().Children.Where(x => x.HasAttribute("class") &&
x.GetAttribute("class").Contains("category_data") &&
(x.GetAttribute("class").Contains("text") || x.GetAttribute("class").Contains("note") || x.GetAttribute("class").Contains("numeric"))))
{
Console.Write(elem.Children.First().TextContent);
elem.RemoveChild(elem.Children.First());
Console.WriteLine(elem.TextContent);
}
}
Console.WriteLine("------------------");
bgelements = doc.QuerySelectorAll("div").Where(x => x.GetAttribute("id") == "field-land-use");
if (bgelements.Count() != 0)
{
foreach (var elem in bgelements.First().Children.Where(x => x.HasAttribute("class") &&
x.GetAttribute("class").Contains("category_data") &&
(x.GetAttribute("class").Contains("text") || x.GetAttribute("class").Contains("note") || x.GetAttribute("class").Contains("numeric"))))
{
Console.Write(elem.Children.First().TextContent);
elem.RemoveChild(elem.Children.First());
Console.WriteLine(elem.TextContent);
}
}
}
catch (Exception e) { Console.WriteLine(e.Message); }
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var categoryMapping = MapTorznabCapsToTrackers(query).Distinct().ToList();
var qc = new List <KeyValuePair <string, string> > // NameValueCollection don't support cat[]=19&cat[]=6
{
{ "in", "1" },
{ "type", categoryMapping.Any() ? categoryMapping.First() : "0" } // type=0 => all categories
};
// resolution filter to improve the search
if (!query.Categories.Contains(TorznabCatType.Movies.ID) && !query.Categories.Contains(TorznabCatType.TV.ID) &&
!query.Categories.Contains(TorznabCatType.Audio.ID))
{
if (query.Categories.Contains(TorznabCatType.MoviesUHD.ID) || query.Categories.Contains(TorznabCatType.TVUHD.ID))
{
qc.Add("video_quality[]", "6"); // 2160p
}
if (query.Categories.Contains(TorznabCatType.MoviesHD.ID) || query.Categories.Contains(TorznabCatType.TVHD.ID))
{
qc.Add("video_quality[]", "2"); // 720p
qc.Add("video_quality[]", "7"); // 1080i
qc.Add("video_quality[]", "3"); // 1080p
}
if (query.Categories.Contains(TorznabCatType.MoviesSD.ID) || query.Categories.Contains(TorznabCatType.TVSD.ID))
{
qc.Add("video_quality[]", "1"); // SD
}
}
// imdb search
if (query.IsImdbQuery)
{
var movieId = await GetMovieId(query.ImdbID);
if (movieId == null)
{
return(releases); // movie not found or service broken => return 0 results
}
qc.Add("movie_id", movieId);
}
else
{
qc.Add("search", GetSearchTerm(query).Trim());
}
var episodeSearchUrl = SearchUrl + qc.GetQueryString();
var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
if (response.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
}
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.Content);
var rows = dom.QuerySelectorAll("table:has(thead) > tbody > tr");
foreach (var row in rows)
{
var release = new ReleaseInfo
{
MinimumRatio = 1,
MinimumSeedTime = 172800 // 48 hours
};
var qLink = row.QuerySelector("a.torrent-filename");
release.Title = qLink.Text().Trim();
release.Comments = new Uri(qLink.GetAttribute("href"));
release.Guid = release.Comments;
var qDownload = row.QuerySelector("a.torrent-download-icon");
release.Link = new Uri(qDownload.GetAttribute("href"));
var qBanner = row.QuerySelector("img.img-tor-poster")?.GetAttribute("data-poster-mid");
if (qBanner != null)
{
release.BannerUrl = new Uri(qBanner);
}
var dateStr = row.QuerySelector("td:nth-of-type(4) > span").Text().Trim();
release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr);
var sizeStr = row.QuerySelector("td:nth-of-type(6) > span").Text().Trim();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(7)").Text().Trim());
release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(8)").Text().Trim()) + release.Seeders;
var resolution = row.QuerySelector("span.badge-extra")?.TextContent.Trim();
var catMatch = _catRegex.Match(row.QuerySelectorAll("td:nth-of-type(1) i").First().GetAttribute("class"));
var cats = new List <int>();
switch (catMatch.Groups[1].Value)
{
case "film":
if (query.Categories.Contains(TorznabCatType.Movies.ID))
{
cats.Add(TorznabCatType.Movies.ID);
}
cats.Add(resolution switch
{
var res when _hdResolutions.Contains(res) => TorznabCatType.MoviesHD.ID,
"2160p" => TorznabCatType.MoviesUHD.ID,
_ => TorznabCatType.MoviesSD.ID
});
break;
case "tv":
if (query.Categories.Contains(TorznabCatType.TV.ID))
{
cats.Add(TorznabCatType.TV.ID);
}
cats.Add(resolution switch
{
var res when _hdResolutions.Contains(res) => TorznabCatType.TVHD.ID,
"2160p" => TorznabCatType.TVUHD.ID,
_ => TorznabCatType.TVSD.ID
});
public Sneaker ParseOneSneaker(Sneaker @sneaker)
{
string url = sneaker.link;
Uri uri = new Uri(url);
WebClient webClient = new WebClient();
webClient.Encoding = Encoding.UTF8;
string source = webClient.DownloadString(uri);
// Create a new parser front-end (can be re-used)
var parser = new HtmlParser();
//Just get the DOM representation
var document = parser.Parse(source);
sneaker.brand = "Nike";
sneaker.title = document.QuerySelector("h1.product__title").InnerHtml;
sneaker.ParseTitle();
sneaker.title = sneaker.brand + " " + sneaker.title.Replace(sneaker.brand, "").Trim();
string[] artikul = document.QuerySelector("span.product__articul__num").InnerHtml.Split('-');
if (artikul.Count() < 2)
{
return(null);
}
sneaker.sku = artikul[0] + '-' + artikul[1];
//description
string sourceDescription = document.QuerySelector("div.product__descr__right").InnerHtml;
var documentDescription = parser.Parse(sourceDescription);
try
{
sneaker.description = documentDescription.QuerySelectorAll("p").ToArray()[1].InnerHtml;
}
catch (IndexOutOfRangeException e)
{
sneaker.description = string.Empty;
//нет описания
}
//price
source = document.QuerySelector("span.product__price").InnerHtml;
var document2 = parser.Parse(source);
var priceString = document2.QuerySelector("span.product__price__num").InnerHtml;
priceString = priceString.Replace("<!--'start_frame_cache_price'-->", "").Replace("<!--'end_frame_cache_price'-->", "");
int price = Int32.Parse(priceString);
sneaker.price = price;
//old price
var oldPriceHTML = document.QuerySelector("span.product__price-old");
if (oldPriceHTML != null)
{
var oldPriceString = oldPriceHTML.QuerySelector("span.product__price__num").InnerHtml.Trim().Replace(" ", "");
oldPriceString = oldPriceString.Replace("<!--'start_frame_cache_old_price'-->", "").Replace("<!--'end_frame_cache_old_price'-->", "");
sneaker.oldPrice = Int32.Parse(oldPriceString);
}
//images
source = document.QuerySelector("div.gallery-preview").InnerHtml;
document2 = parser.Parse(source);
var images = document2.QuerySelectorAll("a");
//List<String> listImage = new List<String>();
foreach (var image in images)
{
string imageStr = image.GetAttribute("href");
if (imageStr.IndexOf("?") > 0)
{
imageStr = imageStr.Substring(0, imageStr.IndexOf("?"));
}
if (!imageStr.Contains("http://") && !imageStr.Contains("https://"))
{
imageStr = "http://street-beat.ru" + imageStr;
}
sneaker.images.Add(imageStr);
}
//sizes
source = document.QuerySelector("div.product__choice_size").InnerHtml;
document2 = parser.Parse(source);
var sizes2 = document2.QuerySelectorAll("li.size-list_item").ToArray();
foreach (var sizeitem in sizes2)
{
Boolean isEnd = false;
foreach (var className in sizeitem.ClassList)
{
if (className == "list__item_disable")
{
isEnd = true;
}
}
if (!isEnd)
{
SneakerSize size = new SneakerSize();
size.sizeUS = sizeitem.GetAttribute("data-article").Split('-')[2];
SneakerSizeStock stock = new SneakerSizeStock();
stock.stockName = "StreetBeat";
stock.quantity = 1;
stock.price = price;
size.stock.Add(stock);
sneaker.sizes.Add(size);
}
}
return(sneaker);
}
private async void ButtonBase_OnClick(object sender, RoutedEventArgs e)
{
Stats.Clear();
Statistics.Clear();
MainGrid.IsEnabled = false;
var address = $"http://{AddressUrl.Text}/stats/";
String encoded = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(User.Text + ":" + Password.Password));
var webRequest = WebRequest.Create(address) as HttpWebRequest;
webRequest.Method = "GET";
webRequest.ProtocolVersion = HttpVersion.Version11;
webRequest.Headers.Add("Authorization", "Basic " + encoded);
using (var response = await webRequest.GetResponseAsync())
{
var stream = response.GetResponseStream();
var streamReader = new StreamReader(stream);
var html = streamReader.ReadToEnd();
var htmlParser = new HtmlParser();
var htmlDoc = htmlParser.Parse(html);
var lis = htmlDoc.QuerySelectorAll("li");
foreach (var li in lis)
{
var a = li.QuerySelector("a");
var url = a.Attributes["href"].Value;
var split = url.Split(new[] { "." }, StringSplitOptions.None);
if (split.Length == 3)
{
var guid = split[0];
if (!Stats.ContainsKey(guid))
{
var st = new Statistic
{
Name = a.InnerHtml.Remove(a.InnerHtml.Length - 7),
Guid = guid
};
Stats.Add(guid, st);
Statistics.Add(st);
}
Stats[guid].Files.Add(new StatisticFile(Stats[guid])
{
Name = a.InnerHtml,
Url = address + url
});
}
}
}
var root = new RootStatstic();
root.Files.AddRange(Statistics.ToList());
TreeView1.ItemsSource = new List <RootStatstic> {
root
};
MainGrid.IsEnabled = true;
}
/// <summary>
/// サイトから新刊情報を取得し、Book型のリストとして返す(同期処理)
/// </summary>
/// <returns></returns>
public static List <Book> GetWebData()
{
var urlstring = @"http://yurinavi.com/yuri-calendar/";
var document = default(IHtmlDocument);
try
{
using (var stream = client.GetStreamAsync(urlstring).Result)
{
var parser = new HtmlParser();
document = parser.ParseDocumentAsync(stream).Result;
}
}
catch
{
Console.WriteLine("GetWebData()でError");
var window = new ErrorDialog("インターネット接続でエラーが発生しました。\nインターネット接続を確認してください。");
bool?res = window.ShowDialog();
Environment.Exit(1);
}
var dates = document.QuerySelectorAll(@"td.column-1:not(#tablepress-152 > tbody > tr > td)");
var books = document.QuerySelectorAll(@"td.column-3");
var count = 0;
var data = new List <Book>();
foreach (var date in dates)
{
if (date.TextContent != "")
{
if (date.TextContent.Substring(0, 1) != "▼")
{
var dateArray = date.TextContent.Split('\n');
var releaseDate = dateArray[0] + "(" + dateArray[1] + ")";
var bookData = books[count].TextContent.Split('\n');
if (bookData.Length == 2)
{
var title = bookData[0];
var author = bookData[1];
data.Add(new Book(count, releaseDate, title, author));
count++;
}
else
{
var title = bookData[0];
data.Add(new Book(count, releaseDate, title));
count++;
}
}
}
else
{
var bookData = books[count].TextContent.Split('\n');
if (bookData.Length == 2)
{
var releaseDate = "";
var index = data.Count - 1;
while (!(releaseDate != "" || index < 0))
{
releaseDate = data[index].dateData;
index--;
}
var title = bookData[0];
var author = bookData[1];
data.Add(new Book(count, releaseDate, title, author));
count++;
}
else
{
var releaseDate = "";
var index = data.Count - 1;
while (!(releaseDate != "" || index < 0))
{
releaseDate = data[index].dateData;
index--;
}
var title = bookData[0];
data.Add(new Book(count, releaseDate, title));
count++;
}
}
}
return(data);
}
async static void DoTask()
{
HttpWebRequest reqw =
(HttpWebRequest)HttpWebRequest.Create("https://ru.wikipedia.org/wiki/%D0%92%D1%81%D0%BF%D1%8B%D1%88%D0%BA%D0%B0_COVID-19");
HttpWebResponse resp = (HttpWebResponse)reqw.GetResponse(); //создаем объект отклика
StreamReader sr = new StreamReader(resp.GetResponseStream(), Encoding.Default);
var parser = new HtmlParser();
var document = parser.ParseDocument(resp.GetResponseStream());
//Console.WriteLine(document.DocumentElement.OuterHtml);
IElement tableElement =
document.QuerySelector("h3:has(> span#Распространение_по_странам_и_территориям) + table > tbody");
//Console.WriteLine(tableElement.QuerySelector("tbody").InnerHtml);
int count = 0;
int totalInfected = 0;
int totalDead = 0;
int totalRecovered = 0;
var rows = tableElement.QuerySelectorAll("tr");
foreach (var item in rows)
{
if (count != 0 && count != rows.Count() - 1)
{
try
{
if (!item.Children[0].InnerHtml.Contains("Макао"))
{
totalInfected +=
(item.Children[1].InnerHtml != "") ? Int32.Parse(item.Children[1].InnerHtml) : 0;
totalDead +=
(item.Children[3].InnerHtml != "") ? Int32.Parse(item.Children[3].InnerHtml) : 0;
totalRecovered +=
(item.Children[4].InnerHtml != "") ? Int32.Parse(item.Children[4].InnerHtml) : 0;
}
}
catch (Exception)
{
// throw;
}
// Console.WriteLine(item.Children[1].InnerHtml);
// Console.WriteLine();
}
count++;
}
Console.WriteLine(totalInfected);
Console.WriteLine(totalDead);
Console.WriteLine(totalRecovered);
var deadPt = ((double)totalDead / (double)totalInfected) * 100d;
var recPt = ((double)totalRecovered / (double)totalInfected) * 100d;
Console.WriteLine(deadPt);
Console.WriteLine(recPt);
using (SqlConnection sqlConnection =
new SqlConnection(@"Data Source=(localdb)\MSSQLLocalDB;Initial Catalog=desease;Integrated Security=True"))
{
sqlConnection.Open();
SqlCommand insertCommand =
sqlConnection.CreateCommand();
Console.WriteLine($"INSERT INTO [TotalStat] ([dead_percent], [recovered_percent]) VALUES ({String.Format("{0:0.##}", deadPt)}, {String.Format("{0:0.##}", recPt)})");
insertCommand.CommandText =
$"INSERT INTO [TotalStat] ([dead_percent], [recovered_percent]) VALUES ({deadPt.ToString().Replace(",", ".")}, {recPt.ToString().Replace(",", ".")})";
Console.WriteLine($"{insertCommand.ExecuteNonQuery()} rows were inserted");
}
sr.Close();
}
private async Task <List <ReleaseInfo> > PerformQueryNewest(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var url = SiteLink + NewTorrentsUrl;
var result = await RequestWithCookiesAsync(url);
if (result.Status != HttpStatusCode.OK)
{
throw new ExceptionWithConfigData(result.ContentString, configData);
}
try
{
var searchResultParser = new HtmlParser();
var doc = searchResultParser.ParseDocument(result.ContentString);
var container = doc.QuerySelector("#main_table_center_center1 table div");
var parsedDetailsLink = new List <string>();
string rowTitle = null;
string rowDetailsLink = null;
string rowPublishDate = null;
string rowQuality = null;
foreach (var row in container.Children)
{
if (row.TagName.Equals("A"))
{
rowTitle = row.TextContent;
rowDetailsLink = SiteLink + row.GetAttribute("href");
}
else if (rowPublishDate == null && row.TagName.Equals("SPAN"))
{
rowPublishDate = row.TextContent;
}
else if (rowPublishDate != null && row.TagName.Equals("SPAN"))
{
rowQuality = row.TextContent;
}
else if (row.TagName.Equals("BR"))
{
// we add parsed items to rowDetailsLink to avoid duplicates in newest torrents
// list results
if (!parsedDetailsLink.Contains(rowDetailsLink))
{
await ParseRelease(releases, rowTitle, rowDetailsLink, null,
rowPublishDate, rowQuality, query, false);
parsedDetailsLink.Add(rowDetailsLink);
}
// clean the current row
rowTitle = null;
rowDetailsLink = null;
rowPublishDate = null;
rowQuality = null;
}
}
}
catch (Exception ex)
{
OnParseError(result.ContentString, ex);
}
return(releases);
}
internal MetaEndpoint(Uri baseUri, HttpClient client, HtmlParser parser) : base(baseUri, client, parser)
{
}
public void FindParseFeedItemValueLocationNameTest()
{
// arrange
string uriPrefix = string.Empty;
HtmlParser target = new HtmlParser(uriPrefix);
HTMLParserFeedItemType expectedType = HTMLParserFeedItemType.LocationName;
// act
string actual = target.FindParseFeedItemValue(target.HtmlParserFeedItems.ToList(), expectedType);
// assert
Assert.AreEqual(expectedType.ToString(), actual);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var matchWords = ((BoolItem)configData.GetDynamic("MatchWords")).Value;
matchWords = query.SearchTerm != "" && matchWords;
// we remove parts from the original query
query = ParseQuery(query);
var qc = new NameValueCollection {
{ "s", query.SearchTerm }
};
var page = 1;
var isLastPage = false;
do
{
var url = SiteLink + "page/" + page + "/?" + qc.GetQueryString();
var result = await RequestStringWithCookies(url);
if (result.Status != HttpStatusCode.OK)
{
throw new ExceptionWithConfigData(result.Content, configData);
}
try
{
var searchResultParser = new HtmlParser();
var doc = searchResultParser.ParseDocument(result.Content);
var table = doc.QuerySelector("table.table");
if (table == null)
{
break;
}
var rows = table.QuerySelectorAll("tr");
isLastPage = rows.Length - 1 <= MaxResultsPerPage; // rows includes the header
var isHeader = true;
foreach (var row in rows)
{
if (isHeader)
{
isHeader = false;
continue;
}
try
{
await ParseRelease(releases, row, query, matchWords);
}
catch (Exception ex)
{
logger.Error($"CardigannIndexer ({ID}): Error while parsing row '{row.ToHtmlPretty()}':\n\n{ex}");
}
}
}
catch (Exception ex)
{
OnParseError(result.Content, ex);
}
page++; // update page number
} while (!isLastPage && page <= MaxSearchPageLimit);
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var queryCollection = new NameValueCollection();
queryCollection.Add("order_by", "time");
queryCollection.Add("order_way", "desc");
if (!string.IsNullOrWhiteSpace(query.ImdbID))
{
queryCollection.Add("imdbid", query.ImdbID);
}
else if (!string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("groupname", searchString);
}
queryCollection.Add("groupname", searchString);
var searchUrl = BrowseUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookies(searchUrl);
if (results.IsRedirect)
{
// re login
await GetConfigurationForSetup();
await ApplyConfiguration(null);
results = await RequestStringWithCookies(searchUrl);
}
Regex IMDBRegEx = new Regex(@"tt(\d+)", RegexOptions.Compiled);
var hParser = new HtmlParser();
var ResultDocument = hParser.Parse(results.Content);
try
{
var Groups = ResultDocument.QuerySelectorAll("div.browsePoster");
foreach (var Group in Groups)
{
var groupPoster = Group.QuerySelector("img.classBrowsePoster");
var bannerURL = new Uri(SiteLink + groupPoster.GetAttribute("src"));
long?IMDBId = null;
var imdbLink = Group.QuerySelector("a[href^=\"http://anonym.to/?http://www.imdb.com/title/tt\"]");
if (imdbLink != null)
{
var IMDBMatch = IMDBRegEx.Match(imdbLink.GetAttribute("href"));
IMDBId = ParseUtil.CoerceLong(IMDBMatch.Groups[1].Value);
}
var GroupTitle = Group.QuerySelector("strong:has(a[title=\"View Torrent\"])").TextContent.Replace(" ]", "]");
var Rows = Group.QuerySelectorAll("tr.group_torrent:has(a[href^=\"torrents.php?id=\"])");
foreach (var Row in Rows)
{
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 72 * 60 * 60;
var title = Row.QuerySelector("a[href^=\"torrents.php?id=\"]");
var link = Row.QuerySelector("a[href^=\"torrents.php?action=download\"]");
var added = Row.QuerySelector("td:nth-child(3)");
var Size = Row.QuerySelector("td:nth-child(4)");
var Grabs = Row.QuerySelector("td:nth-child(6)");
var Seeders = Row.QuerySelector("td:nth-child(7)");
var Leechers = Row.QuerySelector("td:nth-child(8)");
release.Title = GroupTitle + " " + title.TextContent;
release.Category = new List <int> {
TorznabCatType.MoviesHD.ID
};
release.Link = new Uri(SiteLink + link.GetAttribute("href"));
release.Comments = new Uri(SiteLink + title.GetAttribute("href"));
release.Guid = release.Link;
release.Size = ReleaseInfo.GetBytes(Size.TextContent);
release.Seeders = ParseUtil.CoerceInt(Seeders.TextContent);
release.Peers = ParseUtil.CoerceInt(Leechers.TextContent) + release.Seeders;
release.Grabs = ParseUtil.CoerceLong(Grabs.TextContent);
release.PublishDate = DateTimeUtil.FromTimeAgo(added.TextContent);
release.BannerUrl = bannerURL;
release.Imdb = IMDBId;
releases.Add(release);
}
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
/// <summary>
/// Execute our search query
/// </summary>
/// <param name="query">Query</param>
/// <returns>Releases</returns>
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var exactSearchTerm = query.GetQueryString();
var searchUrl = SearchUrl;
// Check login before performing a query
await CheckLogin();
// Check cache first so we don't query the server (if search term used or not in dev mode)
if (!DevMode && !string.IsNullOrEmpty(exactSearchTerm))
{
lock (cache)
{
// Remove old cache items
CleanCache();
// Search in cache
var cachedResult = cache.FirstOrDefault(i => i.Query == exactSearchTerm);
if (cachedResult != null)
{
return(cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray());
}
}
}
var SearchTerms = new List <string> {
exactSearchTerm
};
// duplicate search without diacritics
var baseSearchTerm = StringUtil.RemoveDiacritics(exactSearchTerm);
if (baseSearchTerm != exactSearchTerm)
{
SearchTerms.Add(baseSearchTerm);
}
foreach (var searchTerm in SearchTerms)
{
// Build our query
var request = BuildQuery(searchTerm, query, searchUrl);
// Getting results & Store content
var response = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value);
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.ContentString);
try
{
var firstPageRows = FindTorrentRows(dom);
// If pagination available
int nbResults;
int pageLinkCount;
pageLinkCount = 1;
// Check if we have a minimum of one result
if (firstPageRows?.Length > 1)
{
// Retrieve total count on our alone page
nbResults = firstPageRows.Length;
}
else
{
// No result found for this query
Output("\nNo result found for your query, please try another search term or change the theme you're currently using on the site as this is an unsupported solution...\n", "info");
break;
}
Output("\nFound " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !");
Output("\nThere are " + (firstPageRows.Length - 2) + " results on the first page !");
// Loop on results
foreach (var row in firstPageRows.Skip(1).Take(firstPageRows.Length - 2))
{
Output("Torrent #" + (releases.Count + 1));
// ID
var idOrig = row.QuerySelector("td:nth-of-type(2) > a:nth-of-type(1)").GetAttribute("href").Split('=')[1];
var id = idOrig.Substring(0, idOrig.Length - 4);
Output("ID: " + id);
// Release Name
var name = row.QuerySelector("td:nth-of-type(2) > a:nth-of-type(1)").TextContent.Trim();
// Category
var categoryId = row.QuerySelector("td:nth-of-type(1) > a:nth-of-type(1)").GetAttribute("href").Split('?').Last();
var newznab = MapTrackerCatToNewznab(categoryId);
Output("Category: " + (newznab.Count > 0 ? newznab.First().ToString() : "unknown category") + " (" + categoryId + ")");
// Seeders
var seeders = ParseUtil.CoerceInt(Regex.Match(row.QuerySelector("td:nth-of-type(10)").TextContent, @"\d+").Value);
Output("Seeders: " + seeders);
// Leechers
var leechers = ParseUtil.CoerceInt(Regex.Match(row.QuerySelector("td:nth-of-type(11)").TextContent, @"\d+").Value);
Output("Leechers: " + leechers);
// Files
var files = 1;
files = ParseUtil.CoerceInt(Regex.Match(row.QuerySelector("td:nth-of-type(5)").TextContent, @"\d+").Value);
Output("Files: " + files);
// Completed
var completed = ParseUtil.CoerceInt(Regex.Match(row.QuerySelector("td:nth-of-type(9)").TextContent, @"\d+").Value);
Output("Completed: " + completed);
// Size
var humanSize = row.QuerySelector("td:nth-of-type(8)").TextContent.ToLowerInvariant();
var size = ReleaseInfo.GetBytes(humanSize);
Output("Size: " + humanSize + " (" + size + " bytes)");
// Publish DateToString
var dateTimeOrig = row.QuerySelector("td:nth-of-type(7)").TextContent;
var datestr = Regex.Replace(dateTimeOrig, @"<[^>]+>| ", "").Trim();
datestr = Regex.Replace(datestr, "Today", DateTime.Now.ToString("MMM dd yyyy"), RegexOptions.IgnoreCase);
datestr = Regex.Replace(datestr, "Yesterday", DateTime.Now.Date.AddDays(-1).ToString("MMM dd yyyy"), RegexOptions.IgnoreCase);
var date = DateTimeUtil.FromUnknown(datestr, "DK");
Output("Released on: " + date);
// Torrent Details URL
var detailsLink = new Uri(TorrentDescriptionUrl.Replace("{id}", id.ToString()));
Output("Details: " + detailsLink.AbsoluteUri);
// Torrent Comments URL
var commentsLink = new Uri(TorrentCommentUrl.Replace("{id}", id.ToString()));
Output("Comments Link: " + commentsLink.AbsoluteUri);
// Torrent Download URL
var passkey = row.QuerySelector("td:nth-of-type(3) > a:nth-of-type(1)").GetAttribute("href");
var key = Regex.Match(passkey, "(?<=torrent_pass\\=)([a-zA-z0-9]*)");
var downloadLink = new Uri(TorrentDownloadUrl.Replace("{id}", id.ToString()).Replace("{passkey}", key.ToString()));
Output("Download Link: " + downloadLink.AbsoluteUri);
// Building release infos
var release = new ReleaseInfo
{
Category = newznab,
Title = name,
Seeders = seeders,
Peers = seeders + leechers,
PublishDate = date,
Size = size,
Files = files,
Grabs = completed,
Guid = detailsLink,
Comments = commentsLink,
Link = downloadLink,
MinimumRatio = 1,
MinimumSeedTime = 172800 // 48 hours
};
// IMDB
var imdbLink = row.QuerySelector("a[href*=\"imdb.com/title/tt\"]")?.GetAttribute("href");
release.Imdb = ParseUtil.GetLongFromString(imdbLink);
if (row.QuerySelector("img[title=\"Free Torrent\"]") != null)
{
release.DownloadVolumeFactor = 0;
}
else if (row.QuerySelector("img[title=\"Halfleech\"]") != null)
{
release.DownloadVolumeFactor = 0.5;
}
else if (row.QuerySelector("img[title=\"90% Freeleech\"]") != null)
{
release.DownloadVolumeFactor = 0.1;
}
else
{
release.DownloadVolumeFactor = 1;
}
release.UploadVolumeFactor = 1;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError("Error, unable to parse result \n" + ex.StackTrace, ex);
}
}
// Return found releases
return(releases);
}
private IEnumerable <NewpctRelease> ParseSearchContent(string content)
{
bool someFound = false;
var SearchResultParser = new HtmlParser();
var doc = SearchResultParser.ParseDocument(content);
List <NewpctRelease> releases = new List <NewpctRelease>();
try
{
var rows = doc.QuerySelectorAll(".content .info");
if (rows == null || !rows.Any())
{
return(null);
}
foreach (var row in rows)
{
var anchor = row.QuerySelector("a");
var h2 = anchor.QuerySelector("h2");
var title = Regex.Replace(h2.TextContent, @"\s+", " ").Trim();
var detailsUrl = anchor.GetAttribute("href");
someFound = true;
bool isSeries = h2.QuerySelector("span") != null && h2.TextContent.ToLower().Contains("calidad");
bool isGame = title.ToLower().Contains("pcdvd");
if (isSeries || isGame)
{
continue;
}
var span = row.QuerySelectorAll("span");
var pubDateText = span[1].TextContent.Trim();
var sizeText = span[2].TextContent.Trim();
long size = 0;
try
{
size = ReleaseInfo.GetBytes(sizeText);
}
catch
{
}
DateTime publishDate;
DateTime.TryParseExact(pubDateText, "dd-MM-yyyy", null, DateTimeStyles.None, out publishDate);
var div = row.QuerySelector("div");
NewpctRelease newpctRelease;
newpctRelease = GetReleaseFromData(ReleaseType.Movie, title, detailsUrl, null, null, size, publishDate);
releases.Add(newpctRelease);
}
}
catch (Exception)
{
return(null);
}
if (!someFound)
{
return(null);
}
return(releases);
}
public void SetHtmlParser(HtmlParser htmlParser)
{
_htmlParser = htmlParser;
}
private async Task <IEnumerable <ReleaseInfo> > PerformRegularQueryAsync(TorznabQuery query, string hebName = null)
{
var releases = new List <ReleaseInfo>();
var searchUrl = SearchUrl;
var searchString = query.GetQueryString();
if (query.IsImdbQuery)
{
searchString = query.ImdbID;
}
if (hebName != null)
{
searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode;
}
searchUrl += "?";
if (!string.IsNullOrWhiteSpace(searchString))
{
var strEncoded = WebUtilityHelpers.UrlEncode(searchString, Encoding);
searchUrl += "&query=" + strEncoded + "&matchquery=any";
}
searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}");
var data = await RequestWithCookiesAndRetryAsync(searchUrl);
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(data.ContentString);
var rows = dom.QuerySelectorAll("tr.box_torrent");
foreach (var row in rows)
{
var release = new ReleaseInfo();
var mainTitleLink = row.QuerySelector("div.main_title > a");
release.Title = mainTitleLink.GetAttribute("longtitle");
if (string.IsNullOrWhiteSpace(release.Title))
{
release.Title = mainTitleLink.TextContent;
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800; // 48 hours
release.Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(5)").TextContent.Replace(",", ""));
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(6)").TextContent.Replace(",", ""));
release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(7)").TextContent.Replace(",", "")) +
release.Seeders;
var fullSize = row.QuerySelector("td:nth-child(4)").TextContent;
release.Size = ReleaseInfo.GetBytes(fullSize);
release.Details = new Uri(SiteLink + row.QuerySelector("a.threadlink[href]").GetAttribute("href"));
release.Link = new Uri(SiteLink + row.QuerySelector("a:has(div.dlimg)").GetAttribute("href"));
release.Guid = release.Details;
//some releases have invalid poster URLs, ignore the posters in this case
if (Uri.TryCreate(row.QuerySelector("a[imgsrc]").GetAttribute("imgsrc"),
UriKind.Absolute, out var poster))
{
release.Poster = poster;
}
var dateStringAll = row.QuerySelector("div.up_info2").ChildNodes.Last().TextContent;
var dateParts = dateStringAll.Split(' ');
var dateString = dateParts[dateParts.Length - 2] + " " + dateParts[dateParts.Length - 1];
release.PublishDate = DateTime.ParseExact(dateString, "dd/MM/yy HH:mm", CultureInfo.InvariantCulture);
var categoryLink = row.QuerySelector("a[href^=\"/browse.php?cat=\"]").GetAttribute("href");
var catid = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
release.Category = MapTrackerCatToNewznab(catid);
if (row.QuerySelector("a[href^=\"?freeleech=1\"]") != null)
{
release.DownloadVolumeFactor = 0;
}
else
{
release.DownloadVolumeFactor = 1;
}
release.UploadVolumeFactor = 1;
var subTitle = row.QuerySelector("div.sub_title");
var imdbLink = subTitle.QuerySelector("span.imdb-inline > a");
if (imdbLink != null)
{
release.Imdb = ParseUtil.GetLongFromString(imdbLink.GetAttribute("href"));
}
release.Description = subTitle.FirstChild.TextContent;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(data.ContentString, ex);
}
return(releases);
}
private void CloseContent(HtmlParser htmlParser, string htmlContent)
{
ElementState elementState = htmlParser.CurrentElementState;
switch (elementState.TypeName)
{
case "a":
GuiWidget aWidget = elementsUnderConstruction.Pop();
if (aWidget.Name != "a")
{
throw new Exception("Should have been 'a'.");
}
elementsUnderConstruction.Peek().AddChild(aWidget);
break;
case "body":
break;
case "h1":
case "p":
GuiWidget pWidget = elementsUnderConstruction.Pop();
if (pWidget.Name != "p")
{
throw new Exception("Should have been 'p'.");
}
elementsUnderConstruction.Peek().AddChild(pWidget);
break;
case "div":
GuiWidget divWidget = elementsUnderConstruction.Pop();
if (divWidget.Name != "div")
{
throw new Exception("Should have been 'div'.");
}
elementsUnderConstruction.Peek().AddChild(divWidget);
break;
case "input":
break;
case "table":
break;
case "span":
break;
case "tr":
GuiWidget trWidget = elementsUnderConstruction.Pop();
if (trWidget.Name != "tr")
{
throw new Exception("Should have been 'tr'.");
}
elementsUnderConstruction.Peek().AddChild(trWidget);
break;
case "td":
break;
case "img":
break;
default:
throw new NotImplementedException();
}
}
