// GET: Forex
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.financemagnates.com/forex");
CQ mainArticle = dom.Find("ul.featured_cats_wrapper").Eq(0);
for (int i = 0; i < mainArticle.Children().Length; i++)
{
CQ article = mainArticle.Children().Eq(i).Find("a").Eq(0);
newsData nData = new newsData();
nData.text = article.Text();
nData.alt_url = article.Attr("href").ToString();
nData.url = "";
newsList.Add(nData);
}
CQ addArticle = dom.Find("div.sub-feature").Eq(0);
newsData firstNews = new newsData();
CQ first = addArticle.Children("article").Eq(0);
firstNews.text = first.Find("a").Eq(0).Find("div").Eq(0).Text();
firstNews.url = first.Find("a").Eq(0).Css("background-image");
firstNews.alt_url = first.Find("a").Eq(0).Attr("href").ToString();
newsList.Add(firstNews);
newsData secondNews = new newsData();
CQ second = addArticle.Children("article").Eq(1);
secondNews.text = second.Find("a").Eq(0).Find("div").Eq(0).Text();
secondNews.url = second.Find("a").Eq(0).Css("background-image");
secondNews.alt_url = second.Find("a").Eq(0).Attr("href").ToString();
newsList.Add(secondNews);
ViewBag.newsList = newsList;
return(View());
}
// GET: NewsParse
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
WebClient webClient = new WebClient();
const string strUrl = "http://www.edition.cnn.com/world";
string pageContent = webClient.DownloadString(strUrl);
CQ dom = pageContent;
CQ mainStory = dom["ul.cn.cn-list-hierarchical-xs.cn--idx-0"].Eq(0);
for (int i = 0; i < mainStory.Children().Length; i++)
{
CQ news = mainStory.Children().Eq(i).Children().Eq(0);
string url = "";
string alt_url = "";
if (news.Find("a").Length != 0)
{
alt_url = news.Find("a").Attr("href").ToString();
}
if (news.Find("img").Length != 0)
{
url = news.Find("img").Attr("data-src-large").ToString();
}
string text = news.Find("span.cd__headline-text").Text();
newsData nData = new newsData();
nData.url = url;
nData.text = text;
nData.alt_url = alt_url;
newsList.Add(nData);
}
CQ lastStory = dom["ul.cn.cn-list-hierarchical-xs.cn--idx-0"].Eq(1);
for (int i = 0; i < lastStory.Children().Length; i++)
{
CQ news = lastStory.Children("article").Eq(i).Children().Eq(0);
string alt_url = "";
if (news.Find("a").Length != 0)
{
alt_url = news.Find("a").Attr("href").ToString();
}
string url = "";
if (news.Find("img").Length != 0)
{
url = news.Find("img").Attr("data-src-large").ToString();
}
string text = news.Find("span.cd__headline-text").Text();
newsData nData = new newsData();
nData.url = url;
nData.alt_url = alt_url;
nData.text = text;
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var episodeSearchUrl = string.Format("{0}?search={1}&cat=0", SearchUrl, HttpUtility.UrlEncode(query.GetQueryString()));
var results = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
try
{
CQ dom = results.Content;
var table = dom["tbody > tr > .latest"].Parent().Parent();
foreach (var row in table.Children().Skip(1))
{
var release = new ReleaseInfo();
CQ qDetailsCol = row.ChildElements.ElementAt(1).Cq();
CQ qLink = qDetailsCol.Children("a").First();
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
release.Comments = new Uri(SiteLink + "/" + qLink.Attr("href"));
release.Guid = release.Comments;
release.Title = qLink.Attr("title");
release.Description = release.Title;
//"Tuesday, June 11th 2013 at 03:52:53 AM" to...
//"Tuesday June 11 2013 03:52:53 AM"
var timestamp = qDetailsCol.Children("font").Text().Trim() + " ";
var timeParts = new List <string>(timestamp.Replace(" at", "").Replace(",", "").Split(' '));
timeParts[2] = Regex.Replace(timeParts[2], "[^0-9.]", "");
var formattedTimeString = string.Join(" ", timeParts.ToArray()).Trim();
var date = DateTime.ParseExact(formattedTimeString, "dddd MMMM d yyyy hh:mm:ss tt", CultureInfo.InvariantCulture);
release.PublishDate = DateTime.SpecifyKind(date, DateTimeKind.Utc).ToLocalTime();
release.Link = new Uri(SiteLink + "/" + row.ChildElements.ElementAt(2).Cq().Children("a.index").Attr("href"));
var sizeStr = row.ChildElements.ElementAt(6).Cq().Text();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(8).Cq().Text());
release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(9).Cq().Text()) + release.Seeders;
//if (!release.Title.ToLower().Contains(title.ToLower()))
// continue;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
public ActionResult getDetail(string url)
{
CQ dom = CQ.CreateFromUrl(url);
CQ article = dom.Find("article.post").Eq(0);
CQ aMain = article.Children("div.entry-content").Eq(0);
CQ ads = aMain.Find("div.mobile");
ads.Remove();
CQ entry = aMain.Find("div.entry-meta");
entry.Remove();
string hText = aMain.RenderSelection().ToString();
CQ img = dom.Find("div.site-header-bg").Eq(0);
string srcImg = "";
if (img.Length != 0)
{
srcImg = img.Css("background-image").ToString();
}
//string srcImg = "";
ViewBag.hText = hText;
ViewBag.srcImg = srcImg;
ViewBag.aTitle = dom.Find("div.hero-text").Eq(0).Children("h1").Eq(0).Text();
return(View());
}
public void InnerParsingRules()
{
CQ res = Dom.Select("script");
// bug found 10/31/11
Assert.AreEqual(0, res.Children().Length, "Script cannot have children");
}
public ActionResult getDetail(string url)
{
CQ dom = CQ.CreateFromUrl(url);
CQ article = dom.Find("article.post").Eq(0);
string aMainText = article.Children("header").Eq(0).Children("h1").Eq(0).Text();
string aSubText = article.Children("header").Eq(0).Children("div").Eq(0).Text();
CQ aMain = article.Children("div.entry-content").Eq(0);
string hText = aMain.RenderSelection().ToString();
ViewBag.aMainText = aMainText;
ViewBag.aSubText = aSubText;
ViewBag.hText = hText;
return(View());
}
// GET: CNN
public ActionResult Index()
{
ArrayList articles = new ArrayList();
WebClient webClient = new WebClient();
webClient.Headers.Add("user-agent", "Only a test!");
const string strUrl = "https://edition.cnn.com/middle-east";
string pageContent = webClient.DownloadString(strUrl);
CQ dom = pageContent;
CQ top_news = dom.Find("ul.cn-list-hierarchical-piped").Eq(0);
Article t_news = new Article();
t_news.img_url = top_news.Find("img").Attr("data-src-large").ToString();
top_news = top_news.Find("h3.cd__headline").Eq(0);
t_news.title = top_news.Find("span").Text().ToString();
t_news.url = top_news.Find("a").Attr("href");
if (!t_news.url.Contains("http:"))
{
t_news.url = "https://edition.cnn.com" + t_news.url;
}
if (!t_news.img_url.Contains("http:"))
{
t_news.img_url = "http://" + t_news.img_url;
}
articles.Add(t_news);
CQ t_stories = dom.Find("ul.cn-list-xs").Eq(1);
for (int i = 0; i < t_stories.Children("li").Length; i++)
{
CQ story = t_stories.Children("li").Eq(i);
Article news = new Article();
news.url = story.Find("a").Eq(0).Attr("href").ToString();
if (!news.url.Contains("http:"))
{
news.url = "https://edition.cnn.com" + news.url;
}
news.title = story.Find("span.cd__headline-text").Text().ToString();
articles.Add(news);
}
ViewBag.newsList = articles;
return(View());
}
public void SimpleClone()
{
CQ hlinks = Dom.Select("#hlinks-user");
int spanCount = hlinks.Find("span").Length;
CQ clone = hlinks.Clone();
Assert.AreEqual(hlinks.Find("*").Length + 1, clone.Select("*").Length, "Clone has same total elements as original");
CQ newHome = Dom.Select("#hidden-div");
spanCount = newHome.Find("span").Length;
int cloneSpanCount = clone.Select("span").Length;
Assert.AreEqual(1, newHome.Children().Length, "Sanity check - target has 1 child");
newHome.Append(clone);
Assert.AreEqual(2, newHome.Children().Length, "Target has 2 children after cloning");
Assert.AreEqual(spanCount + cloneSpanCount, newHome.Find("span").Length, "Same # of spans in the clone");
}
// GET: CryptoCoin
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.cryptocoinsnews.com");
CQ mainArticle = dom.Find("div.grid-wrapper").Eq(0);
for (int i = 0; i < mainArticle.Children(".post").Length; i++)
{
CQ article = mainArticle.Children(".post").Eq(i).Find("a").Eq(0);
newsData nData = new newsData();
nData.text = article.Attr("title").ToString();
nData.alt_url = article.Attr("href").ToString();
nData.url = "";
CQ img = article.Children("img").Eq(0);
nData.url = img.Attr("src").ToString();
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
public void SimpleClone()
{
var dom = TestDom("TestHtml");
CQ hlinks = dom.Select("#hlinks-user");
CQ clone = hlinks.Clone();
Assert.AreEqual(hlinks.Find("*").Length, clone.Find("*").Length, "Clone has same total elements as original");
CQ newHome = dom.Select("#hidden-div");
var spanCount = newHome.Find("span").Length;
int cloneSpanCount = clone.Find("span").Length;
Assert.AreEqual(1, newHome.Children().Length, "Sanity check - target has 1 child");
newHome.Append(clone);
Assert.AreEqual(2, newHome.Children().Length, "Target has 2 children after cloning");
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString));
var results = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
try
{
CQ dom = results.Content;
var rows = dom["#torrent_table > tbody > tr.torrent"];
foreach (var row in rows)
{
CQ qRow = row.Cq();
var release = new ReleaseInfo();
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
release.Title = qRow.Find(".torrent_name_link").Text();
release.Description = release.Title;
release.Guid = new Uri(SiteLink + "/" + qRow.Find(".torrent_name_link").Parent().Attr("href"));
release.Comments = release.Guid;
release.Link = new Uri(SiteLink + "/" + qRow.Find(".torrent_handle_links > a").First().Attr("href"));
var dateStr = qRow.Find(".time").Text().Trim();
release.PublishDate = DateTimeUtil.FromTimeAgo(dateStr);
var sizeStr = qRow.Find(".size")[0].ChildNodes[0].NodeValue;
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(qRow.Children().ElementAt(6).InnerText.Trim());
release.Peers = ParseUtil.CoerceInt(qRow.Children().ElementAt(7).InnerText.Trim()) + release.Seeders;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
public ActionResult getDetail(string url)
{
CQ dom = CQ.CreateFromUrl("http://cnbc.com/" + url);
ViewBag.title = dom.Find("h1").Eq(0).Text();
string article = dom.Find("#article_body").Eq(0).RenderSelection().ToString();
if (article.Length == 0)
{
ViewBag.title = "";
CQ aBody = dom.Find(".cnbc-body").Eq(0);
//article = aBody.Remove(".franchise_content").RenderSelection().ToString();
//article = article.Replace("data-img-src", "src");
article = aBody.Children().Eq(2).RenderSelection().ToString();
article += aBody.Children().Eq(3).RenderSelection().ToString();
article = article.Replace("data-img-src", "src");
}
ViewBag.text = article;
return(View());
}
public ActionResult detail(string url)
{
WebClient webClient = new WebClient();
webClient.Headers.Add("user-agent", "Only a test!");
string strUrl = url;
string pageContent = webClient.DownloadString(strUrl);
CQ dom = pageContent;
CQ article_body = dom.Find("div.pg-right-rail-tall").Eq(0).Find("article").Eq(0).Find("div.l-container").Eq(0);
ViewBag.title = article_body.Children("h1").Text().ToString();
article_body = article_body.Children("div.pg-rail-tall__wrapper").Eq(0).Children("div.pg-side-of-rail").Eq(0);
CQ temp = article_body.Find("div.media__video--thumbnail-wrapper").Eq(0);
temp.Remove();
temp = article_body.Find(".el__leafmedia.el__leafmedia--storyhighlights").Eq(0);
temp.Remove();
temp = article_body.Find(".el__embedded.el__embedded--standard");
temp.Remove();
temp = article_body.Find(".el__gallery--expandable.js__gallery--expandable.js__leafmedia--gallery");
temp.Remove();
temp = article_body.Find(".cn.cn-list-hierarchical-small-horizontal.cn--idx-0.cn-");
temp.Remove();
temp = article_body.Find("input.sharebar-video-embed-field");
temp.Remove();
temp = article_body.Find(".zn__containers");
temp.Remove();
temp = article_body.Find("img.media__image.media__image--responsive");
for (int i = 0; i < temp.Length; i++)
{
CQ t = temp.Eq(i);
string str = t.Attr("data-src-large");
t.AttrSet(new { src = str });
}
temp = article_body.Find(".cn__column.carousel__content__item");
temp.Remove();
ViewBag.text = article_body.RenderSelection().ToString();
return(View());
}
// GET: Cnbc
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.cnbc.com/world/?region=world");
CQ mainArticle = dom.Find("ul.stories_assetlist").Eq(0);
for (int i = 0; i < mainArticle.Children(".card,.last-card").Length; i++)
{
CQ article = mainArticle.Children(".card").Eq(i);
CQ data = article.Find("div.headline").Eq(0).Children("a").Eq(0);
newsData nData = new newsData();
nData.text = data.Text().ToString();
nData.alt_url = data.Attr("href").ToString();
nData.url = "";
CQ img = article.Find("img").Eq(0);
nData.url = img.Attr("data-img-src");
newsList.Add(nData);
}
CQ smallArticle = dom.Find("ul.stories_assetlist").Eq(1);
for (int i = 0; i < smallArticle.Children("li").Length; i++)
{
CQ article = smallArticle.Children("li").Eq(i);
CQ data = article.Find("div.headline").Eq(0).Children("a").Eq(0);
if (data.Length == 0)
{
continue;
}
newsData nData = new newsData();
nData.text = data.Text().ToString();
nData.alt_url = data.Attr("href").ToString();
nData.url = "";
CQ img = article.Find("img").Eq(0);
nData.url = img.Attr("data-img-src");
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
// GET: Arabbit
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.arabbit.net");
CQ mainArticle = dom.Find("div.grid-wrapper").Eq(0);
for (int i = 0; i < mainArticle.Children().Length; i++)
{
newsData nData = new newsData();
CQ article = mainArticle.Children().Eq(i).Find("a").Eq(0);
string aText = "";
string aImg = "";
string aAlt = "";
if (article.Length != 0)
{
aText = article.Attr("title").ToString();
aAlt = article.Attr("href").ToString();
}
nData.text = aText;
nData.alt_url = aAlt;
nData.url = "";
CQ img = article.Children("img").Eq(0);
if (img.Length != 0)
{
aAlt = img.Attr("src").ToString();
}
nData.url = aAlt;
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
public ActionResult getDetail(string url)
{
CQ dom = CQ.CreateFromUrl(url);
CQ article = dom.Find("article.post").Eq(0);
CQ aMain = article.Children("div.entry-content").Eq(0);
string hText = aMain.RenderSelection().ToString();
CQ img = dom.Find("div.site-header-bg").Eq(0);
string srcImg = img.Css("background-image").ToString();
ViewBag.hText = hText;
ViewBag.srcImg = srcImg;
return(View());
}
private VSGalleryQAThread TryParseThread(CQ element)
{
int threadId;
if (!int.TryParse(element.Attr("threadid"), out threadId))
{
return(null);
}
var thread = new VSGalleryQAThread();
thread.ThreadId = threadId;
thread.Title = element.Children("div.threadHeader")
.Children("div.title").First().Text();
thread.Posts = new List <VSGalleryQAPost>();
foreach (var p in element.Find("li.post"))
{
var post = TryParsePost(p.Cq());
if (post != null)
{
thread.Posts.Add(post);
}
}
if (thread.Posts.Count == 0)
{
return(null);
}
var firstPost = thread.Posts.First();
thread.StartedByName = firstPost.PostedByName;
thread.StartedByUserId = firstPost.PostedByUserId;
thread.StartedOn = firstPost.PostedOn;
thread.LastPostOn = thread.Posts.Last().PostedOn;
return(thread);
}
private static Influence?ExtractInfluence(CQ domObject)
{
var lastMod = domObject.Children().LastOrDefault();
if (lastMod == null)
{
return(null);
}
if (lastMod.InnerHTML.Contains("Shaped"))
{
return(Influence.Shaper);
}
if (lastMod.InnerHTML.Contains("Elder"))
{
return(Influence.Elder);
}
return(null);
}
// GET: CnbcArabia
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.cnbcarabia.com/news/latest");
CQ mainArticle = dom.Find("div.blog-news.clearfix");
for (int i = 0; i < mainArticle.Length; i++)
{
CQ article = mainArticle.Children().Eq(i);
CQ data = article.Find(".blog-box-title").Eq(0).Children("a").Eq(0);
newsData nData = new newsData();
nData.text = data.Text().ToString();
nData.alt_url = "http://www.cnbcarabia.com" + data.Attr("href").ToString();
nData.url = "";
CQ img = article.Find("img").Eq(0);
nData.url = "http://www.cnbcarabia.com" + img.Attr("src");
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
static List <Tuple <string, string, string> > GetDownloads(string slug, int branchId, WebClient wc)
{
List <Tuple <string, string, string> > downloads = new List <Tuple <string, string, string> > ();
// Get downloads
string downloadsPage = wc.DownloadString(string.Format("http://www.desura.com/games/{0}/download/{1}", slug, branchId));
CQ csqDownload = downloadsPage;
CQ dSearch = csqDownload ["span.action:contains(Download)"];
foreach (var dSpanE in dSearch)
{
CQ dSpan = dSpanE.Cq();
CQ parentA = dSpan.Closest("a");
// Get download link
string link = "http://www.desura.com" + parentA.Attr("href");
// Get download name
CQ infoBlock = parentA.Find(".buycontent .heading.clear");
downloads.Add(new Tuple <string, string, string> (infoBlock.Children().Eq(0).Text(), infoBlock.Children().Eq(1).Text(), link));
}
return(downloads);
}
static void AddKeysToProduct(ProductInfo product, WebClient wc)
{
// Get keys
string keysPage = wc.DownloadString(string.Format("http://www.desura.com/games/{0}/keys", product.Slug));
CQ csqKeys = keysPage;
CQ kSearch = csqKeys ["span.price:contains(Select),span.price:contains(Get)"];
foreach (var kSpanE in kSearch)
{
CQ kSpan = kSpanE.Cq();
CQ parentSpan = kSpan.Closest("span.buy.clear");
// Get key name
string keyType = kSpan.Parent().Find(".action").Text();
CQ infoBlock = parentSpan.Find(".buycontent .heading.clear");
string keyName = infoBlock.Children().Eq(0).Text();
string name = string.Format("{0} ({1})", keyName, keyType);
// Get key
string key = kSpan.Text() == "Select" ? parentSpan.Find(".summary [name=\"key\"]").Attr("value") : string.Format("Reveal key at http://www.desura.com/games/{0}/keys", product.Slug);
product.Keys.Add(new NameKeyStringTuple(name, key));
}
}
private static Socket[] ExtractSockets(CQ domObjects)
{
/*
* <div class="sockets-inner" style="position: relative; width:94px;">
* <div class="socket socketD "></div>
* <div class="socketLink socketLink0"></div>
* <div class="socket socketI "></div>
* <div class="socketLink socketLink1"></div>
* <div class="socket socketS socketRight"></div>
* </div>
*
* or
*
* <div class="sockets-inner" style="position: relative; width:94px;"></div>
*/
var sockets = new List <Socket>();
// Loop though sockets and links
foreach (var domObject in domObjects.Children())
{
if (domObject.HasClass("socket"))
{
var socket = new Socket {
Color = ExtractSocketColor(domObject)
};
sockets.Add(socket);
}
else if (domObject.HasClass("socketLink"))
{
// Get previous socket and mark it as linked
sockets.Last().Linked = true;
}
}
return(sockets.IsNullOrEmpty() ? null : sockets.ToArray());
}
public ActionResult getDetail(string url)
{
string str_url = "http://edition.cnn.com" + url;
CQ dom = CQ.CreateFromUrl(str_url);
dom = dom.Find("article.pg-rail-tall").Eq(0);
CQ article = dom.Children("div.l-container").Eq(0);
article = article.Remove("script");
string aTitle = article.Children("h1").Eq(0).Text();
string aText = article.Find("div.pg-rail-tall__body").Eq(0).Children("section").Eq(0).Children("div").Eq(0).RenderSelection().ToString();
CQ mImg = article.Find("div#large-media_0--thumbnail").Eq(0);
string aImg = "";
if (mImg.Length != 0)
{
aImg = mImg.Children("img").Eq(0).Attr("src").ToString();
}
ViewBag.aTitle = aTitle;
ViewBag.aText = aText;
ViewBag.aImg = aImg;
return(View());
}
public async Task <ReleaseInfo[]> PerformQuery(TorznabQuery query)
{
List <ReleaseInfo> releases = new List <ReleaseInfo>();
foreach (var title in query.ShowTitles ?? new string[] { string.Empty })
{
var searchString = title + " " + query.GetEpisodeSearchString();
var episodeSearchUrl = string.Format("{0}?search={1}&cat=0", SearchUrl, HttpUtility.UrlEncode(searchString));
var results = await client.GetStringAsync(episodeSearchUrl);
try
{
CQ dom = results;
var table = dom["tbody > tr > .latest"].Parent().Parent();
foreach (var row in table.Children().Skip(1))
{
var release = new ReleaseInfo();
CQ qDetailsCol = row.ChildElements.ElementAt(1).Cq();
CQ qLink = qDetailsCol.Children("a").First();
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
release.Comments = new Uri(BaseUrl + "/" + qLink.Attr("href"));
release.Guid = release.Comments;
release.Title = qLink.Attr("title");
release.Description = release.Title;
//"Tuesday, June 11th 2013 at 03:52:53 AM" to...
//"Tuesday June 11 2013 03:52:53 AM"
var timestamp = qDetailsCol.Children("font").Text().Trim() + " ";
var timeParts = new List <string>(timestamp.Replace(" at", "").Replace(",", "").Split(' '));
timeParts[2] = Regex.Replace(timeParts[2], "[^0-9.]", "");
var formattedTimeString = string.Join(" ", timeParts.ToArray()).Trim();
release.PublishDate = DateTime.ParseExact(formattedTimeString, "dddd MMMM d yyyy hh:mm:ss tt", CultureInfo.InvariantCulture);
release.Link = new Uri(BaseUrl + "/" + row.ChildElements.ElementAt(2).Cq().Children("a.index").Attr("href"));
var sizeCol = row.ChildElements.ElementAt(6);
var sizeVal = float.Parse(sizeCol.ChildNodes[0].NodeValue);
var sizeUnit = sizeCol.ChildNodes[2].NodeValue;
release.Size = ReleaseInfo.GetBytes(sizeUnit, sizeVal);
release.Seeders = int.Parse(row.ChildElements.ElementAt(8).Cq().Text(), NumberStyles.AllowThousands);
release.Peers = int.Parse(row.ChildElements.ElementAt(9).Cq().Text(), NumberStyles.AllowThousands) + release.Seeders;
//if (!release.Title.ToLower().Contains(title.ToLower()))
// continue;
releases.Add(release);
}
}
catch (Exception ex)
{
OnResultParsingError(this, results, ex);
throw ex;
}
}
return(releases.ToArray());
}
private static Stat[] ExtractStats(CQ domObject)
{
var stats = new List <Stat>();
foreach (var domChild in domObject.Children())
{
foreach (var domChildChild in domChild.ChildElements)
{
var cq = domChildChild.Cq();
if (!cq.HasAttr("data-name"))
{
continue;
}
var stat = new Stat {
Value = double.Parse(cq.Attr("data-value"), CultureInfo.InvariantCulture)
};
if (stat.Value <= 0)
{
continue;
}
switch (cq.Attr("data-name"))
{
case "q":
stat.Type = Stat.StatType.Quality;
break;
case "quality_pd":
stat.Type = Stat.StatType.Phys;
break;
case "ed":
stat.Type = Stat.StatType.Elem;
break;
case "aps":
stat.Type = Stat.StatType.Aps;
break;
case "quality_dps":
stat.Type = Stat.StatType.Dps;
break;
case "quality_pdps":
stat.Type = Stat.StatType.PDps;
break;
case "edps":
stat.Type = Stat.StatType.EDps;
break;
case "quality_armour":
stat.Type = Stat.StatType.Armour;
break;
case "quality_evasion":
stat.Type = Stat.StatType.Evasion;
break;
case "quality_shield":
stat.Type = Stat.StatType.Shield;
break;
case "block":
stat.Type = Stat.StatType.Block;
break;
case "crit":
stat.Type = Stat.StatType.Crit;
break;
case "level":
stat.Type = Stat.StatType.Level;
break;
}
stats.Add(stat);
}
}
return(stats.IsNullOrEmpty() ? null : stats.ToArray());
}
private static Mod[] ExtractMods(CQ domObject)
{
/*
* <ul class="mods">
* <li class="sortable" data-name="#+#% to Cold Resistance" data-value="30.0" style="color:#3F6DB3">+<b>30</b>% to Cold Resistance
* <span class="item-affix item-affix-S">
* <span class="affix-info-short">S4</span>
* <span class="affix-info-full">Tier 4 suffix: of the Walrus, min=[30] max=[35]</span>
* </span>
* </li>
* <li class="sortable" data-name="#+#% to Lightning Resistance" data-value="13.0" style="color:#ADAA47">+<b>13</b>% to Lightning Resistance
* <span class="item-affix item-affix-S">
* <span class="affix-info-short">S7</span>
* <span class="affix-info-full">Tier 7 suffix: of the Squall, min=[12] max=[17]</span>
* </span>
* </li>
* </ul>
*
* or
*
* <ul class="mods">
* <li class="" data-name="#Enchants a rare item with a new random modifier" data-value="0" style="">Enchants a rare item with a new random modifier</li>
* </ul>
*/
if (domObject.Children().IsNullOrEmpty())
{
return(null);
}
var mods = new List <Mod>();
foreach (var domChild in domObject.Children())
{
var name = domChild.GetAttribute("data-name");
if (name.IsNullOrEmpty())
{
continue;
}
var mod = new Mod {
Name = name
};
var val = domChild.GetAttribute("data-value");
if (!val.IsNullOrEmpty())
{
mod.Value = double.Parse(val, CultureInfo.InvariantCulture);
if (mod.Value.Equals(0.0))
{
mod.Value = null;
}
}
var tiers = domChild.Cq().Find(".item-affix>span");
mod.TierShort = tiers.FirstOrDefault(t => t.HasClass("affix-info-short"))?.InnerHTML;
mod.TierLong = tiers.FirstOrDefault(t => t.HasClass("affix-info-full"))?.InnerHTML;
mods.Add(mod);
}
return(mods.ToArray());
}
private static Requirement[] ExtractRequirements(CQ domObject)
{
/*
* <ul class="requirements proplist">
* <li>Level: 52</li>
* <li>Intelligence: 17</li>
* <li><span class="sortable" data-name="ilvl">ilvl: 76</span></li>
* </ul>
*
* or
*
* <ul class="requirements proplist"></ul>
*/
if (domObject.Children().IsNullOrEmpty())
{
return(null);
}
var requirements = new List <Requirement>();
foreach (var domChild in domObject.Children())
{
var requirement = new Requirement();
requirements.Add(requirement);
if (domChild.InnerHTML.Contains("Level"))
{
requirement.Type = Requirement.RequirementType.Level;
}
else if (domChild.InnerHTML.Contains("Intelligence"))
{
requirement.Type = Requirement.RequirementType.Int;
}
else if (domChild.InnerHTML.Contains("Strength"))
{
requirement.Type = Requirement.RequirementType.Str;
}
else if (domChild.InnerHTML.Contains("Dexterity"))
{
requirement.Type = Requirement.RequirementType.Dex;
}
else if (domChild.InnerHTML.Contains("ilvl"))
{
requirement.Type = Requirement.RequirementType.ItemLevel;
}
else if (domChild.InnerHTML.Contains("Max sockets"))
{
requirement.Type = Requirement.RequirementType.MaxSocket;
}
var match = IntRegex.Match(domChild.InnerHTML);
if (match.Success)
{
requirement.Value = int.Parse(match.Groups[0].Value);
}
}
return(requirements.ToArray());
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchurls = new List <string>();
var searchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(query.GetQueryString()));
var results = await RequestStringWithCookiesAndRetry(searchUrl);
try
{
CQ dom = results.Content;
ReleaseInfo release;
int rowCount = 0;
var rows = dom[".mainblockcontenttt > tbody > tr"];
foreach (var row in rows)
{
CQ qRow = row.Cq();
if (rowCount < 2 || qRow.Children().Count() != 12) //skip 2 rows because there's an empty row & a title/sort row
{
rowCount++;
continue;
}
release = new ReleaseInfo();
release.Title = qRow.Find("td.mainblockcontent b a").Text();
release.Description = release.Title;
if (0 != qRow.Find("td.mainblockcontent u").Length)
{
var imdbStr = qRow.Find("td.mainblockcontent u").Parent().First().Attr("href").Replace("http://www.imdb.com/title/tt", "").Replace("/", "");
long imdb;
if (ParseUtil.TryCoerceLong(imdbStr, out imdb))
{
release.Imdb = imdb;
}
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
int seeders, peers;
if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(9).FirstChild.FirstChild.InnerText, out seeders))
{
release.Seeders = seeders;
if (ParseUtil.TryCoerceInt(qRow.Find("td").Get(10).FirstChild.FirstChild.InnerText, out peers))
{
release.Peers = peers + release.Seeders;
}
}
string fullSize = qRow.Find("td.mainblockcontent").Get(6).InnerText;
release.Size = ReleaseInfo.GetBytes(fullSize);
release.Guid = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent b a").Attr("href"));
release.Link = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent").Get(3).FirstChild.GetAttribute("href"));
release.Comments = new Uri(SiteLink + "/" + qRow.Find("td.mainblockcontent b a").Attr("href") + "#comments");
string[] dateSplit = qRow.Find("td.mainblockcontent").Get(5).InnerHTML.Split(',');
string dateString = dateSplit[1].Substring(0, dateSplit[1].IndexOf('>'));
release.PublishDate = DateTime.Parse(dateString, CultureInfo.InvariantCulture);
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
protected override async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
// search in normal + gems view
foreach (var view in new string[] { "0", "1" })
{
var queryCollection = new NameValueCollection();
queryCollection.Add("view", view);
queryCollection.Add("searchtype", "1");
queryCollection.Add("incldead", "1");
if (!string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("search", searchString);
}
foreach (var cat in MapTorznabCapsToTrackers(query))
{
queryCollection.Add(string.Format("c{0}", cat), "1");
}
var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookiesAndRetry(searchUrl);
if (results.IsRedirect)
{
// re-login
await ApplyConfiguration(null);
results = await RequestStringWithCookiesAndRetry(searchUrl);
}
try
{
CQ dom = results.Content;
var rows = dom["#torrentBrowse > table > tbody > tr"];
foreach (var row in rows.Skip(1))
{
var release = new ReleaseInfo();
CQ qRow = row.Cq();
release.MinimumRatio = 0;
release.MinimumSeedTime = 2 * 24 * 60 * 60;
var qLink = qRow.Find("a[title][href^=\"details.php?id=\"]");
release.Title = qLink.Attr("title");
release.Guid = new Uri(SiteLink + qLink.Attr("href").TrimStart('/'));
release.Comments = release.Guid;
qLink = qRow.Children().ElementAt(3).Cq().Children("a").First();
release.Link = new Uri(string.Format("{0}{1}", SiteLink, qLink.Attr("href")));
var catUrl = qRow.Children().ElementAt(0).FirstElementChild.Cq().Attr("href");
var catNum = catUrl.Split(new char[] { '=', '&' })[2].Replace("c", "");
release.Category = MapTrackerCatToNewznab(catNum);
var dateString = qRow.Children().ElementAt(6).Cq().Text().Trim();
if (dateString.Contains("ago"))
{
release.PublishDate = DateTimeUtil.FromTimeAgo(dateString);
}
else
{
release.PublishDate = DateTime.ParseExact(dateString, "yyyy-MM-ddHH:mm:ss", CultureInfo.InvariantCulture);
}
var sizeStr = qRow.Children().ElementAt(7).Cq().Text().Split(new char[] { '/' })[0];
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(qRow.Children().ElementAt(8).Cq().Text().Split(new char[] { '/' })[0].Trim());
release.Peers = ParseUtil.CoerceInt(qRow.Children().ElementAt(8).Cq().Text().Split(new char[] { '/' })[1].Trim()) + release.Seeders;
release.Files = ParseUtil.CoerceLong(qRow.Find("td:nth-child(5)").Text());
release.Grabs = ParseUtil.CoerceLong(qRow.Find("a[href^=\"snatches.php?id=\"]").Text().Split(' ')[0]);
release.DownloadVolumeFactor = 0;
release.UploadVolumeFactor = 1;
var desc = qRow.Find("td:nth-child(2)");
desc.Find("a").Remove();
desc.Find("small").Remove(); // Remove release name (if enabled in the user cp)
release.Description = desc.Text().Trim(new char[] { '-', ' ' });
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
}
return(releases);
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var searchString = query.GetQueryString();
var queryCollection = new NameValueCollection();
queryCollection.Add("view", "0");
if (!string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("search", searchString);
}
foreach (var cat in MapTorznabCapsToTrackers(query))
{
queryCollection.Add(string.Format("c{0}", cat), "1");
}
var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString();
var results = await RequestStringWithCookiesAndRetry(searchUrl, CookieHeader);
try
{
CQ dom = results.Content;
var rows = dom["#torrentBrowse > table > tbody > tr"];
foreach (var row in rows.Skip(1))
{
var release = new ReleaseInfo();
CQ qRow = row.Cq();
//CQ qLink = qRow.Children().ElementAt(1).Cq().Children("a").ElementAt(1).Cq();
CQ qLink;
CQ qTmp = qRow.Children().ElementAt(1).Cq().Find("a");
if (qTmp.Length < 2)
{
qLink = qRow.Children().ElementAt(1).Cq().Find("a").ElementAt(0).Cq();
}
else
{
qLink = qRow.Children().ElementAt(1).Cq().Find("a").ElementAt(1).Cq();
}
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
release.Title = qLink.Attr("title");
release.Description = release.Title;
release.Guid = new Uri(SiteLink + qLink.Attr("href").TrimStart('/'));
release.Comments = release.Guid;
qLink = qRow.Children().ElementAt(3).Cq().Children("a").First();
release.Link = new Uri(string.Format("{0}{1}", SiteLink, qLink.Attr("href")));
var catUrl = qRow.Children().ElementAt(0).FirstElementChild.Cq().Attr("href");
var catNum = catUrl.Split(new char[] { '=', '&' })[2].Replace("c", "");
release.Category = MapTrackerCatToNewznab(catNum);
var dateString = qRow.Children().ElementAt(6).Cq().Text().Trim();
//var pubDate = DateTime.ParseExact(dateString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
release.PublishDate = Jackett.Utils.DateTimeUtil.FromTimeAgo(dateString);
var sizeStr = qRow.Children().ElementAt(7).Cq().Text().Split(new char[] { '/' })[0];
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(qRow.Children().ElementAt(8).Cq().Text().Split(new char[] { '/' })[0].Trim());
release.Peers = ParseUtil.CoerceInt(qRow.Children().ElementAt(8).Cq().Text().Split(new char[] { '/' })[1].Trim()) + release.Seeders;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
