public static bool BuildSitefinityReference(this CQ htmlElement)
{
bool result = false;
string imgFile = Path.GetFileName(htmlElement.Attr("src"));
var sfImg = App.WorkWith().Images().Where(i => i.Status == ContentLifecycleStatus.Master).Get().ToList().Where(i => Path.GetFileName(i.FilePath) == imgFile.ToLower()).FirstOrDefault();
if (sfImg != null)
{
var manager = LibrariesManager.GetManager();
var album = manager.GetAlbums().Where(a => a.Id == sfImg.Album.Id).FirstOrDefault();
string urlRoot = Telerik.Sitefinity.Configuration.Config.Get <LibrariesConfig>().Images.UrlRoot;
var sfImgLive = App.WorkWith().Image(sfImg.Id).GetLive().Get();
var sfRef = "[" + urlRoot + "|" + (LibrariesDataProvider)album.Provider + "]" + sfImgLive.Id;
htmlElement.Attr("sfref", sfRef);
string relativeUrl = new Uri(sfImg.MediaUrl).AbsolutePath;
htmlElement.Attr("src", relativeUrl);
result = true;
}
return(result);
}
public async Task <IEnumerable <ReleaseInfo> > PerformQuery(TorznabQuery query)
{
var releases = new List <ReleaseInfo>();
var episodeSearchUrl = string.Format("{0}?search={1}&cat=0", SearchUrl, HttpUtility.UrlEncode(query.GetQueryString()));
var results = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
try
{
CQ dom = results.Content;
var table = dom["tbody > tr > .latest"].Parent().Parent();
foreach (var row in table.Children().Skip(1))
{
var release = new ReleaseInfo();
CQ qDetailsCol = row.ChildElements.ElementAt(1).Cq();
CQ qLink = qDetailsCol.Children("a").First();
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800;
release.Comments = new Uri(SiteLink + "/" + qLink.Attr("href"));
release.Guid = release.Comments;
release.Title = qLink.Attr("title");
release.Description = release.Title;
//"Tuesday, June 11th 2013 at 03:52:53 AM" to...
//"Tuesday June 11 2013 03:52:53 AM"
var timestamp = qDetailsCol.Children("font").Text().Trim() + " ";
var timeParts = new List <string>(timestamp.Replace(" at", "").Replace(",", "").Split(' '));
timeParts[2] = Regex.Replace(timeParts[2], "[^0-9.]", "");
var formattedTimeString = string.Join(" ", timeParts.ToArray()).Trim();
var date = DateTime.ParseExact(formattedTimeString, "dddd MMMM d yyyy hh:mm:ss tt", CultureInfo.InvariantCulture);
release.PublishDate = DateTime.SpecifyKind(date, DateTimeKind.Utc).ToLocalTime();
release.Link = new Uri(SiteLink + "/" + row.ChildElements.ElementAt(2).Cq().Children("a.index").Attr("href"));
var sizeStr = row.ChildElements.ElementAt(6).Cq().Text();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(row.ChildElements.ElementAt(8).Cq().Text());
release.Peers = ParseUtil.CoerceInt(row.ChildElements.ElementAt(9).Cq().Text()) + release.Seeders;
//if (!release.Title.ToLower().Contains(title.ToLower()))
// continue;
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}
return(releases);
}
public bool SendSms(string recipient, string message)
{
if (IsLoggedIn == false)
{
throw new Exception("Not logged in");
}
string cookieVal = CookieJar.GetCookies(new Uri(base_url))["JSESSIONID"].Value;
cookieVal = cookieVal.Substring(cookieVal.IndexOf('~') + 1);
CQ sendSmsPage = Client.DownloadString(base_url + "SendSMS?id=" + cookieVal);
NameValueCollection data = new NameValueCollection();
//all inputs
CQ form = sendSmsPage.Find("form[id=frm_sendsms]");
CQ inputs = form.Find("input[type=hidden]");
foreach (var input in inputs)
{
CQ inp = input.Cq();
data.Add(inp.Attr("name"), inp.Attr("value"));
}
//sms input
CQ mobileNumberBox = form.Find("input[placeholder='Enter Mobile Number or Name']")[0].Cq();
data.Add(mobileNumberBox.Attr("name"), recipient);
//textarea
data.Add("sendSMSMsg", message);
string sendSmsPost = base_url + data["fkapps"];
data["hid_exists"] = "no";
data["maxwellapps"] = cookieVal;
//additional vsls
data.Add("messid_0", "");
data.Add("messid_1", "");
data.Add("messid_2", "");
data.Add("messid_3", "");
data.Add("messid_4", "");
data.Add("newsExtnUrl", "");
data.Add("reminderDate", DateTime.Now.ToString("dd-MM-yyyy"));
data.Add("sel_hour", "");
data.Add("sel_minute", "");
data.Add("ulCategories", "29");
Client.UploadValues(sendSmsPost, data);
return(true);
}
private void AddLink(StringBuilder headLinks, IDomObject el)
{
CQ cq = CQ.Create(el);
string relAttr = cq.Attr("rel");
string typeAttr = cq.Attr("type");
string hrefAttr = cq.Attr("href");
var obj = new
{
rel = string.IsNullOrEmpty(relAttr) ? "" : "rel=\"{0}\""._Format(relAttr),
type = string.IsNullOrEmpty(typeAttr) ? "" : "type=\"{0}\""._Format(typeAttr),
href = string.IsNullOrEmpty(hrefAttr) ? "" : "href=\"{0}\""._Format(hrefAttr)
};
headLinks.Append("<link {rel} {type} {href}>".NamedFormat(obj));
}
// 全巡回
public async Task DoAll()
{
string menuUrl = "http://menu.2ch.net/bbsmenu.html";
string menuHtml = await HtmlGetter.GetHtml(menuUrl, "shift_jis");
new CQ(menuHtml).Find("a").Each(_a =>
{
var a = new CQ(_a);
var url = a.Attr("href");
var name = a.Text().Trim();
if (Regex.IsMatch(url, "^http://[^/]+/[^/]+/$"))
{
Console.WriteLine("-----------");
Console.WriteLine(url);
Console.WriteLine(name);
Task.Run(async() =>
{
await ScrapeBoard(url, name);
}).Wait();
}
});
// JSON出力
OutputJson();
}
// GET: Forex
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.financemagnates.com/forex");
CQ mainArticle = dom.Find("ul.featured_cats_wrapper").Eq(0);
for (int i = 0; i < mainArticle.Children().Length; i++)
{
CQ article = mainArticle.Children().Eq(i).Find("a").Eq(0);
newsData nData = new newsData();
nData.text = article.Text();
nData.alt_url = article.Attr("href").ToString();
nData.url = "";
newsList.Add(nData);
}
CQ addArticle = dom.Find("div.sub-feature").Eq(0);
newsData firstNews = new newsData();
CQ first = addArticle.Children("article").Eq(0);
firstNews.text = first.Find("a").Eq(0).Find("div").Eq(0).Text();
firstNews.url = first.Find("a").Eq(0).Css("background-image");
firstNews.alt_url = first.Find("a").Eq(0).Attr("href").ToString();
newsList.Add(firstNews);
newsData secondNews = new newsData();
CQ second = addArticle.Children("article").Eq(1);
secondNews.text = second.Find("a").Eq(0).Find("div").Eq(0).Text();
secondNews.url = second.Find("a").Eq(0).Css("background-image");
secondNews.alt_url = second.Find("a").Eq(0).Attr("href").ToString();
newsList.Add(secondNews);
ViewBag.newsList = newsList;
return(View());
}
public ActionResult getDetail(string url)
{
CQ dom = CQ.CreateFromUrl("http://www.alhurra.com" + url);
string aTitle = "";
string aImg_url = "";
string aText = "";
CQ cTitle = dom.Find("h1.pg-title").Eq(0);
if (cTitle.Length != 0)
{
aTitle = cTitle.Text();
}
CQ cImg = dom.Find(".cover-media").Eq(0).Find("img").Eq(0);
if (cImg.Length != 0)
{
aImg_url = cImg.Attr("src").ToString();
}
CQ cText = dom.Find(".wsw").Eq(0);
if (cText.Length != 0)
{
aText = cText.RenderSelection().ToString();
}
ViewBag.aTitle = aTitle;
ViewBag.aImg = aImg_url.Replace("_w250_r1", "_w1023_r1");
ViewBag.aText = aText;
return(View());
}
/// <summary>
/// Parses the node in an intelligent fashion, trying to find selected optoins, values, and finally inner text..
/// </summary>
/// <param name="node">The node to parse.</param>
/// <param name="attribute">The attribute corresponding to the property being bound.</param>
/// <returns>A string containing either the selected option's value, the value attribute, or the inner text of the node.</returns>
private string ParseSmart(CQ node, HtmlElementAttribute attribute)
{
var option = node.Select("option[selected]", node).FirstOrDefault();
var input = node.Select("input", node).FirstOrDefault();
return(option?.InnerText?.TrimAll() ?? input?.Attributes["value"] ?? node.Attr("value") ?? node.Text()?.TrimAll());
}
public override async Task <ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LandingUrl);
CQ dom = loginPage.Content;
CQ qCaptchaImg = dom.Find("img#regimage").First();
if (qCaptchaImg.Length > 0)
{
var CaptchaUrl = qCaptchaImg.Attr("src");
var captchaImage = await RequestBytesWithCookies(CaptchaUrl, loginPage.Cookies, RequestType.GET, LandingUrl);
var CaptchaImage = new ImageItem {
Name = "Captcha Image"
};
var CaptchaText = new StringItem {
Name = "Captcha Text"
};
CaptchaImage.Value = captchaImage.Content;
configData.AddDynamic("CaptchaImage", CaptchaImage);
configData.AddDynamic("CaptchaText", CaptchaText);
}
else
{
logger.Debug(string.Format("{0}: No captcha image found", ID));
}
return(configData);
}
public Task <WebPage> Submit(bool asJson = false)
{
var formurl = formNode.Attr("action") ?? "";
var uri = new Uri(baseUri, formurl);
return(browser.OpenWithFormData(uri.ToString(), Method, formValues, asJson));
}
private CQ Eval(Step step, CQ html)
{
string[] separators = { " " };
// Multiple filters can be combined with a space
var filters = step.Filter.Split(separators, StringSplitOptions.RemoveEmptyEntries);
switch (step.Action)
{
case "select":
var result = string.Empty;
foreach (var filter in filters)
{
result += html[filter].RenderSelection().Trim() + "\n";
}
return(result);
case "remove":
foreach (var filter in filters)
{
var selection = html[filter];
selection.Remove();
}
return(html);
case "attr":
return(html.Attr(filters[0]));
default:
return(html);
}
}
public override int GetPrice()
{
try
{
CQ Dom = CQ.CreateFromUrl(this.link);
CQ priceNode = Dom["meta[itemprop='price']"];
if (priceNode.Length > 0)
{
string s = priceNode.Attr("content");
if (s != "")
{
return(Convert.ToInt32(s));
}
else
{
return(-1);
}
}
return(-1);
}
catch (Exception e)
{
CConsole.GetInstance().LogError(e.Message);
return(-1);
}
}
public override async Task <ConfigurationData> GetConfigurationForSetup()
{
// Get the login form as we need the CSRF Token
var loginPage = await webclient.GetString(new Utils.Clients.WebRequest()
{
Url = LoginUrl,
Encoding = Encoding,
});
UpdateCookieHeader(loginPage.Cookies);
CQ loginPageDom = loginPage.Content;
csrfIndex = loginPageDom["input[name=\"_CSRF_INDEX\"]"].Last().Attr("value");
csrfToken = loginPageDom["input[name=\"_CSRF_TOKEN\"]"].Last().Attr("value");
CQ qCaptchaImg = loginPageDom.Find("#captcha_img").First();
if (qCaptchaImg.Length == 1)
{
var CaptchaUrl = SiteLink + qCaptchaImg.Attr("src");
var captchaImage = await RequestBytesWithCookies(CaptchaUrl, loginPage.Cookies);
configData.CaptchaImage.Value = captchaImage.Content;
}
else
{
configData.CaptchaImage.Value = new byte[0];
}
configData.CaptchaCookie.Value = loginPage.Cookies;
return(configData);
}
private ParseResults HandleWinner(CQ prizeDivs, CQ redeemAnchor)
{
var results = new ParseResults();
results.IsWinner = true;
results.IsReplay = false;
results.Prizes = prizeDivs.Select(x => CreatePrizeFromDom(x)).ToList();
results.FollowUpUrl = redeemAnchor.Attr("href");
return(results);
}
private void AddScript(StringBuilder scriptTags, IDomObject el)
{
CQ cq = CQ.Create(el);
var obj = new
{
src = "src=\"{0}\""._Format(cq.Attr("src"))
};
scriptTags.Append("<script {src}></script>".NamedFormat(obj));
}
private ParseResults HandleReplay(CQ replayDiv)
{
var results = new ParseResults();
results.IsWinner = false;
results.IsReplay = true;
results.FollowUpUrl = replayDiv.Attr("href");
return(results);
}
// GET: CryptoCoin
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.cryptocoinsnews.com");
CQ mainArticle = dom.Find("div.grid-wrapper").Eq(0);
for (int i = 0; i < mainArticle.Children(".post").Length; i++)
{
CQ article = mainArticle.Children(".post").Eq(i).Find("a").Eq(0);
newsData nData = new newsData();
nData.text = article.Attr("title").ToString();
nData.alt_url = article.Attr("href").ToString();
nData.url = "";
CQ img = article.Children("img").Eq(0);
nData.url = img.Attr("src").ToString();
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
private void ImportContentImages(string content, string postTitle, string blogTitle)
{
CQ document = content;
if (!String.IsNullOrWhiteSpace(document.Render()))
{
var images = document.Select("img");
images.Each((element) =>
{
CQ cqElement = new CQ(element);
string imageSrc = cqElement.Attr("src");
string altText = cqElement.Attr("alt");
string imgFile = Path.GetFileName(imageSrc);
ImportImage(imageSrc, postTitle, blogTitle);
});
}
}
// GET: Alarabiya
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.alhurra.com/p/349.html");
newsData topNews = new newsData();
CQ tNews = dom.Find(".img-overlay").Eq(0).Find("a").Eq(0);
if (tNews.Length != 0)
{
topNews.alt_url = tNews.Attr("href").ToString();
topNews.text = tNews.Attr("title").ToString();
}
CQ cImg = tNews.Find("img").Eq(0);
if (cImg.Length != 0)
{
topNews.url = cImg.Attr("data-src").ToString();
}
newsList.Add(topNews);
CQ mainArticle = dom.Find("#wrowblock-145_12").Eq(0).Find("li");
for (int i = 0; i < mainArticle.Length; i++)
{
CQ article = mainArticle.Eq(i).Find("a").Eq(0);
newsData nData = new newsData();
nData.text = article.Attr("title").ToString();
nData.alt_url = article.Attr("href").ToString();
nData.url = "";
CQ img = article.Find("img").Eq(0);
if (img.Length != 0)
{
nData.url = img.Attr("data-src").ToString();
}
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
private void ImportContentImages(string content, string postTitle, string blogTitle)
{
CQ document = content;
if (!String.IsNullOrWhiteSpace(document.Render()))
{
var images = document.Select("img");
images.Each((element) =>
{
CQ cqElement = new CQ(element);
string imageSrc = cqElement.Attr("src");
string altText = cqElement.Attr("alt");
string imgFile = Path.GetFileName(imageSrc);
ImportImage(imageSrc, postTitle, blogTitle);
});
// Repeat this process for any links that point to image
// resources.
var imageLinks = document.Select("a");
imageLinks.Each((element) =>
{
CQ cqElement = new CQ(element);
string imageSrc = cqElement.Attr("href");
if (imageSrc.EndsWith(".jpg") ||
imageSrc.EndsWith(".png") ||
imageSrc.EndsWith(".gif") ||
imageSrc.EndsWith(".tiff"))
{
ImportImage(imageSrc, postTitle, blogTitle);
}
});
}
}
public override async Task <ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
CQ dom = loginPage.Content;
CQ qCaptchaImg = dom.Find("td.tablea > img").First();
var CaptchaUrl = SiteLink + qCaptchaImg.Attr("src");
var captchaImage = await RequestBytesWithCookies(CaptchaUrl, loginPage.Cookies);
configData.CaptchaImage.Value = captchaImage.Content;
configData.CaptchaCookie.Value = loginPage.Cookies;
return(configData);
}
// GET: Arabbit
public ActionResult Index()
{
ArrayList newsList = new ArrayList();
CQ dom = CQ.CreateFromUrl("http://www.arabbit.net");
CQ mainArticle = dom.Find("div.grid-wrapper").Eq(0);
for (int i = 0; i < mainArticle.Children().Length; i++)
{
newsData nData = new newsData();
CQ article = mainArticle.Children().Eq(i).Find("a").Eq(0);
string aText = "";
string aImg = "";
string aAlt = "";
if (article.Length != 0)
{
aText = article.Attr("title").ToString();
aAlt = article.Attr("href").ToString();
}
nData.text = aText;
nData.alt_url = aAlt;
nData.url = "";
CQ img = article.Children("img").Eq(0);
if (img.Length != 0)
{
aAlt = img.Attr("src").ToString();
}
nData.url = aAlt;
newsList.Add(nData);
}
ViewBag.newsList = newsList;
return(View());
}
public string Parse(CQ node, HtmlElementAttribute attributeInstance)
{
if (node == null)
{
throw new ArgumentNullException(nameof(node));
}
if (attributeInstance == null)
{
throw new ArgumentNullException(nameof(attributeInstance));
}
switch (attributeInstance.BindingBehavior)
{
case BindingBehavior.InnerHtml:
return(node.Html());
case BindingBehavior.Class:
return(node.Attr("class"));
case BindingBehavior.Id:
return(node.Attr("id"));
case BindingBehavior.Value:
return(node.Attr("value"));
case BindingBehavior.Attribute:
return(node.Attr(attributeInstance.AttributeName));
case BindingBehavior.InnerText:
return(node.Text()?.Trim());
case BindingBehavior.Smart:
default:
return(ParseSmart(node, attributeInstance));
}
}
public List <MusicSummary> Search([FromUri] PageParams param)
{
List <MusicSummary> list = new List <MusicSummary>();
string url = $"http://music.baidu.com/search/song?s=1&key={param.Keyword ?? ""}&jump=0&start={(param.Current - 1) * param.Size}&size={param.Size}&third_type=0";
CQ csquery = Http.GetHttpValue(url);
CQ items = csquery.Find("li.bb-dotimg");
for (int i = 0; i < items.Length; i++)
{
CQ item = items.Eq(i);
CQ csTitle = item.Find("span.song-title a");
string title = csTitle.Attr("title");
string ids = Regex.Match(csTitle.Attr("data-songdata") ?? "", "\\d+").Value;
MusicSummary summary = new MusicSummary
{
ArtistName = item.Find("span.author_list").Text().Trim(),
AlbumName = item.Find("span.album-title").Text().Trim(),
SongName = title,
SongId = Convert.ToInt32(ids)
};
list.Add(summary);
}
return(list);
}
public Task <WebPage> Follow(CQ node)
{
if (node.Length == 0)
{
throw new ArgumentException("No nodes found");
}
if (!node.Is("a"))
{
throw new ArgumentException("Node is not a link");
}
var url = node.Attr("href");
return(Open(url));
}
public override async Task <ConfigurationData> GetConfigurationForSetup()
{
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
CQ dom = loginPage.Content;
CQ recaptchaScript = dom.Find("script").First();
string recaptchaSiteKey = recaptchaScript.Attr("src").Split('=')[1];
var result = new ConfigurationDataRecaptchaLogin();
result.CookieHeader.Value = loginPage.Cookies;
result.Captcha.SiteKey = recaptchaSiteKey;
result.Captcha.Version = "1";
return(result);
}
public static Item[] ParsePoeTrade(string element)
{
var items = new List <Item>();
var doc = CQ.Create(element);
doc["div>.search-results>.item"].Each(domObject => {
var domItem = new CQ(domObject);
var domFirstLine = domItem.Find(".first-line");
var domIcon = domFirstLine.Find(".icon-td>.icon");
var domSockets = domIcon.Find(".sockets>.sockets-inner");
var domItemCell = domFirstLine.Find(".item-cell");
var domRequirements = domItemCell.Find(".requirements");
var domMods = domItemCell.Find(".item-mods>li>ul");
var item = new Item {
Seller = domItem.Attr("data-seller"),
Buyout = domItem.Attr("data-buyout"),
Ign = domItem.Attr("data-ign"),
League = domItem.Attr("data-league"),
Name = domItem.Attr("data-name"),
Tab = domItem.Attr("data-tab"),
X = int.Parse(domItem.Attr("data-x")),
Y = int.Parse(domItem.Attr("data-y")),
Icon = domIcon["img"].Attr("src"),
Sockets = ExtractSockets(domSockets),
Corrupted = !domItemCell["h5>a>span.corrupted"].IsNullOrEmpty(),
Requirements = ExtractRequirements(domRequirements),
Influence = ExtractInfluence(domMods),
Mods = ExtractMods(domMods),
Stats = ExtractStats(domFirstLine.Find(".table-stats>table>tbody"))
};
// Get item stack and sockets
item.StackSize = ExtractStackSize(item.Icon);
item.LargestLink = FindLargestLink(item.Sockets);
items.Add(item);
});
return(items.ToArray());
}
public override int GetPrice()
{
try
{
CQ dom = CQ.CreateFromUrl(this.link);
CQ priceNode = dom["#our_price_display"];
if (priceNode.Length > 0)
{
return(Convert.ToInt32(priceNode.Attr("content")));
}
return(-1);
}
catch (Exception e)
{
CConsole.GetInstance().LogError(e.Message);
return(-1);
}
}
private async Task <string> getHebName(string searchTerm)
{
const string site = "http://thetvdb.com";
var url = site + "/index.php?searchseriesid=&tab=listseries&function=Search&";
url += "string=" + searchTerm; // eretz + nehedert
var results = await RequestStringWithCookies(url);
CQ dom = results.Content;
int rowCount = 0;
var rows = dom["#listtable > tbody > tr"];
foreach (var row in rows)
{
if (rowCount < 1)
{
rowCount++;
continue;
}
CQ qRow = row.Cq();
CQ link = qRow.Find("td:nth-child(1) > a");
if (link.Text().Trim().ToLower() == searchTerm.Trim().ToLower())
{
var address = link.Attr("href");
if (string.IsNullOrEmpty(address))
{
continue;
}
var realAddress = site + address.Replace("lid=7", "lid=24");
var realData = await RequestStringWithCookies(realAddress);
CQ realDom = realData.Content;
return(realDom["#content:nth-child(1) > h1"].Text());
}
}
return(string.Empty);
}
private VSGalleryQAThread TryParseThread(CQ element)
{
int threadId;
if (!int.TryParse(element.Attr("threadid"), out threadId))
{
return(null);
}
var thread = new VSGalleryQAThread();
thread.ThreadId = threadId;
thread.Title = element.Children("div.threadHeader")
.Children("div.title").First().Text();
thread.Posts = new List <VSGalleryQAPost>();
foreach (var p in element.Find("li.post"))
{
var post = TryParsePost(p.Cq());
if (post != null)
{
thread.Posts.Add(post);
}
}
if (thread.Posts.Count == 0)
{
return(null);
}
var firstPost = thread.Posts.First();
thread.StartedByName = firstPost.PostedByName;
thread.StartedByUserId = firstPost.PostedByUserId;
thread.StartedOn = firstPost.PostedOn;
thread.LastPostOn = thread.Posts.Last().PostedOn;
return(thread);
}