/// <summary>
/// 返回 报关单号, 报关单pre_entry_id
/// </summary>
/// <param name="htmlInfo"></param>
/// <returns></returns>
internal static Dictionary<string, string> Parse查询所有报关单号2(string htmlInfo)
{
Dictionary<string, string> dic = null;
HtmlAgilityPack.HtmlDocument _html = new HtmlAgilityPack.HtmlDocument();
_html.LoadHtml(htmlInfo);
HtmlAgilityPack.HtmlNode node_table = _html.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[2]");
if (node_table != null)
{
dic = new Dictionary<string, string>();
int trCount = new Regex("<tr>").Matches(node_table.InnerHtml).Count;
for (int i = 2; i <= trCount; i++)
{
//报关单号
HtmlAgilityPack.HtmlNode node_td1 = _html.DocumentNode.SelectSingleNode(string.Format("/html[1]/body[1]/table[2]/tr[{0}]/td[2]/div[1]/a[1]", i));
//地址
HtmlAgilityPack.HtmlNode node_td2 = _html.DocumentNode.SelectSingleNode(string.Format("/html[1]/body[1]/table[2]/tr[{0}]/td[7]/div[1]/a[1]", i));
string url = null;
if (node_td2 != null)
{
//去掉“../”，拼成报关单快照url
//url = "http://www.nbedi.com/h2000eport/pre_bgd/" + node_td2.InnerText.Substring(3);
//报关单pre_entry_id
string att = node_td2.Attributes["href"].Value;
url = att.Substring(att.LastIndexOf("=") + 1);
}
dic.Add(node_td1.InnerText, url);
}
}
return dic;
}
public Salestate[] GetSalestates()
{
var http_salestate_list_url = "https://creator.toranoana.jp/portal/salestate_list.cgi";
var salestate_page_key = "salestate_page";
WebClientEx webClient = new WebClientEx();
webClient.CookieContainer = Cookie;
// 売上ページ一覧を取得
var buffer = webClient.DownloadData(http_salestate_list_url);
var htmlString = ConvertHtmlBufferToString(buffer);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(htmlString);
// 売上ページのみへのリンク取得
var saleStateLinks = doc.DocumentNode.SelectNodes("//a").Where(_=>_.GetAttributeValue("href", "").Contains(salestate_page_key));
var baseURI = new Uri(http_salestate_list_url);
List<Salestate> salestates = new List<Salestate>();
foreach (var link in saleStateLinks)
{
// 売上ページを走査
var uri = new Uri( baseURI, link.GetAttributeValue("href", ""));
salestates.Add(GetSalestate(webClient, uri.ToString()));
}
return salestates.ToArray();
}
static void Main(string[] args)
{
string html =
@"<div class=""article-content"">
<p>text I want text I want text I want text I want <strong> TEXT I WANT TOO </strong></p>
<p>text I want text I want text I want text I want <strong> TEXT I WANT TOO </strong></p>
</div>";
using (WebClient client = new WebClient())
{
client.Encoding = System.Text.Encoding.Unicode;
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
var nodes = doc.DocumentNode.SelectNodes("//div[@class='article-content']/p");
nodes.ToList().ForEach(n =>
{
Console.WriteLine(n.InnerText);
});
Console.ReadLine();
}
}
List<Show> Grab(GrabParametersBase p, ILogger logger)
{
var pp = (GrabParameters)p;
var url = string.Format(URL, pp.Channel.ToString().Replace("_", "-").Replace("AANNDD", "%26").Replace("PPLLUUSS", "%2B"));
var wr = WebRequest.Create(url);
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
logger.WriteEntry(string.Format("Grabbing Channel {0}", pp.Channel), LogType.Info);
doc.Load(res.GetResponseStream());
var shows = new List<Show>();
foreach (Day d in Enum.GetValues(typeof(Day)))
{
var dayOfWeek = (DayOfWeek)d;
var div = doc.DocumentNode.Descendants("div").FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == d.ToString());
if (div != null)
{
var date = NextDateOfDayOfWeek(dayOfWeek);
foreach (var ul in div.Descendants("ul"))
{
foreach (var li in ul.Descendants("li"))
{
var par = li.Descendants("p").First();
var a = li.Descendants("a").First();
var show = new Show();
show.Channel = pp.Channel.ToString();
show.Title = a.InnerText.Trim();
show.StartTime = DateTime.SpecifyKind(date + Convert.ToDateTime(par.InnerText.Trim()).TimeOfDay, DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
shows.Add(show);
}
}
}
}
return shows;
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
var shows = new List<Show>();
var doc = XDocument.Parse(xmlParameters);
var sdElement = doc.Descendants("StartDate").FirstOrDefault();
var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
var edElement = doc.Descendants("EndDate").FirstOrDefault();
var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;
for (int i =0; i <= endDateDays; i++)
{
var date = DateTime.Now.Date.AddDays(i);
logger.WriteEntry(string.Format("Grabbing reshet.tv for date {0}", date.ToString("d")), LogType.Info);
var wr = WebRequest.Create(Url);
wr.Method = "POST";
wr.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
using (var sw = new StreamWriter(wr.GetRequestStream()))
{
sw.Write(string.Format("Values={0}%2F{1}%2F{2}",date.Day.ToString("00"),date.Month.ToString("00"),date.Year));
}
var res = (HttpWebResponse)wr.GetResponse();
var html = new HtmlAgilityPack.HtmlDocument();
html.Load(res.GetResponseStream(),Encoding.UTF8);
foreach (var li in html.DocumentNode.Descendants("li"))
{
var time = li.Descendants("span").First().InnerText;
var text = li.Descendants("p").First().InnerText;
}
}
return shows;
}
/// <summary>
/// Get all transfers data as list of strings.
/// </summary>
/// <param name="htmlDocument">HTML document string representation.</param>
/// <returns>String list of data.</returns>
public List<String> GetTransfersData(string htmlDocument)
{
// Data container.
List<string> dataToSave = new List<string>();
// Load downloaded website.
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(htmlDocument);
// Find proper HTML element with transfers 'table'.
HtmlAgilityPack.HtmlNode div = htmlDoc.DocumentNode.SelectSingleNode("//div[@class='panes']");
// Get all players with data, each data element as separate list element.
if (div != null)
{
dataToSave = div.Descendants("li")
.Select(a => a.InnerText)
.ToList();
}
else
{
throw new Exception("Could not find div with transfers!");
}
return dataToSave;
}
public override int DiscoverDynamicCategories()
{
int dynamicCategoriesCount = 0;
String baseWebData = GetWebData(PrimaUtil.categoriesUrl, forceUTF8: true);
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(baseWebData);
HtmlAgilityPack.HtmlNodeCollection categories = document.DocumentNode.SelectNodes(".//div[@class='programs-menu-genres']/*/a");
foreach (var category in categories)
{
this.Settings.Categories.Add(
new RssLink()
{
Name = category.InnerText,
HasSubCategories = true,
Url = Utils.FormatAbsoluteUrl(category.Attributes["href"].Value, PrimaUtil.categoriesUrl)
});
dynamicCategoriesCount++;
}
this.Settings.DynamicCategoriesDiscovered = true;
return dynamicCategoriesCount;
}
List<Show> Grab(GrabParametersBase p)
{
var shows = new List<Show>();
try
{
var param = (GrabParameters)p;
var wr = WebRequest.Create(string.Format(urlFormat, (int)param.ChannelId));
_logger.WriteEntry(string.Format("Grabbing Channel {0} ...", param.ChannelId), LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(res.GetResponseStream());
doc.OptionOutputAsXml = true;
var writer = new StringWriter();
doc.Save(writer);
var xml = XDocument.Load(new StringReader(writer.ToString()));
FillShows(xml, shows);
for (int i = shows.Count - 1; i >= 0; i--)
{
var show = shows[i];
show.Channel = param.ChannelId.ToString();
if (i == shows.Count - 1)
show.EndTime = show.StartTime.AddHours(12);// usually 3-4 days from now , not that important
else
show.EndTime = shows[i + 1].StartTime;
}
}
catch (Exception ex)
{
_logger.WriteEntry(ex.Message, LogType.Error);
}
_logger.WriteEntry(string.Format("Found {0} Shows", shows.Count), LogType.Info);
return shows;
}
/// <summary>
/// Converts onlyconnect.IHTMLDocument2 class to HtmlAgilityPack.HtmlDocument
/// </summary>
/// <param name="doc">IHTMLDocument2 document</param>
/// <returns>Converted HtmlDocument</returns>
public static HtmlAgilityPack.HtmlDocument mshtmlDocToAgilityPackDoc(onlyconnect.IHTMLDocument2 doc)
{
HtmlAgilityPack.HtmlDocument outDoc = new HtmlAgilityPack.HtmlDocument();
string html = doc.GetBody().innerHTML;
outDoc.LoadHtml(html);
return outDoc;
}
private async Task ParseFromAddress(string url)
{
var response = await new System.Net.Http.HttpClient().GetStreamAsync(url);
var page = new HtmlAgilityPack.HtmlDocument();
page.Load(response);
var itemNodes = page.DocumentNode.SelectNodes("//form[@id='list-form']/div[@class='list-items']/div[@class='percent-wrap']");
foreach (var item in itemNodes)
{
var titleNode = item.SelectNodes(".//p[@class='title']/a").FirstOrDefault();
var priceNode = item.SelectNodes(".//p[@class='price']").FirstOrDefault();
string link = titleNode.Attributes["href"].Value;
string title = titleNode.Attributes["title"].Value;
double price;
double.TryParse(priceNode.InnerText.Trim().Replace("US$", "").Replace(".",","), out price);
var lumenMatch = new Regex(@"\d{3,4}[-\s]?(lm|lumen)", RegexOptions.IgnoreCase).Match(title);
if (!lumenMatch.Groups[0].Success)
continue;
string lumenCountString = Regex.Replace(lumenMatch.Groups[0].Value, "[^0-9]", "");
int lumenCount;
if (!int.TryParse(lumenCountString, out lumenCount))
continue;
Items.Add(new RowItem
{
LumenCount = lumenCount,
Price = price,
Link = "http://dx.com" + link
});
}
this.Title = Items.Count.ToString();
}
public static async Task<DataGroup> NewsGoVnGroup_Parse(string xmlString, DataModel.DataGroup group, int takeNum)
{
StringReader _stringReader = new StringReader(xmlString);
XDocument _xdoc = XDocument.Load(_stringReader);
var channelElement = _xdoc.Element("rss").Element("channel");
if (channelElement != null)
{
group.Title = channelElement.Element("title").Value;
group.Subtitle = channelElement.Element("title").Value;
group.Description = channelElement.Element("description").Value;
var items = channelElement.Elements("item");
foreach (var item in items)
{
if (group.Items.Count == takeNum && takeNum >= 0) break;
DataItem dataItem = new DataItem();
dataItem.Title = item.Element("title").Value;
dataItem.Description = StripHTML(item.Element("description").Value);
dataItem.Link = new Uri(item.Element("link").Value, UriKind.Absolute);
dataItem.PubDate = item.Element("pubDate").Value;
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.Load(new StringReader(item.Element("description").Value));
HtmlAgilityPack.HtmlNode imageLink = getFirstNode("img", htmlDoc.DocumentNode);
dataItem.ImageUri = new Uri(imageLink.GetAttributeValue("src", string.Empty).Replace("96.62.jpg", "240.155.jpg"), UriKind.Absolute);
dataItem.Group = group;
group.Items.Add(dataItem);
}
}
return group;
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
logger.WriteEntry("Grabbing jn1 schedule", LogType.Info);
var wr = WebRequest.Create(URL);
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(res.GetResponseStream());
var lst = new List<Show>();
var ul = doc.DocumentNode.Descendants("ul").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "program_list");
foreach (var li in ul.Descendants("li"))
{
var show = new Show();
show.Channel = "Jewish News One";
var startTime = li.Descendants("span").First().InnerText.Replace("::", ":");
show.StartTime = DateTime.SpecifyKind(DateTime.Now.Date + Convert.ToDateTime(startTime).TimeOfDay, DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
show.Title = li.Descendants("span").ToList()[1].InnerText.Trim().ToLower();
show.Title = show.Title.First().ToString().ToUpper() + String.Join("", show.Title.Skip(1));
show.Description = li.Descendants("span").ToList()[2].InnerText.Trim();
lst.Add(show);
}
var secondList = new List<Show>();
foreach (var show in lst)
{
var s = show.Clone(); // has same daily schedule every day , so just duplicate entries with tommorow date
s.StartTime = show.StartTime.AddDays(1);
secondList.Add(s);
}
lst.AddRange(secondList);
FixShowsEndTimeByStartTime(lst);
return lst;
}
public ProgramOptions Get(string configPath)
{
var html = new HtmlAgilityPack.HtmlDocument();
html.Load(configPath);
var options = new ProgramOptions();
options.BlogUrl = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/blogurl", "", true);
options.BlogUser = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/bloguser", "", true);
options.BlogPassword = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/blogpassword", "", true);
options.DatabaseUrl = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databaseurl", "", true);
options.DatabaseName = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databasename", "", true);
options.DatabaseUser = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databaseuser", "", true);
options.DatabasePassword = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databasepassword", "", true);
options.FtpUrl = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/ftpurl", "", true);
options.FtpUser = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/ftpuser", "", true);
options.FtpPassword = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/ftppassword", "", true);
options.ProxyAddress = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/proxyaddress", "", true);
options.ProxyPort = XmlParse.GetIntegerNodeValue(html.DocumentNode, "/programoptions/proxyport", 0);
options.UseProxy = XmlParse.GetBooleanNodeValue(html.DocumentNode, "/programoptions/useproxy", false);
options.YoutubeClient = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/youtubeclient", "", true);
options.YoutubeClientSecret = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/youtubeclientsecret", "", true);
return options;
}
/// <summary>
/// Gets school info: name and departments name from html polish Wikipedia page.
/// </summary>
/// <param name="filePath">Path to locally saved html file.</param>
/// <returns>School object with name and list of departments name.</returns>
public static School GetSchool(string filePath)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
var e = htmlDoc.Encoding;
htmlDoc.Load(filePath, Encoding.UTF8);
var school = new School();
if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0)
{
// Handle any parse errors as required
}
else
{
if (htmlDoc.DocumentNode != null)
{
var nodes = htmlDoc.DocumentNode.Descendants();
school.Name = nodes.First(x => x.Id == "firstHeading").InnerText;
var content = nodes.First(x => x.Id == "mw-content-text").Descendants().Where(x=>x.Name=="li").ToList();
foreach (var item in content)
{
if(item.InnerText.Contains("Wydział")) {
school.Departments.Add(item.InnerText);
}
}
}
}
return school;
}
private static HtmlAgilityPack.HtmlDocument LoadHtml()
{
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(Resource.GetString("Test1.html"));
return doc;
}
/// <summary>
/// Cleans up an HTML string by removing elements on the blacklist and all elements that start
/// with onXXX.
/// </summary>
/// <param name="html">The HTML string to sanitize.</param>
/// <returns>The sanitized version of the HTML.</returns>
public static string SanitizeHtml(string html)
{
// Load in the HTML string into an HTML document
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
// Sanitize the root document node
SanitizeHtmlNode(doc.DocumentNode);
string output = null;
// Use an XmlTextWriter to create self-closing tags
using (StringWriter sw = new StringWriter())
{
XmlWriter writer = new XmlTextWriter(sw);
doc.DocumentNode.WriteTo(writer);
output = sw.ToString();
// strip off XML doc header
if (!string.IsNullOrEmpty(output))
{
int at = output.IndexOf("?>");
output = output.Substring(at + 2);
}
writer.Close();
}
doc = null;
return output;
}
private AmazonProduct GetContent(String ASin)
{
var content = "";
var client = new WebClient();
var headers = new WebHeaderCollection();
headers.Add(HttpRequestHeader.Accept, "text/html, application/xhtml+xml, */*");
//headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
headers.Add(HttpRequestHeader.AcceptLanguage, "en-GB");
headers.Add(HttpRequestHeader.UserAgent, "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko");
client.Headers = headers;
var rawhtml = client.DownloadString("http://www.amazon.co.uk/dp/"+ ASin);
HtmlAgilityPack.HtmlDocument Html = new HtmlAgilityPack.HtmlDocument();
Html.LoadHtml(rawhtml);
var title = GetTitle(Html);
var description = GetDescription(Html);
AmazonProduct prod = new AmazonProduct() { Description = description, Title = title };
return prod;
}
public IPttRequest RequestWithCaptchaValue(IPttRequest request, IPttCaptcha pttCaptcha)
{
var htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(request.Response);
var challengeField = htmlDoc.DocumentNode.SelectSingleNode("//input[@id='recaptcha_challenge_field']");
if (challengeField == null)
{
return null;
}
var challengeFieldValue = challengeField.Attributes["value"].Value;
var pttRequestFactory = new PttRequestFactory(request);
var newReq = pttRequestFactory.Deserialize(string.Format("<Request><Url>{0}</Url><Method>POST</Method></Request>", request.Url));
newReq.PostValue = string.Format("recaptcha_challenge_field={0}&recaptcha_response_field={1}&submit=I%27m+a+human", challengeFieldValue, pttCaptcha.Value ?? "");
newReq.WrappedRequest.ContentType = "application/x-www-form-urlencoded";
var pttResponse = new PttResponse();
//bu gelen htmlsource dan textarea degerini oku
htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(pttResponse.GetResponse(newReq));
var valueToPasteNode = htmlDoc.DocumentNode.SelectSingleNode("//textarea");
if (valueToPasteNode == null)
{
return null;
}
return null;
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
var shows = new List<Show>();
var doc = XDocument.Parse(xmlParameters);
var sdElement = doc.Descendants("StartDate").FirstOrDefault();
var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
var edElement = doc.Descendants("EndDate").FirstOrDefault();
var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;
foreach (Channel c in Enum.GetValues(typeof(Channel)))
{
var wr = WebRequest.Create(string.Format(url, (int)c, DateTime.Now.Date.AddDays(startDateDiff).ToString(DateFormat), DateTime.Now.Date.AddDays(endDateDays).ToString(DateFormat)));
wr.Timeout = 30000;
logger.WriteEntry(string.Format("Grabbing hot.net.il channel {0} ", c.ToString()), LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
var html = new HtmlAgilityPack.HtmlDocument();
html.Load(res.GetResponseStream(), Encoding.UTF8);
foreach (var tr in html.DocumentNode.Descendants("tr").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "redtr_off"))
{
var tds = tr.Descendants("td").ToList();
var show = new Show();
show.Title = tds[2].InnerText;
show.StartTime = DateTime.SpecifyKind(Convert.ToDateTime(tds[4].InnerText),DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.Local, TimeZoneInfo.Utc);
show.EndTime = show.StartTime.Add(Convert.ToDateTime(tds[5].InnerText).TimeOfDay);
show.Channel = c.ToString();
shows.Add(show);
}
}
return shows;
}
private static List<Exhibitor> GetExtendedData(List<Exhibitor> exhibitors)
{
foreach(var exhibitor in exhibitors)
{
string url = baseUrl + exhibitor.DetailUrl;
string html = WebHelper.HttpGet(url);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
exhibitor.Overview = HtmlDocHelper.GetInnerText(doc, "//*[@id=\"page_content_ctl00_ctl00_divDescription\"]");
exhibitor.Tags = HtmlDocHelper.GetInnerText(doc, "//*[@id=\"page_content_ctl00_ctl00_ulDetailTags\"]");
exhibitor.Email = HtmlDocHelper.GetInnerText(doc, "//*[@id=\"page_content_ctl00_ctl00_hlEmail\"]");
exhibitor.Phone = HtmlDocHelper.GetInnerText(doc, "//*[@id=\"page_content_ctl00_ctl00_divTel\"]").Replace("Tel:", "").Trim();
exhibitor.Fax = HtmlDocHelper.GetInnerText(doc, "//*[@id=\"page_content_ctl00_ctl00_divFax\"]").Replace("Fax:", "").Trim();
exhibitor.ImageUrl = HtmlDocHelper.GetAttribute(doc.DocumentNode, "//*[@id=\"page_content_ctl00_ctl00_imgExhibitorDetail\"]", "src");
var nodes = doc.DocumentNode.SelectNodes("//ul[@id=\"page_content_ctl00_ctl00_ulDetailAddress\"]/li");
if(nodes != null)
{
List<string> addressParts = new List<string>();
foreach(var node in nodes)
{
addressParts.Add(node.InnerText);
}
exhibitor.Address = string.Join(", ", addressParts);
}
}
return exhibitors;
}
public IPttCaptcha CaptchaImage(IPttRequest request, string htmlSource)
{
var htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(htmlSource);
var iframe = htmlDoc.DocumentNode.SelectSingleNode("//iframe");
if (iframe == null)
{
return null;
}
var iframeSrc = iframe.Attributes["src"].Value;
var newRequest = new PttRequest(iframeSrc) {WrappedRequest = {ContentType = "text/html", Method = "GET"}};
IPttResponse response = new PttResponse();
var captchaFormHtml = response.GetResponse(newRequest);
htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(captchaFormHtml);
var captchaImg = htmlDoc.DocumentNode.SelectSingleNode("//img");
if (captchaImg == null)
{
return null;
}
var captchaSrc = string.Format("http://{0}/recaptcha/api/{1}", googleHost, captchaImg.Attributes["src"].Value);
return new PttCaptcha() { Url = captchaSrc, RequestUsed = newRequest, FormUrl = iframeSrc};
}
public XmlNodeList GetNCTSongs(string html)
{
XmlDocument songDoc = new XmlDocument();
try
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(html);
//http://www.nhaccuatui.com/flash/xml?html5=true&key2=469177aa57117104d2e3d1ea0da2b3bc
string pattern = @"http:\/\/www\.nhaccuatui.com\/flash\/xml\?(html5=true&)?key(2|1)=.*";
Match match = Regex.Match(html, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline);
string xmlURL = "";
if (match.Success)
{
xmlURL = match.Value;
}
int objIndex = xmlURL.IndexOf("\"");
xmlURL = xmlURL.Substring(0, objIndex);
AlbumName = htmlDoc.DocumentNode.SelectSingleNode("//h1[@itemprop='name']").InnerText;
objHTML.URL = xmlURL;
string songXML = objHTML.GetHTMLString();
songDoc.LoadXml(songXML);
}
catch (Exception ex)
{
throw ex;
}
return songDoc.SelectNodes("//track");
}
public static void FetchRates(int bankId, string bankName)
{
var bankExchange = new BankExchange();
bankExchange.BankId = bankId;
if (bankName.ToLowerInvariant() == "bank asya")
{
var ds = new DataSet("fxPrices");
ds.ReadXml("http://www.bankasya.com.tr/xml/kur_list.xml");
bankExchange.USDBuying = Int32.Parse(ds.Tables[1].Rows[0]["Kur"].ToString().Replace(".", ""));
bankExchange.USDSelling = Int32.Parse(ds.Tables[1].Rows[1]["Kur"].ToString().Replace(".", ""));
bankExchange.EURBuying = Int32.Parse(ds.Tables[1].Rows[2]["Kur"].ToString().Replace(".", ""));
bankExchange.EURSelling = Int32.Parse(ds.Tables[1].Rows[3]["Kur"].ToString().Replace(".", ""));
bankExchange.Save();
}
else if(bankName.ToLowerInvariant() == "finansbank")
{
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load("http://www.finansbank.com.tr/bankacilik/alternatif-dagitim-kanallari/internet-bankaciligi/doviz_kurlari.aspx?IntSbMO_FB_Mevduatoranlari_PU".DownloadPage());
}
else
{
throw new Exception("Banka adı bunlardan biri olabilir: Bank Asya");
}
}
public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
{
// 例えば ConverterParameter=true とすると、parameter に string "true" が入ります。
if (value == null)
return "";
// HtmlAgilityPack を使用してタグ付きの string からタグを除去します。
var hap = new HtmlAgilityPack.HtmlDocument();
hap.LoadHtml(value.ToString());
var doc = hap.DocumentNode.InnerText;
doc = doc.Replace(@" ", " ").Replace(@"<", "<").Replace(@">", ">").Replace(@"&", "&").Replace(@""", "\"");
var multiCRRegex = new Regex(@"\n\n\n+");
doc = multiCRRegex.Replace(doc, "\n\n");
// parameter には数字が入りますので、数値があればその文字数でカットした string を返します。
int strLength;
if (int.TryParse(parameter as string, out strLength)) {
if (doc.Length > strLength)
{
return doc.Substring(0, strLength) + "…";
}
return doc;
}
else
{
return doc;
}
}
private async Task LoadContent(DataItem _item)
{
string contentStr = string.Empty;
try
{
contentStr = await Define.DownloadStringAsync(_item.Link);
}
catch (Exception ex)
{
//throw;
}
if (contentStr != string.Empty)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(contentStr);
HtmlAgilityPack.HtmlNode htmlNode = htmlDoc.GetElementbyId("ContentContainer");
while (htmlNode.Descendants("script").Count() > 0)
{
htmlNode.Descendants("script").ElementAt(0).Remove();
}
while (htmlNode.Descendants("meta").Count() > 0)
{
htmlNode.Descendants("meta").ElementAt(0).Remove();
}
//contentStr = "<p><i>This blog post was authored by Andrew Byrne (<a href=\"http://twitter.com/AndrewJByrne\" target=\"_blank\">@AndrewJByrne</a>), a Senior Content Developer on the Windows Phone Developer Content team.</i> <p><i></i> <p><i>- Adam</i></p> <hr> <p> <table cellspacing=\"1\" cellpadding=\"2\" width=\"722\" border=\"0\"> <tbody> <tr> <td valign=\"top\" width=\"397\"> <p>The Windows Phone Developer Content team is continually adding new code samples that you can download from MSDN. In this post, we introduce you to the 10 latest samples that we’ve posted on MSDN. Each sample includes a readme file that walks you through building and running the sample, and includes links to relevant, related documentation. We hope these samples help you with your app development and we look forward to providing more samples as we move forward. You can check out all of our <a href=\"http://code.msdn.microsoft.com/wpapps/site/search?f%5B0%5D.Type=Contributors&f%5B0%5D.Value=Windows%20Phone%20SDK%20Team&f%5B0%5D.Text=Windows%20Phone%20SDK&sortBy=Date\" target=\"_blank\">samples on MSDN</a>.</p></td> <td valign=\"top\" width=\"320\"> <p><a href=\"http://blogs.windows.com/cfs-file.ashx/__key/communityserver-blogs-components-weblogfiles/00-00-00-53-84-metablogapi/clip_5F00_image002_5F00_1765A66A.png\"><img title=\"clip_image002\" style=\"border-top: 0px; border-right: 0px; background-image: none; border-bottom: 0px; float: none; padding-top: 0px; padding-left: 0px; margin-left: auto; border-left: 0px; display: block; padding-right: 0px; margin-right: auto\" border=\"0\" alt=\"clip_image002\" src=\"http://blogs.windows.com/cfs-file.ashx/__key/communityserver-blogs-components-weblogfiles/00-00-00-53-84-metablogapi/clip_5F00_image002_5F00_thumb_5F00_7B083E7C.png\" width=\"121\" height=\"213\"></a></p></td></tr></tbody></table></p> <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306704\" target=\"_blank\">Stored Contacts Sample</a></h3> <p>This sample illustrates how to use the <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.phone.personalinformation.contactstore(v=vs.105).aspx\" target=\"_blank\">ContactStore</a> class and related APIs to create a contact store for your app. This feature is useful if your app uses an existing cloud-based contact store. You can use the APIs you to create contacts on the phone that represent the contacts in your remote store. You can display and modify the contacts in the People Hub on your phone, just like contacts that are added through the built-in experience. You can use the APIs to update and delete contacts you have created on the phone and also to query for any changes the user has made to the contacts locally so you can sync those changes to your remote store. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306701\" target=\"_blank\">Basic Storage Recipes</a></h3> <p>This is a “Windows Runtime Storage 101” sample for Windows Phone developers moving from isolated storage and <b>System.IO</b> to <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.storage.aspx\" target=\"_blank\">Windows.Storage</a> and <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.storage.streams.aspx\" target=\"_blank\">Windows.Storage.Streams</a>. The sample demonstrates how to write to and read files, in addition to how to enumerate directory trees. It also demonstrates how to pass data from one page to the next, and how to persist application state when the app is deactivated. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=301509\" target=\"_blank\">Trial Experience Sample</a></h3> <p>This sample shows you how to design your app to detect its license state when the app launches, and how to detect changes to the license state while running. It comes with a helper class that you can use in your app to wrap <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.applicationmodel.store.licenseinformation.aspx\" target=\"_blank\">LicenseInformation</a> functionality. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=302059\" target=\"_blank\">Windows Runtime Component Sample</a></h3> <p>This sample demonstrates the basics of creating a Windows Phone Runtime component in C++ and consuming it in a XAML app. The sample demonstrates three scenarios: the first scenario illustrates how to call synchronous and asynchronous methods to perform a computation. The second scenario uses the same computation component to demonstrate progress reporting and cancellation of long-running tasks. Finally, the third scenario shows how to use a component to wrap logic that uses <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/jj206944(v=vs.105).aspx\" target=\"_blank\">XAudio2 APIs</a> to play a sound. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306097\" target=\"_blank\">Company Hub Sample</a></h3> <p>This sample demonstrates the construction of an app that is capable of deploying line-of-business (LOB) apps to employees of a corporation. The sample uses an example XML file to define the company XAPs that are available to employees for secure download, and shows you how to dynamically access that file at run time. Then it shows you how to install company apps, enumerate the apps, and then launch the installed company apps. This app is just an example framework and requires additional work beyond the sample to be functional. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306702\" target=\"_blank\">Image Recipes</a></h3> <p>This sample illustrates how to use images in your app efficiently, while giving your users a great experience. It tackles downsampling images, implementing pinch and zoom, and downloading images with a progress display and an option to cancel the download. We’ve taken a recipe approach: each recipe is delivered in a self-contained page in the app so you can focus your attention on the particular part of the sample you are most interested in. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306026\" target=\"_blank\">Azure Voice Notes</a></h3> <p>This sample uses Windows Phone speech recognition APIs and Windows Azure Mobile Services to record voice notes as text and store the notes in the cloud. It shows how Mobile Services can be used to authenticate a user with their Microsoft Account. It also demonstrates how to use Mobile Services to store, retrieve, and delete data from an Azure database table. The app generates text from speech using the Windows Phone speech recognition APIs and the phone’s predefined dictation grammar. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=299241\" target=\"_blank\">Kid's Corner Sample</a></h3> <p>This sample illustrates how to use the <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.phone.applicationmodel.applicationprofile.modes(v=vs.105).aspx\" target=\"_blank\">ApplicationProfile.Modes</a> property to recognize Kid’s Corner mode. When the app runs, it checks the <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.phone.applicationmodel.applicationprofile.modes(v=vs.105).aspx\" target=\"_blank\">ApplicationProfile.Modes</a> property. If the value is <b>ApplicationProfileModes.Alternate</b>, you’ll know that the app is running in Kid’s Corner mode. Depending on the content of your app, you may want to change its appearance or features when it runs in Kid’s Corner mode. Some features that you should consider disabling when running in Kid’s Corner mode include in-app purchases, launching the web browser, and the ad control. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=267468\" target=\"_blank\">URI Association Sample</a></h3> <p>Use this sample to learn how to automatically launch your app via URI association. This sample includes three apps: a URI launcher, and two apps that handle the URI schemes that are built in to the launcher app. You can launch an app that is included with the sample or edit the URI and launch a different app. There is also a button for launching a URI on another phone using Near Field Communication (NFC). <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=275007\" target=\"_blank\">Speech for Windows Phone: Speech recognition using a custom grammar</a></h3> <p>A grammar defines the words and phrases that an app will recognize in speech input. This sample shows you how to go beyond the basics to create a powerful grammar (based on the grammar schema) with which your app can recognize commands and phrases constructed in different ways. <p> <p>This post is just a glimpse of the latest Windows Phone samples we’ve added to the MSDN code gallery. From launching apps with URI associations, to dictating notes and storing them in the cloud, we hope that there’s something for everyone. We’ll be sure to keep you posted as new samples are added to the collection, so stay tuned. In the meantime, grab the samples, experiment with them, and use the code to light up your apps. You can download all Windows Phone samples at <a href=\"http://code.msdn.microsoft.com/wpapps\" target=\"_blank\">http://code.msdn.microsoft.com/wpapps</a>. <div style=\"clear:both;\"></div><img src=\"http://blogs.windows.com/aggbug.aspx?PostID=588575&AppID=5384&AppType=Weblog&ContentType=0\" width=\"1\" height=\"1\">";
Items[Items.IndexOf(_item)].Content = htmlNode.InnerHtml.Replace("\r", "").Replace("\n", "");
}
}
internal static string ExtractFilmTitleUsingXPath(string html)
{
//TODO: Currently the film's Original Title is used as the default Title. If no Original Title is available, we should get the film's default title.
var doc = new HtmlDocument();
doc.LoadHtml(html);
HtmlAgilityPack.HtmlNode htmlNode
= doc.DocumentNode.SelectSingleNode
(@"//span[@class='title-extra']");
string originalTitle = htmlNode.InnerText;
////TODO: The xml data should be loaded and received from memory instead of a file.
originalTitle = RegExDataMiners.MatchRegexExpressionReturnFirstMatchFirstGroup
(originalTitle, "\"(?<Title>.*?)\"");
//MessageBox.Show(originalTitle);
return (originalTitle);
}
public CookieContainer DoLogin()
{
const string formUrl = "http://dbunet.dbu.dk";
var req = (HttpWebRequest)WebRequest.Create(formUrl);
req.Method = "GET";
var resp = req.GetResponse() as HttpWebResponse;
var loginPage = new HtmlDocument();
loginPage.Load(resp.GetResponseStream());
var viewstate = loginPage.DocumentNode.SelectSingleNode("//input[@name='__VIEWSTATE']").GetAttributeValue("value","---");
string proxy = null;
var formParams = string.Format("?__VIEWSTATE={0}&_ctl2:sys_txtUsername={1}&_ctl2:sys_txtPassword={2}&_ctl2:cbRememberMe=true&_ctl2:ibtnSignin=Continue&_ctl2:ibtnSignin.x=33&_ctl2:ibtnSignin.y=3&TopMenu_ClientState=", viewstate, UserName, Password);
req = (HttpWebRequest)WebRequest.Create(formUrl);
req.ContentType = "application/x-www-form-urlencoded";
req.Method = "POST";
req.KeepAlive = true;
req.AllowAutoRedirect = false;
req.Proxy = new WebProxy(proxy, true); // ignore for local addresses
req.CookieContainer = new CookieContainer(); // enable cookies
byte[] bytes = Encoding.ASCII.GetBytes(formParams);
req.ContentLength = bytes.Length;
using (var os = req.GetRequestStream())
{
os.Write(bytes, 0, bytes.Length);
}
resp = (HttpWebResponse)req.GetResponse();
return req.CookieContainer;
}
public static void FillDbFromSite()
{
// Add Currency
banks.Сurrency.Add(new Сurrency() { Name = "USD" });
banks.Сurrency.Add(new Сurrency() { Name = "EUR" });
banks.Сurrency.Add(new Сurrency() { Name = "RUB" });
banks.Сurrency.Add(new Сurrency() { Name = "EUR -> USD" });
//Add Banks
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
doc.LoadHtml(wc.DownloadString("http://myfin.by/currency/minsk"));
foreach (HtmlAgilityPack.HtmlNode tablet in doc.DocumentNode.SelectNodes("//*[@id='workarea']/div[3]/div[3]/div/table/tbody/tr[1]"))
{
foreach (HtmlAgilityPack.HtmlNode row in tablet.SelectNodes("td[1]"))
{
banks.Bank.Add(new Bank() { Name = row.InnerText });
}
}
//banks.Department.Add(new Department() { });
//banks.DeprtmentsСurrencies.Add(new Department());
banks.SaveChanges();
}
internal static string[] MatchXpathExpressionReturnAllMatches
(string html, string xPathExpression)
{
var doc = new HtmlDocument();
doc.LoadHtml(html);
HtmlAgilityPack.HtmlNodeCollection htmlNodesCollection
= doc.DocumentNode.SelectNodes
(xPathExpression);
var matches = new List<string>();
foreach (var htmlNode in htmlNodesCollection)
{
matches.Add(htmlNode.InnerText);
}
return matches.ToArray();
}
private static void GetChannelList(string webHtml)
{
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(webHtml);
var aList = doc.DocumentNode.Descendants("a")
.Where(u =>
(!string.IsNullOrEmpty(u.GetAttributeValue("href", "")) &&
u.GetAttributeValue("class", "").Equals("btn btn-block btn-primary"))
|| (string.IsNullOrEmpty(u.GetAttributeValue("href", "")) && string.IsNullOrEmpty(u.GetAttributeValue("class", "")))
).ToArray();
var length = aList.Count(u => string.IsNullOrEmpty(u.GetAttributeValue("href", "")));
var clModels = new ChannelListJsonModel[length];
var i = -1;
foreach (var a in aList)
{
if (string.IsNullOrEmpty(a.GetAttributeValue("href", "")))
{
clModels[++i] = new ChannelListJsonModel();
clModels[i].ChannelList = new List<ChannelModel>();
clModels[i].ChannelBelong = a.InnerText;
continue;
}
if (i < 0) continue;
clModels[i].ChannelList.Add(
new ChannelModel
{
ChannelUrl = a.GetAttributeValue("href", ""),
ChannelName = a.InnerText
});
}
Channels = new ObservableCollection<ChannelListJsonModel>(clModels);
}
public All_Grade()
{
InitializeComponent();
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); //euc-kr을 지원하지 않으므로 인코딩 메소드 생성
var endcoingCode = 51949; //euc-kr 코드
System.Text.Encoding euckr = System.Text.Encoding.GetEncoding(endcoingCode); //euc-kr 인코딩 작성
WebClient wc = new WebClient();
//wc.Headers.Add(HttpRequestHeader.Cookie, CookieTemp.Cookie); //cookieTemp에서 Cookie가져옴
string url = "https://hems.hoseo.or.kr/hoseo/stu/mem/inf/hom/STUHOM0110L0.jsp";
byte[] docBytes = wc.DownloadData(url);
string encodeType = wc.ResponseHeaders["Content-Type"];
string charsetKey = "charset";
int pos = encodeType.IndexOf(charsetKey);
Encoding currentEncoding = Encoding.Default;
if (pos != -1)
{
pos = encodeType.IndexOf("=", pos + charsetKey.Length);
if (pos != -1)
{
string charset = encodeType.Substring(pos + 1);
currentEncoding = Encoding.GetEncoding(charset);
}
}
string result = currentEncoding.GetString(docBytes); // 대상 웹 서버가 인코딩한 설정으로 디코딩!
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(result);
string subject1 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[4]/td[1]").InnerHtml;
string title1 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[4]/td[3]/a/font").InnerHtml;
string subject2 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[6]/td[1]").InnerHtml;
string title2 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[6]/td[3]/a/font").InnerHtml;
string subject3 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[8]/td[1]").InnerHtml;
string title3 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[8]/td[3]/a/font").InnerHtml;
string subject4 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[10]/td[1]").InnerHtml;
string title4 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[10]/td[3]/a/font").InnerHtml;
string subject5 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[12]/td[1]").InnerHtml;
string title5 = doc.DocumentNode.SelectSingleNode("//*[@id='tab1']/tbody/tr[12]/td[3]/a/font").InnerHtml;
AG_Items1 = new ObservableCollection <string>
{
subject1,
subject2,
subject3,
subject4,
subject5
};
AG_Items2 = new ObservableCollection <string>
{
title1,
title2,
title3,
title4,
title5
};
subject.ItemsSource = AG_Items1;
title.ItemsSource = AG_Items2;
}
private async Task WriteStockStockF10_FHPG_ToDb(string stockId, Stream stream)
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
using (var scope = _serviceScopeFactory.CreateScope())
{
var scopedServices = scope.ServiceProvider;
var db = scopedServices.GetRequiredService <BlogContext>();
var page = reader.ReadToEnd();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(page);
//写入到数据库中
//如果存在就更新，不存在就插入
var query = from i in db.SharingSet
where i.StockId == stockId
select i;
var collection = await query.ToListAsync();
var table = doc.DocumentNode.SelectSingleNode("//table[@class='table_bg001 border_box limit_sale']");
foreach (HtmlAgilityPack.HtmlNode row in table.SelectNodes("tr"))
{
///This is the row.
var cells = row.SelectNodes("th|td");
string dateString = cells[0].InnerText.Trim();
DateTime tempDate;
if (DateTime.TryParse(dateString, out tempDate))
{
//有可能存在暂无数据的情况
///This the cell.
Sharing sharing = collection.FirstOrDefault(s => s.DateGongGao == tempDate);
if (sharing == null)
{
//不存在就新增，存在就更新
sharing = new Sharing();
sharing.StockId = stockId;
sharing.DateGongGao = tempDate;
db.SharingSet.Add(sharing);
}
float temp = 0;
if (float.TryParse(cells[2].InnerText.Trim(), out temp))
{
//
sharing.SongGu = temp;
}//else use default value which be set in contructor
if (float.TryParse(cells[3].InnerText.Trim(), out temp))
{
//
sharing.ZhuanZeng = temp;
}//else use default value which be set in contructo
if (float.TryParse(cells[4].InnerText.Trim(), out temp))
{
//
sharing.PaiXi = temp;
}//else use default value which be set in contructo
dateString = cells[5].InnerText.Trim();
if (DateTime.TryParse(dateString, out tempDate))
{
sharing.DateDengJi = tempDate;//日期可能为空
}
dateString = cells[6].InnerText.Trim();
if (DateTime.TryParse(dateString, out tempDate))
{
sharing.DateChuXi = tempDate;//日期可能为空
}
}
}
await db.SaveChangesAsync();
}
}
}
private async Task WriteStockNumInfo_ToDb(string stockId, Stream stream)
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
using (var scope = _serviceScopeFactory.CreateScope())
{
var scopedServices = scope.ServiceProvider;
var db = scopedServices.GetRequiredService <BlogContext>();
var page = reader.ReadToEnd();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(page);
//写入到数据库中
//如果存在就更新，不存在就插入
var query = from i in db.StockNumSet
where i.StockId == stockId
select i;
var collection = await query.ToListAsync();
var table = doc.DocumentNode.SelectSingleNode("//table/tr[1]/th[1][contains(text(),'股份构成(万股)')]/ancestor::table");
if (table == null)
{
throw new Exception("东方财富数据解析错误!" + stockId);
}
var rows = table.SelectNodes("tr");
///This is the row.
var row0cells = rows[0].SelectNodes("th|td");
var row1cells = rows[1].SelectNodes("th|td");
var row2cells = rows[2].SelectNodes("th|td");
if (row1cells[0].InnerText.Trim() != "股份总数")
{
throw new Exception("东方财富数据解析错误(股份总数的行不正确)!" + stockId);
}
if (row2cells[0].InnerText.Trim() != "已上市流通A股")
{
throw new Exception("东方财富数据解析错误(已上市流通A股)!" + stockId);
}
for (int i = 1; i < row0cells.Count; i++)
{
string dateString = row0cells[i].InnerText.Trim();
DateTime tempDate;
if (DateTime.TryParse(dateString, out tempDate))
{
//有可能存在暂无数据的情况
///This the cell.
StockNum item = collection.FirstOrDefault(s => s.Date == tempDate);
if (item == null)
{
//不存在就新增，存在就更新
item = new StockNum();
item.StockId = stockId;
item.Date = tempDate;
db.StockNumSet.Add(item);
}
double temp = 0;
if (double.TryParse(row1cells[i].InnerText.Trim(), out temp))
{
item.All = temp;
}
else
{
throw new Exception("东方财富数据解析错误(总股数)!" + stockId);
}
if (double.TryParse(row2cells[i].InnerText.Trim(), out temp))
{
item.LiuTongA = temp;
}
else
{
item.LiuTongA = 0;
// throw new Exception("东方财富数据解析错误(已上市流通A股)!"+stockId);
}
}
}
await db.SaveChangesAsync();
}
}
}
public whoistest()
{
doc = new HtmlAgilityPack.HtmlDocument();;
doc.LoadHtml(File.ReadAllText(@"pages/Yalantis.html"));
}
public string RenderHtmlWithTemplate(string template, EntityRecord record = null, ErpRequestContext requestContext = null, ErpAppContext appContext = null)
{
var foundTags = Regex.Matches(template, @"(?<=\{\{)[^}]*(?=\}\})").Cast <Match>().Select(match => match.Value).Distinct().ToList();
foreach (var tag in foundTags)
{
var processedTag = tag.Replace("{{", "").Replace("}}", "").Trim();
var defaultValue = "";
if (processedTag.Contains("??"))
{
//this is a tag with a default value
int questionMarksLocation = processedTag.IndexOf("??");
var tagValue = processedTag.Substring(0, questionMarksLocation).Trim();
var tagDefault = processedTag.Substring(questionMarksLocation + 2).Trim().Replace("\"", "").Replace("'", "");
processedTag = tagValue;
defaultValue = tagDefault;
}
if (processedTag.StartsWith("Record[") && record != null)
{
//{{Record["fieldName"]}}
var fieldName = processedTag.Replace("Record[\"", "").Replace("\"]", "").ToLowerInvariant();
if (record.Properties.ContainsKey(fieldName) && record[fieldName] != null)
{
template = template.Replace("{{" + tag + "}}", record[fieldName].ToString());
}
else
{
template = template.Replace("{{" + tag + "}}", defaultValue);
}
}
else if (processedTag.StartsWith("ErpRequestContext.") && requestContext != null)
{
var propertyPath = processedTag.Replace("ErpRequestContext.", "");
var propertyValue = requestContext.GetPropValue(propertyPath);
if (propertyValue != null)
{
template = template.Replace("{{" + tag + "}}", propertyValue.ToString());
}
else
{
template = template.Replace("{{" + tag + "}}", defaultValue);
}
}
else if (processedTag.StartsWith("ErpAppContext.") && requestContext != null)
{
var propertyPath = processedTag.Replace("ErpAppContext.", "");
var propertyValue = requestContext.GetPropValue(propertyPath);
if (propertyValue != null)
{
template = template.Replace("{{" + tag + "}}", propertyValue.ToString());
}
else
{
template = template.Replace("{{" + tag + "}}", defaultValue);
}
}
else if (processedTag.StartsWith("ListMeta.") && requestContext != null)
{
var propertyPath = processedTag.Replace("ListMeta.", "");
var propertyValue = requestContext.GetPropValue(propertyPath);
if (propertyValue != null)
{
template = template.Replace("{{" + tag + "}}", propertyValue.ToString());
}
else
{
template = template.Replace("{{" + tag + "}}", defaultValue);
}
}
else if (processedTag.StartsWith("ViewMeta.") && requestContext != null)
{
var propertyPath = processedTag.Replace("ViewMeta.", "");
var propertyValue = requestContext.GetPropValue(propertyPath);
if (propertyValue != null)
{
template = template.Replace("{{" + tag + "}}", propertyValue.ToString());
}
else
{
template = template.Replace("{{" + tag + "}}", defaultValue);
}
}
else if (processedTag.StartsWith("erp-allow-roles") && processedTag.Contains("="))
{
//Check if "erp-authorize" is present
var authoriseTag = foundTags.FirstOrDefault(x => x.Contains("erp-authorize"));
if (authoriseTag != null)
{
var tagArray = processedTag.Split('=');
var rolesCsv = tagArray[1].Replace("\"", "").Replace("\"", "").ToLowerInvariant();
var rolesList = rolesCsv.Split(',').ToList();
var currentUser = SecurityContext.CurrentUser;
if (currentUser == null && !rolesList.Any(x => x == "guest"))
{
return("");
}
var authorized = false;
foreach (var askedRole in rolesList)
{
if (currentUser.Roles.Any(x => x.Name == askedRole))
{
authorized = true;
break;
}
}
if (!authorized)
{
return("");
}
}
}
else if (processedTag.StartsWith("erp-block-roles") && processedTag.Contains("="))
{
//Check if "erp-authorize" is present
var authoriseTag = foundTags.FirstOrDefault(x => x.Contains("erp-authorize"));
if (authoriseTag != null)
{
var tagArray = processedTag.Split('=');
var rolesCsv = tagArray[1].Replace("\"", "").Replace("\"", "").ToLowerInvariant();
var rolesList = rolesCsv.Split(',').ToList();
var currentUser = SecurityContext.CurrentUser;
if (currentUser == null && rolesList.Any(x => x == "guest"))
{
return("");
}
var authorized = true;
foreach (var askedRole in rolesList)
{
if (currentUser.Roles.Any(x => x.Name == askedRole))
{
authorized = false;
break;
}
}
if (!authorized)
{
return("");
}
}
}
else if (processedTag == "CurrentUrlEncoded" && requestContext != null && requestContext.PageContext != null &&
requestContext.PageContext.HttpContext != null && requestContext.PageContext.HttpContext.Request != null)
{
var currentUrl = requestContext.PageContext.HttpContext.Request.Path + requestContext.PageContext.HttpContext.Request.QueryString;
var propertyValue = HttpUtility.UrlEncode(currentUrl);
if (!String.IsNullOrWhiteSpace(propertyValue))
{
template = template.Replace("{{" + tag + "}}", propertyValue);
}
else
{
template = template.Replace("{{" + tag + "}}", defaultValue);
}
}
}
//Tags that depend on others to be already applied
foreach (var tag in foundTags)
{
var processedTag = tag.Replace("{{", "").Replace("}}", "").Trim();
if (processedTag == "erp-active-page-equals")
{
//erp-active-page-equals
var tagComplete = "{{" + processedTag + "}}";
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(template);
var links = htmlDocument.DocumentNode.SelectNodes("//a[@href]");
foreach (var link in links)
{
var hrefString = link.GetAttributeValue("href", "");
var erpTag = link.Attributes.FirstOrDefault(x => x.Name == tagComplete);
var emptyTag = link.Attributes.FirstOrDefault(x => x.Name == "dasdasd");
if (erpTag != null)
{
if (!String.IsNullOrWhiteSpace(hrefString) && requestContext != null && requestContext.PageContext != null)
{
string currentUrl = requestContext.PageContext.HttpContext.Request.Path.ToString().ToLowerInvariant();
if (currentUrl == hrefString)
{
var classes = link.GetClasses().ToList();
if (classes.FirstOrDefault(x => x == "active") == null)
{
classes.Add("active");
link.SetAttributeValue("class", String.Join(" ", classes));
}
}
}
link.Attributes.Remove(tagComplete);
}
}
template = htmlDocument.DocumentNode.InnerHtml;
}
else if (processedTag == "erp-active-page-starts")
{
//erp-active-page-equals
var tagComplete = "{{" + processedTag + "}}";
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(template);
var links = htmlDocument.DocumentNode.SelectNodes("//a[@href]");
foreach (var link in links)
{
var hrefString = link.GetAttributeValue("href", "");
var erpTag = link.Attributes.FirstOrDefault(x => x.Name == tagComplete);
var emptyTag = link.Attributes.FirstOrDefault(x => x.Name == "dasdasd");
if (erpTag != null)
{
if (!String.IsNullOrWhiteSpace(hrefString) && requestContext != null && requestContext.PageContext != null)
{
string currentUrl = requestContext.PageContext.HttpContext.Request.Path.ToString().ToLowerInvariant();
if (currentUrl.StartsWith(hrefString))
{
var classes = link.GetClasses().ToList();
if (classes.FirstOrDefault(x => x == "active") == null)
{
classes.Add("active");
link.SetAttributeValue("class", String.Join(" ", classes));
}
}
}
link.Attributes.Remove(tagComplete);
}
}
template = htmlDocument.DocumentNode.InnerHtml;
}
}
return(template);
}
public void BenchHtmlAgility()
{
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
}
private List <CrossReferenceNotResolvedException> TransformHtmlCore(IDocumentBuildContext context, string sourceFilePath, string destFilePath, HtmlAgilityPack.HtmlDocument html)
{
var xrefLinkNodes = html.DocumentNode.SelectNodes("//a[starts-with(@href, 'xref:')]");
if (xrefLinkNodes != null)
{
foreach (var xref in xrefLinkNodes)
{
TransformXrefLink(xref, context);
}
}
var xrefExceptions = new List <CrossReferenceNotResolvedException>();
var xrefNodes = html.DocumentNode.SelectNodes("//xref")?
.Where(s => !string.IsNullOrEmpty(s.GetAttributeValue("href", null)) || !string.IsNullOrEmpty(s.GetAttributeValue("uid", null))).ToList();
if (xrefNodes != null)
{
foreach (var xref in xrefNodes)
{
try
{
UpdateXref(xref, context, Constants.DefaultLanguage);
}
catch (CrossReferenceNotResolvedException e)
{
xrefExceptions.Add(e);
}
}
}
var srcNodes = html.DocumentNode.SelectNodes("//*/@src");
if (srcNodes != null)
{
foreach (var link in srcNodes)
{
UpdateHref(link, "src", context, sourceFilePath, destFilePath);
}
}
var hrefNodes = html.DocumentNode.SelectNodes("//*/@href");
if (hrefNodes != null)
{
foreach (var link in hrefNodes)
{
UpdateHref(link, "href", context, sourceFilePath, destFilePath);
}
}
return(xrefExceptions);
}
public void LoadDeepSearch(object state)
{
if (LocalDatabase.problemList == null)
{
return;
}
_InDeepSearch = true;
//clear data
_deepSearchRes.Clear();
_DeepSearchProgress = 0;
//set data
this.BeginInvoke((MethodInvoker) delegate
{
allProbButton.Checked = false;
markedButton.Checked = false;
plistLabel.Text = "Deep Search Result";
problemListView.AdditionalFilter = null;
searchBox1.search_text.ReadOnly = true;
cancelDeepSearchButton.Visible = true;
problemViewSplitContainer.Panel1.Enabled = false;
_SetObjects(_deepSearchRes);
problemListView.Sort(priorityProb, SortOrder.Descending);
});
//search string
string src = (string)state;
if (!src.StartsWith("*"))
{
src = "\\b" + src;
}
if (!src.EndsWith("*"))
{
src += "\\b";
}
var regex = new System.Text.RegularExpressions.Regex(src,
System.Text.RegularExpressions.RegexOptions.Compiled |
System.Text.RegularExpressions.RegexOptions.IgnoreCase);
//search
foreach (ProblemInfo prob in LocalDatabase.problemList)
{
if (_CancelSearch)
{
break;
}
prob.Priority = 0;
//search in problem data
prob.Priority += regex.Matches(prob.ToString()).Count;
//search in problem description
try
{
string file = LocalDirectory.GetProblemHtml(prob.pnum);
if (LocalDirectory.GetFileSize(file) > 100)
{
var hdoc = new HtmlAgilityPack.HtmlDocument();
hdoc.Load(file);
string dat = hdoc.DocumentNode.Element("body").InnerText;
//string dat = System.IO.File.ReadAllText(file);
prob.Priority += regex.Matches(dat).Count;
}
}
catch { }
//add to list
if (prob.Priority > 0)
{
_deepSearchRes.Add(prob);
}
_DeepSearchProgress++;
}
//complete
_InDeepSearch = false;
this.BeginInvoke((MethodInvoker) delegate
{
_SetObjects(_deepSearchRes);
searchBox1.search_text.ReadOnly = false;
cancelDeepSearchButton.Visible = false;
problemViewSplitContainer.Panel1.Enabled = true;
if (_CancelSearch) //if canceled
{
ClearSearch();
}
});
}
static string[] get_UrlHtml(string url, string htm)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(htm);
string[] auri = url.Split('/');
string uri_root = string.Join("/", auri.Where((x, k) => k < 3).ToArray());
string uri_path1 = string.Join("/", auri.Where((x, k) => k < auri.Length - 2).ToArray());
string uri_path2 = string.Join("/", auri.Where((x, k) => k < auri.Length - 3).ToArray());
var lsURLs = doc.DocumentNode
.SelectNodes("//a")
.Where(p => p.InnerText != null && p.InnerText.Trim().Length > 0)
.Select(p => p.GetAttributeValue("href", string.Empty))
.Select(x => x.IndexOf("../../") == 0 ? uri_path2 + x.Substring(5) : x)
.Select(x => x.IndexOf("../") == 0 ? uri_path1 + x.Substring(2) : x)
.Where(x => x.Length > 1 && x[0] != '#')
.Select(x => x[0] == '/' ? uri_root + x : (x[0] != 'h' ? uri_root + "/" + x : x))
.Select(x => x.Split('#')[0])
.ToList();
//string[] a = htm.Split(new string[] { "http" }, StringSplitOptions.None).Where((x, k) => k != 0).Select(x => "http" + x.Split(new char[] { '"', '\'' })[0]).ToArray();
//lsURLs.AddRange(a);
//????????????????????????????????????????????????????????????????????????????????
uri_root = "https://dictionary.cambridge.org/grammar/british-grammar/";
var u_html = lsURLs
.Where(x => x.IndexOf(uri_root) == 0)
.GroupBy(x => x)
.Select(x => x.First())
//.Where(x =>
// !x.EndsWith(".pdf")
// || !x.EndsWith(".txt")
// || !x.EndsWith(".ogg")
// || !x.EndsWith(".mp3")
// || !x.EndsWith(".m4a")
// || !x.EndsWith(".gif")
// || !x.EndsWith(".png")
// || !x.EndsWith(".jpg")
// || !x.EndsWith(".jpeg")
// || !x.EndsWith(".doc")
// || !x.EndsWith(".docx")
// || !x.EndsWith(".ppt")
// || !x.EndsWith(".pptx")
// || !x.EndsWith(".xls")
// || !x.EndsWith(".xlsx"))
.Distinct()
.ToArray();
//if (!string.IsNullOrEmpty(setting_URL_CONTIANS))
// foreach (string key in setting_URL_CONTIANS.Split('|'))
// u_html = u_html.Where(x => x.Contains(key)).ToArray();
//var u_audio = lsURLs.Where(x => x.EndsWith(".mp3")).Distinct().ToArray();
//var u_img = lsURLs.Where(x => x.EndsWith(".gif") || x.EndsWith(".jpeg") || x.EndsWith(".jpg") || x.EndsWith(".png")).Distinct().ToArray();
//var u_youtube = lsURLs.Where(x => x.Contains("youtube.com/")).Distinct().ToArray();
return(u_html);
}
private void btn_check_Click(object sender, EventArgs e)
{
if (cancellationToken != null && !cancellationToken.IsCancellationRequested)
{
cancellationToken.Cancel();
return;
}
cancellationToken = new CancellationTokenSource();
cancellationToken.Token.Register(() =>
{
label7.Text = "查询取消";
btn_check.Text = "查询";
});
label7.Text = string.Empty;
var lines = tbx_word.Lines;
var words = new List <Models.SearchResult>();
foreach (var item in lines)
{
if (string.IsNullOrEmpty(item) || string.IsNullOrWhiteSpace(item))
{
continue;
}
var sp = item.Split(new[] { '\t' });
if (sp.Length > 3)
{
words.Add(new Models.SearchResult()
{
SearchNodes = new List <Models.SearchNode>(),
Host = sp[0].Trim(),
Word = sp[1].Trim(),
Device = sp[2].ToLower()
});
}
}
if (words.Count < 1)
{
return;
}
words = words.OrderByDescending(m => m.Host).ToList();
progressBar1.Maximum = words.Count;
progressBar1.Value = 0;
progressBar1.Step = 1;
progressBar2.Maximum = words.Count;
progressBar2.Value = 0;
progressBar2.Step = 1;
var maxpage = comboBox1.SelectedIndex;
btn_check.Text = "取消查询";
tbx_result.Clear();
var maxTask = cbx_qps.SelectedIndex + 1;
Task.Run(() =>
{
Parallel.ForEach(words, new ParallelOptions()
{
MaxDegreeOfParallelism = maxTask, CancellationToken = cancellationToken.Token
}, (item, loopstate) =>
{
try
{
if (cancellationToken.IsCancellationRequested)
{
loopstate.Stop();
}
var client = new RestClient("https://www.baidu.com");
client.CookieContainer = new System.Net.CookieContainer();
client.UserAgent = $"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{new Random().Next(70, 80)}.0.{new Random().Next(3000, 4000)}.{new Random().Next(100, 300)} Safari/537.36";
client.FollowRedirects = false;
// client.CookieContainer.SetCookies(new Uri("https://baidu.com"), "kleck=37e9bf91aa6b140e37384f2eea05cfe7; max-age=86400; domain=.baidu.com; path=/");
var status = "";
var request = new RestRequest();
var htmldoc = new HtmlAgilityPack.HtmlDocument();
if (item.Device == "pc")
{
#region pc
request.Resource = "/";
var resp = client.Get(request);
request.Resource = $"s?ie=utf-8&wd={System.Web.HttpUtility.UrlEncode(item.Word)}";
resp = client.Get(request);
for (int i = 0; i < 2; i++)
{
if (!resp.IsSuccessful)
{
Task.Delay(2000);
resp = client.Get(request);
}
else
{
break;
}
}
progressBar1.Invoke(new MethodInvoker(() =>
{
// toolTip1.Show(status, label7);
label7.Text = "预热首页：" + item.Word;
}));
if (resp.IsSuccessful)
{
status = resp.StatusCode.ToString();
htmldoc.LoadHtml(resp.Content);
var nodes = htmldoc.DocumentNode.SelectNodes("//div[@id='content_left']/div/h3//a");
var nextpage = htmldoc.DocumentNode.SelectSingleNode("//*[@id='page']//a[@class='n']");
if (nodes == null)
{
return;
}
nodes.ToList().ForEach(m =>
{
item.SearchNodes.Add(new Models.SearchNode
{
LinkUrl = m.GetAttributeValue("href", ""),
Rank = m.SelectSingleNode(m.XPath + "/../..").GetAttributeValue("id", "0"),
Title = m.InnerText
});
});
#region 剩余页码
for (int i = 0; i < maxpage; i++)
{
if (nextpage != null)
{
request.Resource = nextpage.GetAttributeValue("href", "");
resp = client.Get(request);
if (resp.IsSuccessful)
{
progressBar1.Invoke(new MethodInvoker(() =>
{
// toolTip1.Show(status, label7);
label7.Text = $"预热{i + 2}页：" + item.Word;
}));
htmldoc.LoadHtml(resp.Content);
nextpage = htmldoc.DocumentNode.SelectSingleNode("//*[@id='page']//a[@class='n'][2]");
nodes = htmldoc.DocumentNode.SelectNodes("//div[@id='content_left']/div/h3//a");
if (nodes != null)
{
nodes.ToList().ForEach(m =>
{
item.SearchNodes.Add(new Models.SearchNode
{
LinkUrl = m.GetAttributeValue("href", ""),
Rank = m.SelectSingleNode(m.XPath + "/../..").GetAttributeValue("id", "0"),
Title = m.InnerText
});
});
}
}
else
{
}
// Console.WriteLine(Task.CurrentId + "-" + i);
}
}
#endregion
}
#endregion
}
else
{
#region mobile
client.UserAgent = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1";
request.Resource = "/";
var resp = client.Get(request);
request.Resource = $"s?&word={System.Web.HttpUtility.UrlEncode(item.Word)}&ts=8330407&t_kt=0&ie=utf-8";
resp = client.Get(request);
if (resp.StatusCode == System.Net.HttpStatusCode.Found)
{
var l = resp.Headers.FirstOrDefault(f => f.Name == "Location");
if (l != null)
{
request.Resource = l.Value.ToString();
resp = client.Get(request);
}
}
if (resp.IsSuccessful)
{
status = resp.StatusCode.ToString();
htmldoc.LoadHtml(resp.Content);
const string nodexpath = "//div[@class='results']/div[@order][@data-log][@tpl='www_normal']";
const string nextxpath = "//a[@class='new-nextpage-only' or @class='new-nextpage']";
var nodes = htmldoc.DocumentNode.SelectNodes(nodexpath);
var nextpage = htmldoc.DocumentNode.SelectSingleNode(nextxpath);
nodes.ToList().ForEach(m =>
{
var tn = m.SelectSingleNode(m.XPath + "//h3");
if (tn == null)
{
return;
}
var mu = Regex.Match(m.GetAttributeValue("data-log", "{'mu':'about:blank'}").Replace("'mu':''", "'mu':'about:blank'"), "(?<='mu':').+(?=')").Value;
if (string.IsNullOrEmpty(mu))
{
mu = "about:blank";
}
var node = new Models.SearchNode
{
Rank = m.GetAttributeValue("order", "0"),
Url = new Uri(mu), //string.IsNullOrEmpty(mulnk) ? "about:blank" : mulnk,
Title = tn != null ? tn.InnerText : ""
};
item.SearchNodes.Add(node);
});
for (int i = 0; i < maxpage; i++)
{
if (nextpage == null)
{
break;
}
request.Resource = System.Web.HttpUtility.HtmlDecode(nextpage.GetAttributeValue("href", "").Replace("https://www.baidu.com/", ""));
resp = client.Get(request);
htmldoc.LoadHtml(resp.Content);
nodes = htmldoc.DocumentNode.SelectNodes(nodexpath);
nextpage = htmldoc.DocumentNode.SelectSingleNode(nextxpath);
if (nodes != null)
{
var has = false;
nodes.ToList().ForEach(m =>
{
var tn = m.SelectSingleNode(m.XPath + "//h3");
if (tn == null)
{
return;
}
var node = new Models.SearchNode
{
Rank = ((i + 1) * 10 + m.GetAttributeValue("order", 0)).ToString(),
Url = new Uri(Regex.Match(m.GetAttributeValue("data-log", "{'mu':'about:blank'}").Replace("'mu':''", "'mu':'about:blank'"), "(?<='mu':').+(?=')").Value), //string.IsNullOrEmpty(mulnk) ? "about:blank" : mulnk,
Title = tn != null ? tn.InnerText : ""
};
item.SearchNodes.Add(node);
if (node.Url.Host.Contains(item.Host))
{
has = true;
}
});
if (has)
{
break;
}
}
}
}
#endregion
}
progressBar1.Invoke(new MethodInvoker(() =>
{
// toolTip1.Show(status, label7);
label7.Text = "预热完成：" + item.Word;
progressBar1.PerformStep();
}));
// });
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
});;
if (cancellationToken.IsCancellationRequested)
{
return;
}
var client2 = new RestClient("http://www.baidu.com");
client2.CookieContainer = new System.Net.CookieContainer();
client2.UserAgent = $"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{new Random().Next(70, 80)}.0.{new Random().Next(3000, 4000)}.{new Random().Next(100, 200)} Safari/537.36";
client2.FollowRedirects = false;
int index = 0;
words.ForEach(item =>
{
progressBar2.BeginInvoke(new MethodInvoker(() =>
{
label7.Text = "查询进度：" + item.Word.ToString();
progressBar2.PerformStep();
}));
if (item.SearchNodes != null)
{
if (item.Device == "pc")
{
try
{
Parallel.ForEach(item.SearchNodes, new ParallelOptions()
{
MaxDegreeOfParallelism = maxTask, CancellationToken = cancellationToken.Token
}, (m, loopstate) =>
{
if (cancellationToken.IsCancellationRequested)
{
loopstate.Stop();
}
var request2 = new RestRequest();
request2.Resource = m.LinkUrl.Replace("http://www.baidu.com/", "");
var resp2 = client2.Head(request2);
if (resp2.StatusCode == System.Net.HttpStatusCode.Found)
{
var l = resp2.Headers.FirstOrDefault(f => f.Name == "Location");
if (l != null && l.Value.ToString().StartsWith("http"))
{
m.Url = new Uri(l.Value.ToString());
if (m.Url.Host.StartsWith(item.Host.Replace("www.", "")))
{
loopstate.Stop();
}
}
}
});
//item.SearchNodes.AsParallel().ForAll((m) =>
//{
//});
}
catch (Exception ex)
{
}
// Task.WaitAll(tasks.ToArray());
}
else
{
}
}
if (!item.Rank.Contains("+") && item.Rank.Length < 2)
{
var rank = int.Parse(item.Rank);
if (rank > 0 && rank < 11)
{
index += 1;
}
}
tbx_result.Invoke(new MethodInvoker(() =>
{
tbx_result.AppendText(item.ToString() + "\r\n");
var lvi = new ListViewItem(listView1.Items.Count.ToString());
lvi.SubItems.Add(item.Host);
lvi.SubItems.Add(item.Word);
lvi.SubItems.Add(item.Device);
lvi.SubItems.Add(item.Rank);
lvi.UseItemStyleForSubItems = false;
listView1.Items.Add(lvi);
}));
});
listView1.EndUpdate();
var file = File.AppendText($"wd-{DateTime.Now.ToString("yyyyMMdd")}.txt");
file.WriteLine($"=========={DateTime.Now.ToString("yyyy-MM-dd HH:mm")}==========");
file.WriteLine(tbx_result.Text);
file.Flush();
file.Close();
btn_check.BeginInvoke(new MethodInvoker(() =>
{
cancellationToken.Cancel();
label7.Text = "查询完成：首页数量 " + index;
btn_check.Text = "查询";
}));
}, cancellationToken.Token);
}
private void button1_Click(object sender, EventArgs e)
{
HtmlAgilityPack.HtmlDocument htmldocu = new HtmlAgilityPack.HtmlDocument();
try
{
WebClient myWebClient = new WebClient();
myWebClient.Encoding = Encoding.UTF8;
this.textBox1.Text = myWebClient.DownloadString(this.txPagina.Text);
htmldocu.LoadHtml(this.textBox1.Text);
}
catch (Exception ex)
{
guardarLog("Error al obtener el datos:\n" + ex.Message);
}
try
{
if (System.IO.File.Exists(this.txArchivoResultado.Text))
{
System.IO.File.Delete(this.txArchivoResultado.Text);
System.IO.StreamWriter stream = System.IO.File.CreateText(this.txArchivoResultado.Text);
stream.Close();
}
//dolar
if (this.txXpath.Text != "")
{
string resultado = htmldocu.DocumentNode.SelectSingleNode(this.txXpath.Text).InnerHtml;
//si tiene datos que validar
if (this.txXpathValor.Text != "")
{
string resultado_valida = htmldocu.DocumentNode.SelectSingleNode(this.txXpathVal.Text).InnerHtml;
guardarLog("Compara el valor:" + this.txXpathValor.Text + " contra el resultado: " + resultado_valida);
if (resultado_valida == this.txXpathValor.Text)
{
guardarEnArchivo(resultado);
}
}
else
{
guardarEnArchivo(resultado);
}
}
//euro
if (this.txXpath2.Text != "")
{
string resultado = htmldocu.DocumentNode.SelectSingleNode(this.txXpath2.Text).InnerHtml;
//si tiene datos que validar
if (this.txXpathValor2.Text != "")
{
string resultado_valida = htmldocu.DocumentNode.SelectSingleNode(this.txXpathVal2.Text).InnerHtml;
guardarLog("Compara el valor:" + this.txXpathValor2.Text + " contra el resultado: " + resultado_valida);
if (resultado_valida == this.txXpathValor2.Text)
{
guardarEnArchivo(resultado);
}
}
else
{
guardarEnArchivo(resultado);
}
}
}
catch (Exception ex)
{
guardarLog("Error al guardar el datos:\n" + ex.Message);
}
}
public static ctrl_TableList add_Jint_Stats_Row(this ctrl_TableList tableList, int index, string url, string html, HtmlAgilityPack.HtmlDocument htmlDocument, List <KeyValuePair <string, string> > javaScripts)
{
var links = htmlDocument.select("//a");
var images = htmlDocument.select("//img");
var forms = htmlDocument.select("//form");
var allJavaScriptsCompileOk = javaScripts.allJavaScriptsCompileOk();
var allJavaScriptCompiledAst = javaScripts.jint_Compile();
var sizeOfJointJavaScripts = javaScripts.getStringWithJointJavascripts().size();
//var scripts = htmlDocument.select("//script");
tableList.add_Row(
"{0:00}".format(index), //(mapped++).str(),
url,
"{0:000}".format(links.size()),
"{0:000}".format(images.size()),
"{0:000}".format(forms.size()),
"{0:000}".format(javaScripts.size()),
allJavaScriptsCompileOk.str(),
"{0:0000}".format(allJavaScriptCompiledAst.functions().size()),
"{0:0000}".format(allJavaScriptCompiledAst.values().size()),
sizeOfJointJavaScripts.kBytesStr(),
javaScripts.totalValueSize().kBytesStr(),
html.size().kBytesStr()
);
tableList.lastRow().textColor((allJavaScriptsCompileOk) ? Color.DarkGreen : Color.DarkRed);
return(tableList);
}
public void ProcessMailQueue(int instance = 1)
{
//This method will be called 3 times(one with ScheduleJob as 'SendQueueMail' & other as 'ParseMailChimpResponse' & other as 'SendTextQueue') in every 6 min
//update the job schedule
Logger.Current.Informational("Which instance is currently running: " + instance);
var requests = ((EfUnitOfWork)unitOfWork).ExecuteProcessQueue(30, instance: instance);
Logger.Current.Informational("No of requests received for sending mails is : " + requests.Count);
var mailRequests = ConvertToSendMailRequest(requests);
var mailResponses = new ConcurrentBag <SendMailResponse>();
var mailService = new MailService();
while (mailRequests != null && mailRequests.Count > 0)
{
Logger.Current.Verbose("Processing mail requests paralley");
foreach (var mailRequest in mailRequests)
{
if (mailRequest.CategoryID == (byte)EmailNotificationsCategory.ContactSendEmail || mailRequest.CategoryID == (byte)EmailNotificationsCategory.WorkflowSendEmail)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(mailRequest.Body);
List <string> notlinks = new List <string>()
{
"#", "not found", "javascript: void(0)", ""
};
var hrefList = doc.DocumentNode.SelectNodes("//a") != null?doc.DocumentNode.SelectNodes("//a").Select(p => p.GetAttributeValue("href", "not found")).ToList() : new List <string>();
var finalLinks = hrefList.Where(s => !notlinks.Contains(s)).ToList();
if (finalLinks.IsAny())
{
//inserting links to DB
GetInsertedLinkIndexes(finalLinks, mailRequest.SentMailDetailID);
foreach (var link in finalLinks.Select((value, index) => new { Value = value, Index = index }))
{
doc.DocumentNode.SelectNodes("//a").Each(attr =>
{
string dummyLink = "*|LINK" + link.Index.ToString() + "|*";
if (attr.GetAttributeValue("href", "not found") == link.Value)
{
attr.SetAttributeValue("href", dummyLink);
}
});
}
}
//For 1*1 Pixel Tracking;
string pixelTrackingContent = "<div><img src='*|PTX|*' width='1' height='1' alt='ptx'/></div>";
HtmlAgilityPack.HtmlNode newNode = HtmlAgilityPack.HtmlNode.CreateNode(pixelTrackingContent);
doc.DocumentNode.PrependChild(newNode);
string mailBody = doc.DocumentNode.InnerHtml;
mailRequest.Body = mailBody;
//get merge field contents
mailRequest.MergeValues = ((EfUnitOfWork)unitOfWork).GetMergeFieldValues(mailRequest.SentMailDetailID);
}
var to = mailRequest.To.IsAny() ? mailRequest.To.First().Split(',').ToList() : new List <string>();
mailRequest.To = to;
var cc = mailRequest.CC.IsAny() ? mailRequest.CC.First().Split(',').ToList() : new List <string>();
mailRequest.CC = cc;
var bcc = mailRequest.BCC.IsAny() ? mailRequest.BCC.First().Split(',').ToList() : new List <string>();
mailRequest.BCC = bcc;
SendMailResponse response = mailService.RawSendMail(mailRequest);
mailResponses.Add(response);
mailService.LogScheduledMessageResponse(mailRequest, response);
}
Logger.Current.Verbose("Mail requests processed parallely. Requests and responses are fed to Mail StoredProcedure");
requests = ((EfUnitOfWork)unitOfWork).ExecuteProcessQueue(30, requests, mailResponses.ToList(), instance);
mailRequests = ConvertToSendMailRequest(requests);
}
}
public ActionResult CompareTrips()
{
CrawelCustomModel CrawelCustomModel_Obj = new CrawelCustomModel();
CrawelCustomModel_Obj.Packages = PackagesDB_Obj.GetAll();
HtmlAgilityPack.HtmlWeb web = new HtmlAgilityPack.HtmlWeb();
HtmlAgilityPack.HtmlDocument doc = web.Load("https://www.saiyah.com.pk/tour/");
// Counter is leye liya hai k title zeyada or numb of days kam a rai thay to jitne NumOfDay ai usko counter mai dal kr us ki base pr title fetch hon
//or yehi chez pe view k lop mai bhi use hui hai k k hmri web k packges and dsuir web k packges same hon or view distrub na ho
int Counter = 0;
//FETCHING NUMBER OF DAYS
foreach (var item in doc.DocumentNode.SelectNodes(" //*[@class='tourmaster-tour-info tourmaster-tour-info-duration-text '] "))
{
try
{
string NumOfDay = item.InnerText;
NumOfDay = Regex.Replace(NumOfDay, "&", "&");
CrawelCustomModel_Obj.NumOfDayList.Add(new NumOfDay()
{
NumOfDays = NumOfDay
});
Counter = CrawelCustomModel_Obj.NumOfDayList.Count();
}
catch (Exception e)
{
TempData["CrawelError"] = "There is error in fetching data from the other website" + e.Message;
}
}
//FETCHING TITLE
for (int i = 0; i < Counter; i++)
{
try
{
var item = doc.DocumentNode.SelectNodes("//*[@class='tourmaster-tour-title gdlr-core-skin-title']")[i];
string name = item.InnerText;
name = Regex.Replace(name, "&", "&");
name = Regex.Replace(name, "–", "&");
CrawelCustomModel_Obj.NameList.Add(new Name()
{
names = name
});
}
catch (Exception e)
{
TempData["CrawelError"] = "There is error in fetching data from the other website" + e.Message;
}
}
//FETCHING Months
for (int i = 0; i < Counter; i++)
{
try
{
var item = doc.DocumentNode.SelectNodes("//*[@class='tourmaster-tour-info tourmaster-tour-info-availability '] ")[i];
string Months = item.InnerText;
Months = Regex.Replace(Months, "Availability :", "");
CrawelCustomModel_Obj.MonthsList.Add(new Month()
{
Months = Months
});
}
catch (Exception e)
{
TempData["CrawelError"] = "There is error in fetching data from the other website" + e.Message;
}
}
//FETCHING Random Images form folder
for (int i = 0; i < Counter; i++)
{
string file = null;
string path = @"C:\Users\Ali\Desktop\desktop data\visual studio\TourAgencyProject\UI(user interface )\Imges_Crawel";
if (!string.IsNullOrEmpty(path))
{
var extensions = new string[] { ".png", ".jpg", ".gif" };
try
{
var di = new DirectoryInfo(path);
var rgFiles = di.GetFiles("*.*").Where(f => extensions.Contains(f.Extension.ToLower()));
Random R = new Random();
file = rgFiles.ElementAt(R.Next(0, rgFiles.Count())).FullName;
file = file.Replace(@"C:\Users\Ali\Desktop\desktop data\visual studio\TourAgencyProject\UI(user interface )", "");
CrawelCustomModel_Obj.ImagesList.Add(new ImageModel()
{
Images = file
});
}
catch (Exception) { }
}
}
ViewBag.Counter = Counter;
return(View(CrawelCustomModel_Obj));
}
static void RunScraper()
{
Console.WriteLine("[{0}] Extracting url's from each page....", DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.ff"));
for (int n = 0; n < finishPage; n++)
{
if (genderSelected == "m")
{
url = @"https://chaturbate.com/male-cams?page=" + n.ToString();
}
if (genderSelected == "f")
{
url = @"https://chaturbate.com/female-cams?page=" + n.ToString();
}
if (genderSelected == "t")
{
url = @"https://chaturbate.com/trans-cams?page=" + n.ToString();
}
if (genderSelected == "c")
{
url = @"https://chaturbate.com/couple-cams?page=" + n.ToString();
}
HtmlAgilityPack.HtmlWeb hw = new HtmlAgilityPack.HtmlWeb();
HtmlAgilityPack.HtmlDocument doc = hw.Load(url);
List <string> htmls = new List <string>();
foreach (HtmlAgilityPack.HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
{
string hrefValue = link.GetAttributeValue("href", string.Empty);
if (hrefValue.Contains("/") && hrefValue.Contains("attachment"))
{
htmls.Add(hrefValue);
}
if (
!hrefValue.Contains("/tag/") &&
!hrefValue.Equals("/") &&
!hrefValue.Equals("#") &&
!hrefValue.Contains("/accounts/register/") &&
!hrefValue.Contains("/tube/") &&
!hrefValue.Contains("https") &&
!hrefValue.Contains("/tipping/free_tokens/") &&
!hrefValue.Contains("/auth/login/") &&
!hrefValue.Contains("/female-cams/") &&
!hrefValue.Contains("/male-cams/") &&
!hrefValue.Contains("/trans-cams/") &&
!hrefValue.Contains("/supporter/upgrade/") &&
!hrefValue.Contains("/tags/") &&
!hrefValue.Contains("/auth/password_reset/") &&
!hrefValue.Contains("/couple-cams/") &&
!hrefValue.Contains("/?page=") &&
!hrefValue.Contains("http://") &&
!hrefValue.Contains("/teen-cams/") &&
!hrefValue.Contains("/18to21-cams/") &&
!hrefValue.Contains("/20to30-cams/") &&
!hrefValue.Contains("/30to50-cams/") &&
!hrefValue.Contains("/mature-cams/") &&
!hrefValue.Contains("/north-american-cams/") &&
!hrefValue.Contains("/other-region-cams/") &&
!hrefValue.Contains("/euro-russian-cams/") &&
!hrefValue.Contains("/asian-cams/") &&
!hrefValue.Contains("/south-american-cams/") &&
!hrefValue.Contains("/exhibitionist-cams/") &&
!hrefValue.Contains("/hd-cams/") &&
!hrefValue.Contains("/spy-on-cams/") &&
!hrefValue.Contains("/new-cams/") &&
!hrefValue.Contains("/6-tokens-per-minute-private-cams/female/") &&
!hrefValue.Contains("/12-tokens-per-minute-private-cams/female/") &&
!hrefValue.Contains("/18-tokens-per-minute-private-cams/female/") &&
!hrefValue.Contains("/30-tokens-per-minute-private-cams/female/") &&
!hrefValue.Contains("/60-tokens-per-minute-private-cams/female/") &&
!hrefValue.Contains("/90-tokens-per-minute-private-cams/female/") &&
!hrefValue.Contains("/terms/") &&
!hrefValue.Contains("/privacy/") &&
!hrefValue.Contains("/security/") &&
!hrefValue.Contains("/law_enforcement/") &&
!hrefValue.Contains("/billingsupport/") &&
!hrefValue.Contains("/security/privacy/deactivate/") &&
!hrefValue.Contains("/apps/") &&
!hrefValue.Contains("/contest/details/") &&
!hrefValue.Contains("/affiliates/") &&
!hrefValue.Contains("/affiliates/") &&
!hrefValue.Contains("/sitemap/")
)
{
if (genderSelected == "m")
{
File.AppendAllText("males.txt", Environment.NewLine + hrefValue);
}
if (genderSelected == "f")
{
File.AppendAllText("females.txt", Environment.NewLine + hrefValue);
}
if (genderSelected == "t")
{
File.AppendAllText("trans.txt", Environment.NewLine + hrefValue);
}
if (genderSelected == "c")
{
File.AppendAllText("couples.txt", Environment.NewLine + hrefValue);
}
}
}
}
// remove and duplicate lines
Console.WriteLine("[{0}] Removing duplicate lines...", DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.ff"));
if (genderSelected == "m")
{
string[] lines = File.ReadAllLines("males.txt");
File.WriteAllLines("males.txt", lines.Distinct().ToArray());
}
if (genderSelected == "f")
{
string[] lines = File.ReadAllLines("females.txt");
File.WriteAllLines("females.txt", lines.Distinct().ToArray());
}
if (genderSelected == "t")
{
string[] lines = File.ReadAllLines("trans.txt");
File.WriteAllLines("trans.txt", lines.Distinct().ToArray());
}
if (genderSelected == "c")
{
string[] lines = File.ReadAllLines("couples.txt");
File.WriteAllLines("couples.txt", lines.Distinct().ToArray());
}
// start searching for emails on all profiles added to the list
Console.WriteLine("[{0}] Searching each page for emails...", DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.ff"));
if (genderSelected == "m")
{
liness = File.ReadAllLines("males.txt");
}
if (genderSelected == "f")
{
liness = File.ReadAllLines("female.txt");
}
if (genderSelected == "t")
{
liness = File.ReadAllLines("trans.txt");
}
if (genderSelected == "c")
{
liness = File.ReadAllLines("couples.txt");
}
foreach (var line in liness)
{
String url = @"http://chaturbate.com" + line;
WebClient client = new WebClient();
string page = client.DownloadString(url);
Regex emailRegex = new Regex(@"\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*", RegexOptions.IgnoreCase);
//find items that matches with our pattern
MatchCollection emailMatches = emailRegex.Matches(page);
StringBuilder sb = new StringBuilder();
foreach (Match emailMatch in emailMatches)
{
if (!emailMatch.Value.Contains("highwebmedia.com") && !emailMatch.Value.Contains("chaturbate.com"))
{
sb.AppendLine(emailMatch.Value);
Console.WriteLine("[{0}] Email Found! - {1} @ {2}", DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss.ff"), emailMatch.Value, line);
}
}
//store to file
if (genderSelected == "m")
{
File.AppendAllText("male-emails.txt", sb.ToString());
}
if (genderSelected == "f")
{
File.AppendAllText("female-emails.txt", sb.ToString());
}
if (genderSelected == "t")
{
File.AppendAllText("trans-emails.txt", sb.ToString());
}
if (genderSelected == "c")
{
File.AppendAllText("couple-emails.txt", sb.ToString());
}
}
}
private void startBtn_Click(object sender, EventArgs e)
{
string id = idTextBox.Text;
string password = passwordTextBox.Text;
if (!serverListLoad)
{
MessageBox.Show("HTML file did not loaded", "HTML file ERROR");
return;
}
else if (!banListLoad)
{
MessageBox.Show("JSON file did not loaded", "JSON file ERROR");
return;
}
if (id.Length == 0 || password.Length == 0)
{
MessageBox.Show("Please enter your ID or password.");
return;
}
int count = 0;
int max = banBox.Items.Count * serverList.Items.Count;
Thread work = new Thread(() => {
for (int i = 0; i < serverList.Items.Count; i++)
{
string url = serverList.Items[i].ToString();
serverLabel.Text = url;
string token = "";
CookieContainer cookies = new CookieContainer();
//get session and certificate cookies
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url + "/ServerAdmin/");
req.Credentials = CredentialCache.DefaultCredentials;
req.CookieContainer = cookies;
req.KeepAlive = true;
HttpWebResponse result = (HttpWebResponse)req.GetResponse();
var head = result.Headers;
foreach (Cookie cook in result.Cookies)
{
cookies.Add(cook);
}
if (result.StatusCode != HttpStatusCode.OK)
{
MessageBox.Show("LOGIN FAIL");
return;
}
using (Stream dataStream = result.GetResponseStream())
{
StreamReader reader = new StreamReader(dataStream);
string rawStr = reader.ReadToEnd();
var myDoc = new HtmlAgilityPack.HtmlDocument();
myDoc.LoadHtml(rawStr);
//get HTML login token
var nodes = myDoc.DocumentNode.SelectNodes("//input[@name='token']");
token = nodes[0].Attributes["value"].Value;
}
//sending login post
String postData = String.Format("token={0}&username={1}&password={2}&remember=-1", token, id, password);
cookies = PostSend(postData, url + "/ServerAdmin/", cookies);
foreach (string user in banBox.Items)
{
count++;
progressBar.Value = (int)((double)count / max * 100);
//sending ban post
userLable.Text = user;
postData = String.Format("action=add&steamint64={0}uniqueid=", user);
cookies = PostSend(postData, url + "/ServerAdmin/policy/bans", cookies);
}
}
MessageBox.Show("Complete Ban");
});
work.Start();
}
public DeHtml()
{
xd = new HtmlAgilityPack.HtmlDocument();
}
public static bool ParseGoogleLoginPage(string strEmail, string strPassword, string strHTMLLoginPage, string strReferer, out string strData, out string strPostUri)
{
// Expected strData BEGIN
// ======================
// GALX=K2kN1B3RhzI
// &continue=https%3A%2F%2Faccounts.google.com%2Fo%2Foauth2%2Fauth%3Fscope%3Demail%26response_type%3Dcode%26redirect_uri%3Dhttps%3A%2F%2Fteechip.com%2Fgoogle%26client_id%3D759311199253-liule4h0ij4trokni0clghutesmfrpaq.apps.googleusercontent.com%26hl%3Dvi%26from_login%3D1%26as%3D6c13d32c4c484e0a
// &service=lso
// <mpl=popup
// &shdf=Cm4LEhF0aGlyZFBhcnR5TG9nb1VybBoADAsSFXRoaXJkUGFydHlEaXNwbGF5TmFtZRoHVGVlQ2hpcAwLEgZkb21haW4aB1RlZUNoaXAMCxIVdGhpcmRQYXJ0eURpc3BsYXlUeXBlGgdERUZBVUxUDBIDbHNvIhR9f65sYBLeiTyA3ksWmSKfvjjy-CgBMhR69Mfqff5tofmPaXiU63kbTQdAzw
// &scc=1
// &sarp=1
// &_utf8=%E2%98%83
// &bgresponse=%21qapCVouLleiO7EZE_x4-8E88lh4CAAAAMFIAAAAFKgEfCMGz3oizuMFi9s1elOqqg8JcOP5CRdkNPXH--weicJaEAyf-pqsUcGZzFbdUIPWD6hVYEdNFEz35CvkIrsyHVpQRGe5ksfhAee9ehS3pR6bzwXyNFX7Pap-J7SQaN1swC53RMscX-GgiAB7Bd4pINn3QSjFPup0K0JDhJsMqBdkFQikNK4TCndA3NFztVcnkdr91yQuMEwxpfG6CjcHt3ckSyopOpBsFuHRcRqeEorlpvItyd9ITIvZdWd6LNZJgkDdAVbI2TdP3bPWBv9Lu5ux73a-RC4Oc7ow5uAWWbcMQ2D_4hKd4xT66Ie6pV8U4JxgvxhegMPaylafpK6NrqFNbEtEHjTksqILJKLRInylFLU37UpdQuwqPtUWz240
// &pstMsg=1
// &dnConn=
// &checkConnection=youtube%3A3988%3A1
// &checkedDomains=youtube
// &Email=<strEmail>
// &Passwd=<strPassword>
// &signIn=Sign+in
// &PersistentCookie=yes
// ======================
// Expected strData END
strData = "";
strPostUri = "";
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(strHTMLLoginPage);
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode frmNode = htmlDoc.DocumentNode.SelectSingleNode("//form[@id=\"gaia_loginform\"]");
if (frmNode != null)
{
strPostUri = frmNode.GetAttributeValue("action", "");
BuildDataString(frmNode, "Page", ref strData, true);
BuildDataString(frmNode, "GALX", ref strData, string.IsNullOrEmpty(strData));
BuildDataString(frmNode, "gxf", ref strData, string.IsNullOrEmpty(strData));
BuildDataString(GetValueByName(frmNode, "input", "continue"), "continue", ref strData);
BuildDataString(GetValueByName(frmNode, "input", "followup"), "followup", ref strData);
BuildDataString(frmNode, "service", ref strData);
BuildDataString(frmNode, "ProfileInformation", ref strData);
BuildDataString(GetValueByName(frmNode, "input", "_utf8"), "_utf8", ref strData);
BuildDataString(frmNode, "bgresponse", ref strData);
BuildDataString(frmNode, "pstMsg", ref strData);
BuildDataString(frmNode, "dnConn", ref strData);
BuildDataString(frmNode, "checkConnection", ref strData);
BuildDataString(frmNode, "checkedDomains", ref strData);
BuildDataString(frmNode, "identifiertoken", ref strData);
BuildDataString(frmNode, "identifiertoken_audio", ref strData);
BuildDataString(strEmail, "Email", ref strData);
BuildDataString(strPassword, "Passwd", ref strData);
BuildDataString(frmNode, "rmShown", ref strData);
}
}
return(false);;
}
void document_MouseDown(object sender, System.Windows.Forms.HtmlElementEventArgs e)
{
System.Windows.Forms.HtmlElement element = this.automation_browser.Document.GetElementFromPoint(e.ClientMousePosition);
//System.Windows.Forms.HtmlElement xPath_element = this.automation_browser.Document.GetElementFromPoint(e.ClientMousePosition);
//******************* for testing XPATH *****************
var savedId = element.Id;
var uniqueId = Guid.NewGuid().ToString();
element.Id = uniqueId;
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(element.Document.GetElementsByTagName("html")[0].OuterHtml);
element.Id = savedId;
var node = doc.GetElementbyId(uniqueId);
string xpath = node.XPath;
string[] parentNodes = xpath.Split('/');
parentCounting = 0;
System.Windows.Forms.HtmlElement starting_element = Finding_Parent_With_Identifier(element);
string starting_element_idName = starting_element.GetAttribute("id");
string startingPattern = @"//*[@";
string middlePattern = "=" + '"'; //"="";
string endingPattern = '"' + "]"; //""]";
string XPATH = startingPattern + "id" + middlePattern + starting_element_idName + endingPattern;
for (int i = parentNodes.Length - parentCounting + 1; i < parentNodes.Length; i++)
{
string tmp = '/' + parentNodes[i];
XPATH += tmp;
}
//******************* for testing XPATH *****************
if (e.MouseButtonsPressed == System.Windows.Forms.MouseButtons.Right)
{
string element_tagName = element.TagName.ToString();
//string innerHtml=element.TagName.
//string element_outerHTML = element2.OuterHtml;
//MessageBox.Show("inner html= " + element.InnerHtml);
//MessageBox.Show("inner text= "+ element.InnerText);
//MessageBox.Show("class name= "+element.GetAttribute("class").Trim());
//MessageBox.Show(element_tagName);
//MessageBox.Show("id= "+element.GetAttribute("id"));
//MessageBox.Show("name= "+element.GetAttribute("name"));
//MessageBox.Show("inner text= " + element.InnerText);
//MessageBox.Show("inner html= " + element.InnerHtml);
selenium_id = element.GetAttribute("id").Trim();
System.Windows.Forms.HtmlElement aelement = automation_browser.Document.GetElementById(selenium_id);
string element_type = element.GetAttribute("type").Trim();
//MessageBox.Show(element.InnerHtml);
//MessageBox.Show(element.OuterHtml);
// MessageBox.Show(selectedIndex.ToString());
//MessageBox.Show(element_tagName);
if (element_type.ToLower() == "radio")
{
selenium_id = element.GetAttribute("id").Trim();
if (!input_id_check_str.Contains(selenium_id.ToLower()))
{
string msgtext = "Select this Radio Button?";
string txt = Application.Current.MainWindow.Title;
MessageBoxButton button = MessageBoxButton.YesNoCancel;
MessageBoxResult result = MessageBox.Show(msgtext, txt, button);
switch (result)
{
case MessageBoxResult.Yes:
input_id_check_str.Add(selenium_id.ToLower());
selenium_name = element.GetAttribute("name").Trim();
//object objElement = element.DomElement;
//object objSelectedIndex = objElement.GetType().InvokeMember("selectedIndex",
//BindingFlags.GetProperty, null, objElement, null);
//int selectedIndex = (int)objSelectedIndex;
// MessageBox.Show("in yes indes= "+selectedIndex);
AutomationBrowser_HtmlElementInfo.Add(new HtmlElementInfo()
{
Html_element_id = selenium_id,
Html_element_name = selenium_name,
Html_element_XPATH = XPATH
});
//(this.Parent as MetroWindow).DialogResult = true;
//(this.Parent as MetroWindow).Close();
break;
case MessageBoxResult.No:
break;
case MessageBoxResult.Cancel:
break;
}
}
else
{
//MessageBox.Show("You have already entered value in thid field.");
}
}
else
{
MessageBox.Show("Please, Focus only Radio Button");
}
}
}
public async Task LoadBadgesAsync()
{
// Settings.Default.myProfileURL = http://steamcommunity.com/id/USER
var profileLink = Settings.Default.myProfileURL + "/badges";
var pages = new List <string>()
{
"?p=1"
};
var document = new HtmlDocument();
int pagesCount = 1;
try
{
// Load Page 1 and check how many pages there are
var pageURL = string.Format("{0}/?p={1}", profileLink, 1);
var response = await CookieClient.GetHttpAsync(pageURL);
// Response should be empty. User should be unauthorised.
if (string.IsNullOrEmpty(response))
{
RetryCount++;
if (RetryCount == 18)
{
ResetClientStatus();
return;
}
throw new Exception("");
}
document.LoadHtml(response);
// If user is authenticated, check page count. If user is not authenticated, pages are different.
var pageNodes = document.DocumentNode.SelectNodes("//a[@class=\"pagelink\"]");
if (pageNodes != null)
{
pages.AddRange(pageNodes.Select(p => p.Attributes["href"].Value).Distinct());
pages = pages.Distinct().ToList();
}
string lastpage = pages.Last().ToString().Replace("?p=", "");
pagesCount = Convert.ToInt32(lastpage);
// Get all badges from current page
ProcessBadgesOnPage(document);
// Load other pages
for (var i = 2; i <= pagesCount; i++)
{
lblDrops.Text = string.Format(localization.strings.reading_badge_page + " {0}/{1}, " + localization.strings.please_wait, i, pagesCount);
// Load Page 2+
pageURL = string.Format("{0}/?p={1}", profileLink, i);
response = await CookieClient.GetHttpAsync(pageURL);
// Response should be empty. User should be unauthorised.
if (string.IsNullOrEmpty(response))
{
RetryCount++;
if (RetryCount == 18)
{
ResetClientStatus();
return;
}
throw new Exception("");
}
document.LoadHtml(response);
// Get all badges from current page
ProcessBadgesOnPage(document);
}
}
catch (Exception ex)
{
Logger.Exception(ex, "Badge -> LoadBadgesAsync, for profile = " + Settings.Default.myProfileURL);
// badge page didn't load
picReadingPage.Image = null;
picIdleStatus.Image = null;
lblDrops.Text = localization.strings.badge_didnt_load.Replace("__num__", "10");
lblIdle.Text = "";
// Set the form height
var graphics = CreateGraphics();
var scale = graphics.DpiY * 1.625;
Height = Convert.ToInt32(scale);
ssFooter.Visible = false;
ReloadCount = 1;
tmrBadgeReload.Enabled = true;
return;
}
RetryCount = 0;
SortBadges(Settings.Default.sort);
picReadingPage.Visible = false;
UpdateStateInfo();
if (CardsRemaining == 0)
{
IdleComplete();
}
}
private void AnalyzeContent(ref dynamic results, string resultPath, string[] ignoreList, string filterFile)
{
WebClient webClient = new WebClient();
DirectoryInfo dir = new DirectoryInfo(resultPath);
if (String.IsNullOrEmpty(filterFile))
{
filterFile = "*";
}
// Get all files in directory with matched prefix name
FileInfo[] files = dir.GetFiles($"{filterFile}.html");
if (files.Count() == 0)
{
throw new Exception($"There is no file with name \"{filterFile}.html\" in folder \"{resultPath}\"");
}
// Loop all files and get data base on defined keywords
foreach (FileInfo file in files)
{
var filePath = "file:///" + resultPath.Replace("\\", "/") + "/" + file.Name;
string page = webClient.DownloadString(filePath);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(page);
var tables = doc.DocumentNode.SelectNodes("//ul[@id='test-collection']/li").ToList();
for (int i = 1; i <= tables.Count; i++)
{
var testNode = doc.DocumentNode.SelectNodes($"(//ul[@id='test-collection']/li)[{i}]").Single();
string testID = testNode.SelectNodes(".//span[@class='test-name']").Single().InnerText.Trim();
string startTime = testNode.SelectNodes(".//span[@class='label start-time']").Single().InnerText.Trim();
string endTime = testNode.SelectNodes(".//span[@class='label end-time']").Single().InnerText.Trim();
string status = Helper.UpperFirstCharacter(testNode.SelectNodes(".//div[@class='test-heading']/span[contains(@class,'test-status')]").Single().InnerText.Trim()) + "ed";
string message = "";
// Get error/warning message if test case is not pass
if (status != "Passed")
{
var lastNode = testNode.SelectNodes("(.//li[contains(@class,'node')])[last()]").Single();
if (lastNode.Attributes["status"].Value != "pass")
{
var lastLog = lastNode.SelectNodes("(.//tr[@class='log'])[last()]").Single();
if (lastLog.Attributes["status"].Value != "info" && lastLog.Attributes["status"].Value != "pass")
{
message = lastLog.SelectNodes(".//td[@class='step-details']").Single().InnerText.Trim();
}
}
}
// Ignore all test cases in ignore list
if (!Array.Exists(ignoreList, E => E == testID))
{
// If get only one data for each test case, compare and get data has latest end time
// Else get all raw data for each test case (one test case id can be run many times with different test result)
if (results.GetType() == typeof(Dictionary <string, string[]>))
{
if (results.ContainsKey(testID))
{
DateTime storedDate = DateTime.ParseExact(results[testID][Constant.TEST_END_TIME_INDEX].ToString(), REPORT_DATE_FORMAT, null);
DateTime currentDate = DateTime.ParseExact(endTime, REPORT_DATE_FORMAT, null);
if (storedDate < currentDate)
{
results[testID] = new string[] { startTime, endTime, status, message };
}
}
else
{
results[testID] = new string[] { startTime, endTime, status, message };
}
}
else
{
results.Add(new KeyValuePair <string, string[]>(testID, new string[] { startTime, endTime, status, message }));
}
}
}
}
}
//----------------------------------------------------------------------------------------------------------
public async void entryScan(String scanneo)// Metodo para poder Scanear
{
try
{
string Strings = scanneo;
var scaneo = scanneo.Length;
if (scanneo.Contains("http"))
{
var x = Strings.IndexOf("http");
Strings = Strings.Substring(x, 44);
scanneo = Strings;
Uri uri = new Uri(Strings);
string p = HttpUtility.ParseQueryString(uri.Query).Get("dm");//Obteniendo el paramentro dm que es el codigo unico de cada persona con marbete
Debug.WriteLine("Data: " + p);
var db = new SQLiteConnection(Preferences.Get("DB_PATH", ""));
var TB = db.Query <PLACA>("SELECT PLACA_CODE from PLACA WHERE codigo ='" + p + "'");// Buscando en la base de datos placa la placa que concuerde con ese codigo sin importar que persona este ingresando
if (TB.Any())
{
entPlaca.Text = TB.First().PLACA_CODE;
}
else
{
if (Connectivity.NetworkAccess == NetworkAccess.Internet) // En caso de que no encuentre a ninguna placa y haya conexion a internet buscara en la pagina de la DGII
{
if (scanneo.Contains("FOLIO")) // Este es una parte del string de la placas de del ano 2017-2018
{
await DisplayAlert("Error", "Este Marbete no es Valido", "Ok");
}
else
{
try
{
WebClient webClient = new WebClient();
string page = webClient.DownloadString(scanneo);
HtmlAgilityPack.HtmlDocument docu = new HtmlAgilityPack.HtmlDocument();
docu.LoadHtml(page);
List <List <string> > table = docu.DocumentNode.SelectSingleNode("//table[@class='data_table']")
.Descendants("tr")
.Skip(1)
.Where(tr => tr.Elements("td").Count() > 1)
.Select(tr => tr.Elements("td").Select(td => td.InnerText.Trim()).ToList())
.ToList();
var placa = table[5][1].ToString();
entPlaca.Text = placa;
Debug.WriteLine("Data: " + p);
var registroPlaca = new PLACA();
registroPlaca.CODIGO = p;
registroPlaca.PLACA_CODE = placa;
db.Insert(registroPlaca);
Console.WriteLine("Insertando: " + registroPlaca);
}
catch (Exception ex)
{
await DisplayAlert("Error", "Ha ocurrido un error", "Ok");
}
}
}
else
{
if (scanneo.Contains("FOLIO"))// Este es una parte del string de la placas de del ano 2017-2018
{
await DisplayAlert("Error", "Este Marbete no es Valido", "Ok");
}
else
{
DependencyService.Get <IToastMessage>().DisplayMessage("No posee internet digite manualmente.");
}
}
}
}
else
{
await DisplayAlert("Error", "Este Marbete no es Valido", "Ok");
}
}
catch (Exception ex)
{
Debug.WriteLine("Error en entryScan");
Analytics.TrackEvent("Error al escanear: " + ex.Message + "\n Escaner: " + Preferences.Get("LECTOR", "N/A"));
DependencyService.Get <IToastMessage>().DisplayMessage("Ha ocurrido un error: " + ex.Message);
}
}
public List <MarioMakerLevel> GetLevels(int page, int gameStyle, int courseTheme, int region, int difficulty, int tag, int uploadDate)
{
List <MarioMakerLevel> marioMakerLevels = new List <MarioMakerLevel>(); // Get a new list of marioMakerLevels
string finalTag = ""; // Create a finalTag variable
if (tag != 0) // If the tag is not 0
{
finalTag = tag.ToString(); // Then make the finalTag equal to the tag value
}
string startString = "https://supermariomakerbookmark.nintendo.net/search/result?"; // Starting string of website
string pageString = $"page={page + 1}"; // Get website string for page
string skinString = $"&q%5Bskin%5D={ gameStyles[gameStyle] }"; // Get website string for skin
string sceneString = $"&q%5Bscene%5D={ courseThemes[courseTheme] }"; // Get website string for theme
string areaString = $"&q%5Barea%5D={ regions[region] }"; // Get website string for region
string difficultyString = $"&q%5Bdifficulty%5D={ difficulties[difficulty] }"; // Get website string for difficulty
string tagString = $"&q%5Btag_id%5D={ finalTag }"; // Get website string for tag
string createdAtString = $"&q%5Bcreated_at%5D={ uploadDates[uploadDate] }"; // Get website string for uploaddate
string endString = $"&q%5Bsorting_item%5D=like_rate_desc"; // The ending string of website
// Compile all these strings into one string variable
string finalString = startString +
pageString +
skinString +
sceneString +
areaString +
difficultyString +
tagString +
createdAtString +
endString;
HtmlAgilityPack.HtmlWeb web = new HtmlAgilityPack.HtmlWeb(); // Get a html web object
HtmlAgilityPack.HtmlDocument doc = web.Load(finalString); // Create new doc object using web load of the string
if (doc != null) // If the doc collected is not null
{
for (int i = 0; i < 10; i++)
{
MarioMakerLevel level = new MarioMakerLevel();
// Get level title
try { level.Title = doc.DocumentNode.SelectNodes("//div[@class='course-title']")[i].InnerText; } catch { }
// Get level author
try { level.Author = doc.DocumentNode.SelectNodes("//div[@class='name']")[i].InnerText; } catch { }
// Get level theme
try { level.GameStyle = doc.DocumentNode.SelectNodes("//div[@class='gameskin-wrapper']")[i].FirstChild.Attributes[0].Value.Replace("gameskin bg-image common_gs_", ""); } catch { }
// Get level difficulty
try { level.Difficulty = doc.DocumentNode.SelectNodes("//div[@class='rank nonprize']")[i].NextSibling.InnerText; } catch { }
// Get level clear rate (Redacted)
//try { level.ClearRate = GetValueFromFont(doc.DocumentNode.SelectNodes("//div[@class='clear-rate']")[i]); } catch { }
// Get level stars
try { level.Stars = int.Parse(GetValueFromFont(doc.DocumentNode.SelectNodes("//div[@class='liked-count ']")[i])); } catch { }
// Get level players played
try { level.PlayersPlayed = int.Parse(GetValueFromFont(doc.DocumentNode.SelectNodes("//div[@class='played-count ']")[i])); } catch { }
// Get level clears
try { level.Clears = int.Parse(GetValueFromClears(doc.DocumentNode.SelectNodes("//div[@class='tried-count ']")[i])[0]); } catch { }
// Get level rounds played
try { level.RoundsPlayed = int.Parse(GetValueFromClears(doc.DocumentNode.SelectNodes("//div[@class='tried-count ']")[i])[1]); } catch { }
// Get level clear rate
try { level.ClearRate = GetValueFromClears(doc.DocumentNode.SelectNodes("//div[@class='tried-count ']")[i])[2]; } catch { }
// Get level tag
try { level.Tag = doc.DocumentNode.SelectNodes("//div[@class='course-tag radius5']")[i].InnerText; } catch { }
// Get level flag
try { level.Flag = doc.DocumentNode.SelectNodes("//div[@class='creator-info']")[i].FirstChild.Attributes[0].Value.Replace("flag ", ""); } catch { }
// Get level date uploaded
//level.DateUploaded = doc.DocumentNode.SelectNodes("//div[@class='created_at']")[i].InnerText;
// Get level link
try { level.Link = "https://supermariomakerbookmark.nintendo.net" + doc.DocumentNode.SelectNodes("//a[@class='button course-detail link']")[i].GetAttributeValue("href", "null"); } catch { }
marioMakerLevels.Add(level); // Add the level to the list of levels
}
return(marioMakerLevels); // Return levels
}
return(null); // Otherwise return null
}
void CrawlSingleHotel(Models.Hotel h)
{
_wk.ReportProgress(0, "Crawl hotel " + h.HotelName);
objBrowse._inputDatas["HMSID"] = h.HMSID;
objBrowse.ExecuteAction(false);
objGetWebRoomTypes.ExecuteAction(false);
//Find out what room types are available
var crTypes = h.ContractRooms.ToList();
var opthtml = objGetWebRoomTypes._ReturnDataCollection["cmbRoomTypeInnerHtml"] as string;
var rtdoc = new HtmlAgilityPack.HtmlDocument();
rtdoc.LoadHtml(opthtml);
var opts = rtdoc.DocumentNode.SelectNodes("//option");
var rts = opts.Select(o => o.InnerText).ToList();
foreach (var cr in crTypes)
{
if (!opts.Any(x => x.InnerText == cr.RoomName))
{
var editor = new frmRoomTypeReMatch(cr, rts);
if (editor.ShowDialog() == DialogResult.Cancel)
{
return;
}
}
}
var validDate = h.ContractValidTo;
if (validDate < DateTime.Now.Date)
{
return;
}
//here is the limit of HMS
if (validDate > DateTime.Today.AddDays(365))
{
validDate = DateTime.Today.AddDays(365);
}
var fromDate = h.ContractValidFrom;
var minDt = DateTime.UtcNow.Date.AddDays(h.MinCutOffDates);
if (fromDate < minDt)
{
fromDate = minDt;
}
//reload room types to update
var crbo = new BO.ContractRoomBO();
crTypes = crbo.GetQueryable(h.HotelId).ToList();
objEnterData._inputDatas["ToDate"] = validDate.ToString("MM/dd/yyyy");
objEnterData._inputDatas["FromDate"] = fromDate.ToString("MM/dd/yyyy");
objEnterData.ExecuteAction(false);
foreach (var cr in crTypes)
{
objClickCmbRoomTypes.ExecuteAction(false);
/* We don't want to get text into XPath because it can causes err*/
var cmbXPath = objGetWebRoomTypes._ActionData.ElementLabels.FirstOrDefault().XPath;
cmbXPath += "/option";
var elmOption = Modules.BrowserSupport.FindElementInCollection(cmbXPath, cr.RoomName);
elmOption.Click();
objClick.ExecuteAction(false);
objGetHtml.ExecuteAction(false);
var html = objGetHtml._ReturnDataCollection["PageHtml"] as string;
var analyzer = new Modules.AllotmentAnalyzer(true);
var records = analyzer.AnalyzeData(html);
foreach (var record in records)
{
var rbo = new BO.AllotmentRoomTypeBO();
var rtype = rbo.GetRecord(h.HotelId, record.RoomName);
if (rtype == null)
{
rtype = new Models.AllotmentRoomType();
rtype.RoomName = record.RoomName;
rtype.HotelId = h.HotelId;
rtype.DefaultAllotment = 1;
rbo.Add(rtype);
}
if (record.Allotment >= rtype.DefaultAllotment)
{
continue;
}
var arbo = new BO.AllotmentRecordBO();
//while record date is 12:00:00AM, saving in db will change it to 00:00:00
//so we never get the record. need to reset all to zero here
var dt = record.Date;
dt = new DateTime(dt.Year, dt.Month, dt.Day);
if (record.Allotment < rtype.DefaultAllotment)
{
var r = arbo.GetRecord(dt, rtype.RecordId);
if (r == null)
{
r = new Models.AllotmentRecord();
r.CurrentAllotment = record.Allotment;
r.AllotmentDate = dt;
r.Acknowledged = false;
r.AllotmentRoomTypeId = rtype.RecordId;
arbo.Add(r);
}
else
{
//REcord existed, may be acknowledged may be not. Just leave it there
if (record.Allotment < r.CurrentAllotment)
{
//Need to notify us again
r.Acknowledged = false;
arbo.Save();
}
}
}
}
}
// var path = @"C:\Users\Thuy Tran\Documents\AllDocs\Disposable\Test\allotment" + h.HotelId + "-" + DateTime.Now.ToString("mm-dd-yyyy-hh-mm-ss") + ".txt";
//System.IO.File.WriteAllText(path, html);
}
public Dictionary <string, List <List <string> > > Parse(string url)
{
try
{
Dictionary <string, List <List <string> > > result = new Dictionary <string, List <List <string> > >();
using (HttpClientHandler handler = new HttpClientHandler()
{
AllowAutoRedirect = false, AutomaticDecompression = System.Net.DecompressionMethods.Deflate | System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.None
})
{
using (var client = new HttpClient(handler))
{
using (HttpResponseMessage response = client.GetAsync(url).Result)
{
if (response.IsSuccessStatusCode)
{
var html = response.Content.ReadAsStringAsync().Result;
if (!string.IsNullOrEmpty(html))
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
var tables = doc.DocumentNode.SelectNodes(".//div[@class='block_content']//div[@class='two-table-row']//div[@class]");
if (tables != null && tables.Count > 0)
{
foreach (var table in tables)
{
var titleNode = table.SelectSingleNode(".//div[@class='head_tb']");
if (titleNode != null)
{
var table_b = table.SelectSingleNode(".//div[@class='tab_champ']//table");
if (table_b != null)
{
var rows = table_b.SelectNodes(".//tr");
if (rows != null && rows.Count > 0)
{
var res = new List <List <string> >();
foreach (var row in rows)
{
var cells = row.SelectNodes(".//td");
if (cells != null && cells.Count > 0)
{
res.Add(new List <string>(cells.Select(c => c.InnerText)));
}
}
result[titleNode.InnerText] = res;
}
}
}
}
return(result);
}
else
{
Console.WriteLine("Oooooopssss");
}
}
}
}
}
}
}
catch (Exception e)
{
}
return(null);
}
/// <summary>
/// Searches the official beatmap listing for beatmaps.
/// </summary>
public static async Task <SearchResultSet> Search(string query, SearchFilters.OsuRankStatus rankedFilter, SearchFilters.OsuModes modeFilter, int page = 1)
{
// ranked filter = r
// mode filter = m
string rParam;
// Ranked filter into website param
if (rankedFilter == SearchFilters.OsuRankStatus.RankedAndApproved)
{
rParam = "0";
}
else if (rankedFilter == SearchFilters.OsuRankStatus.Approved)
{
rParam = "6";
}
else if (rankedFilter == SearchFilters.OsuRankStatus.Qualified)
{
rParam = "11";
}
else if (rankedFilter == SearchFilters.OsuRankStatus.Loved)
{
rParam = "12";
}
else
{
rParam = "4";
}
// Search time. Need to use cookies.
// Same as bloodcat, construct QS
var qs = HttpUtility.ParseQueryString(string.Empty);
qs["q"] = query;
qs["m"] = ((int)modeFilter).ToString();
qs["r"] = rParam;
if (page > 1)
{
qs["page"] = page.ToString();
}
string rawData = await GetRawWithCookies("https://osu.ppy.sh/p/beatmaplist?" + qs.ToString());
// Check if still logged in
if (rawData.Contains("Please enter your credentials"))
{
throw new CookiesExpiredException();
}
// Parse.
var htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionUseIdAttribute = true;
htmlDoc.LoadHtml(rawData);
HtmlAgilityPack.HtmlNodeCollection beatmapNodes = htmlDoc.DocumentNode.SelectNodes("//div[@class='beatmapListing']/div[@class='beatmap']");
if (beatmapNodes == null)
{
return(new SearchResultSet(new List <BeatmapSet>(), false)); // empty
}
var sets = beatmapNodes.Select(b => {
var difficulties = new Dictionary <string, string>();
try
{
int i = 1;
foreach (var d in b.SelectNodes("div[@class='left-aligned']/div[starts-with(@class, 'difficulties')]/div"))
{
string _d = d.Attributes["class"].Value.Replace("diffIcon ", "");
if (_d.Contains("-t"))
{
_d = "1"; // taiko
}
else if (_d.Contains("-f"))
{
_d = "2"; // ctb
}
else if (_d.Contains("-m"))
{
_d = "3"; // mania
}
else
{
_d = "0"; // standard
}
difficulties.Add(i.ToString(), _d);
i++;
}
}
catch { } // rip
// we can only base this off that green/red bar, lol -- or the search filter
string rankStatus;
if (rankedFilter == SearchFilters.OsuRankStatus.Loved)
{
rankStatus = "Loved";
}
else if (rankedFilter == SearchFilters.OsuRankStatus.Approved)
{
rankStatus = "Approved";
}
else if (rankedFilter == SearchFilters.OsuRankStatus.Qualified)
{
rankStatus = "Qualified";
}
else if (rankedFilter == SearchFilters.OsuRankStatus.RankedAndApproved)
{
rankStatus = "Ranked/Approved";
}
else if (b.SelectSingleNode("div[@class='right-aligned']/div[@class='rating']") != null)
{
rankStatus = "Ranked/Apprv./Quali./Loved";
}
else
{
rankStatus = "Pending/Graveyard";
}
return(new BeatmapSet(
b.Id,
TryGetNodeText(b, "div[@class='maintext']/span[@class='artist']"),
TryGetNodeText(b, "div[@class='maintext']/a[@class='title']"),
TryGetNodeText(b, "div[@class='left-aligned']/div[1]/a"),
rankStatus,
difficulties,
null
));
});
bool canLoadMore = htmlDoc.DocumentNode.SelectNodes("//div[@class='pagination']")
.Descendants("a")
.Any(d => d.InnerText == "Next");
return(new SearchResultSet(sets, canLoadMore));
}
/// <summary>
/// Generate card data from files
/// </summary>
/// <param name="setList"></param>
private void GenerateCards(ref Dictionary <int, string> setList)
{
ArchivistDatabase adb = new ArchivistDatabase();
string currentCardName = string.Empty;
string paraCardName = string.Empty, paraCost = string.Empty, paraPowTgh = string.Empty, paraRulesText = string.Empty, paraType = string.Empty;
string paraCardExtCID = string.Empty, paraCardExtRar = string.Empty, paraMultiverseidString = string.Empty;
int /*paraCardExtEID,*/ paraMultiverseid = 0;
int id = 1;
int extId = setList.Count + 1;
foreach (KeyValuePair <int, string> ext in setList)
{
string extoutfile = String.Format("{0}\\{1}.dat", tempDirectory, System.Web.HttpUtility.UrlEncode(ext.Value));
UpdateListText(String.Format("Analyzing extension file for {0}...", ext.Value));
if (!System.IO.File.Exists(extoutfile))
{
UpdateListText("File not found. Skipping.");
continue;
}
HtmlAgilityPack.HtmlWeb web = new HtmlAgilityPack.HtmlWeb();
HtmlAgilityPack.HtmlDocument doc = web.Load(extoutfile);
HtmlAgilityPack.HtmlNode textspoilerNode = doc.DocumentNode.SelectSingleNode("//div[@class=\"textspoiler\"]/table");
foreach (HtmlAgilityPack.HtmlNode rows in textspoilerNode.SelectNodes("tr"))
{
HtmlAgilityPack.HtmlNodeCollection cols = rows.SelectNodes("td");
if (cols.Count == 2)
{
string key = cols[0].InnerText.Replace(":", "").Trim();
string value = cols[1].InnerText.TrimStart().TrimEnd();
if (key == "Name")
{
currentCardName = value;
paraCardName = value;
string href = cols[1].SelectSingleNode("a").GetAttributeValue("href", ""); //../Card/Details.aspx?multiverseid=201281
paraMultiverseidString = href.Substring(href.LastIndexOf("=") + 1); //201281
paraMultiverseid = Convert.ToInt32(paraMultiverseidString);
}
if (key == "Cost")
{
paraCost = value;
}
if (key == "Type")
{
paraType = value.Replace("—", "-").Replace(" ", " ");
}
if (key == "Pow/Tgh")
{
paraPowTgh = value;
}
if (key == "Rules Text")
{
paraRulesText = value;
}
if (key == "Set/Rarity")
{
paraCardExtRar = value.Replace("\"", """);
}
}
else
{
if (currentCardName != "")
{
string[] setrlist = paraCardExtRar.Split(',');
string cid = string.Empty;
foreach (string setr in setrlist)
{
if (setr.Contains(ext.Value))
{
string set = ext.Value.Replace(""", "\"").Trim();
string rarity = setr.Replace(ext.Value, "").Trim(); // Might be Common/Uncomm/Rare/Mythic Rare
Card card = MagicCardFactory.BuildCard(paraCardName, paraCost, paraPowTgh, paraRulesText, paraType, rarity, set, paraMultiverseid);
cid = adb.InsertCard(card);
break;
}
}
if (string.IsNullOrEmpty(cid))
{
UpdateListText("Error inserting card: " + currentCardName);
}
}
// New card
paraCardName = null; paraCost = null; paraType = null;
paraPowTgh = null; paraRulesText = null; paraCardExtRar = null;
paraMultiverseidString = null;
currentCardName = "";
}
}
UpdateTotalStatus(setList.Count + id + 1, 2 * setList.Count + 2);
id++;
}
}
private TorrentzResult GetSearchResult(string hash, string details)
{
string[] tmpArray = details.Split('|');
_searchResult = new TorrentzResult();
_searchResult.Trackers = new List <string>();
_torrentDates = new List <string>();
_torrentLinks = new List <string>();
//trackerLinks = new List<string>();
_torrentTitle = string.Empty;
_torrentContents = string.Empty;
_torrentSize = string.Empty;
_torrentSeeders = string.Empty;
_torrentLeechers = string.Empty;
_searchResult.InfoHash = hash.Replace("/", string.Empty);
if (tmpArray[0] == _verifiedChar.ToString())
{
_torrentVerified = string.Format("[{0}]", _verifiedChar);
}
else
{
_torrentVerified = "[-]";
}
_torrentSize = tmpArray[2];
_torrentSeeders = tmpArray[3];
_torrentLeechers = tmpArray[4];
_document = new HtmlAgilityPack.HtmlDocument();
_response = _client.DownloadString(_baseUrl + hash);
_document.LoadHtml(_response);
var downloadNode = _document.DocumentNode.SelectSingleNode("//div[@class='downlinks']");
var trackersNode = _document.DocumentNode.SelectSingleNode("//div[@class='trackers']");
var filesNode = _document.DocumentNode.SelectSingleNode("//div[@class='files']");
if (downloadNode != null)
{
var datesNode = downloadNode.SelectNodes("//span[@title]");
var titleNode = downloadNode.SelectSingleNode("//h2");
var dtNode = downloadNode.SelectNodes("//dt");
if (datesNode != null)
{
foreach (var d in datesNode)
{
_torrentDates.Add(d.InnerHtml.Trim());
}
}
if (titleNode != null)
{
string[] tmp = titleNode.InnerHtml.Split(new string[] { "</span>" }, StringSplitOptions.RemoveEmptyEntries);
tmp[0] = tmp[0].Replace("<span>", string.Empty);
_torrentTitle = tmp[0].Trim();
}
if (dtNode != null)
{
foreach (var dt in dtNode)
{
var linkNode = dt.SelectNodes("//a[@href]");
//var siteNode = dt.SelectNodes("//span[@class='u']");
var nameNode = dt.SelectNodes("//span[@class='n']");
if (linkNode != null)
{
foreach (var c in linkNode)
{
if (!_torrentLinks.Contains(c.GetAttributeValue("href", string.Empty)))
{
_torrentLinks.Add(c.GetAttributeValue("href", string.Empty));
}
}
//FilterResults(ref torrentLinks);
}
//if (siteNode != null)
//{
// foreach (var sn in siteNode)
// {
// if (!IsOnlyDigits(sn.InnerText))
// {
// if (!trackerLinks.Contains(sn.InnerText))
// {
// trackerLinks.Add(sn.InnerText);
// }
// }
// }
//}
}
}
_searchResult.Dates = _torrentDates;
_searchResult.Age = _torrentDates[0];
_searchResult.Links = _torrentLinks;
_searchResult.Title = _torrentTitle;
_searchResult.Dates.Remove(_searchResult.Dates.Last());
//SearchResult.Dates.Remove(SearchResult.Dates.First());
//SearchResult.Trackers = trackerLinks;
_searchResult.Leechers = _torrentLeechers;
_searchResult.Seeders = _torrentSeeders;
_searchResult.Size = _torrentSize;
_searchResult.Verified = _torrentVerified;
}
if (trackersNode != null)
{
var dtNodes = trackersNode.SelectNodes("//dt");
if (dtNodes != null)
{
foreach (var dt in dtNodes)
{
if (dt.InnerText.Contains("://"))
{
_searchResult.Trackers.Add(dt.InnerText.Trim());
}
}
}
}
if (filesNode != null)
{
var liNodes = filesNode.SelectNodes("//li[@class='t']");
if (liNodes != null)
{
foreach (var li in liNodes)
{
_torrentContents = SanitizeTorrentContents(li.InnerHtml);
break;
}
}
}
_searchResult.Contents = _torrentContents;
return(_searchResult);
}
/*
*
* how to use getvideos
* Pagecount:is the number of pages that will be scraped
* querry:is used when you want to search video results that match with the written criteria
* page: is used to load the current page of that criteria
* if you leave querry with the "" it will automatically search results from the home
* .
* as result of the getvideos it will return yo an pagedata who contains the following
* pagedata {
* navigationmax:is the max number of pages that you can use in the current search
* videomodels[]: is an array of videomodels who contains some information of each videos that match
* with the criteria
*
* }
*
*/
public Modals.pagedata getvideos(int pagecount, string querry = "", int page = 0)
{
var videos = new List <Modals.videosmodels>();
Modals.pagedata pagedataa = new Modals.pagedata();
string baseurl = "";
if (querry == "")
{
baseurl = "http://www.xvideos.com/";
}
else
{
baseurl = "http://www.xvideos.com/?k=" + querry.Replace(' ', '+');
}
for (int i = 0; i < pagecount; i++)
{
int pageno = i;
if (page > 0)
{
pageno = page;
}
var doc2 = new HtmlAgilityPack.HtmlWeb();
HtmlAgilityPack.HtmlDocument htmlDoc2 = null;
/////////////se busca la pagina de info de el video
if (querry != "")
{
htmlDoc2 = doc2.LoadFromWebAsync(baseurl + "&p=" + pageno).Result;
}
else
{
if (page > 0)
{
htmlDoc2 = doc2.LoadFromWebAsync(baseurl + "new/" + (pageno + 1) + "/").Result;
}
else
{
htmlDoc2 = doc2.LoadFromWebAsync(baseurl).Result;
}
}
var paginations = htmlDoc2.DocumentNode.SelectNodes("//*[contains(@class,'pagination')]");
if (!paginations.Last().Attributes["class"].Value.Contains("pagination-with-settings"))
{
var elemsx = paginations.Last().ChildNodes["ul"].ChildNodes;
var outfake = 0;
var numeros = elemsx.Where(aax => int.TryParse(aax.InnerText, out outfake));
pagedataa.navigationmax = int.Parse(numeros.Last().InnerText);
}
else
{
pagedataa.navigationmax = 0;
}
var elems = htmlDoc2.DocumentNode.SelectNodes("//*[contains(@class,'thumb-block')]");
foreach (var xd in elems)
{
var elemento = new Modals.videosmodels();
elemento.link = "http://www.xvideos.com" + xd.Descendants().Where(aax => aax.Attributes["class"].Value == "thumb").First().ChildNodes["a"].Attributes["href"].Value;
if (!elemento.link.Contains("/pornstar-channels/") && !elemento.link.Contains("/model-channels/") && !elemento.link.Contains("/profiles/"))
{
var elemthumb = xd.Descendants().Where(aax => aax.Attributes["class"].Value == "thumb").First().ChildNodes["a"].ChildNodes["img"];
try
{
elemento.thumb = elemthumb.Attributes["data-src"].Value;
}
catch (Exception)
{
elemento.thumb = elemthumb.Attributes["src"].Value;
}
elemento.title = WebUtility.HtmlDecode(xd.ChildNodes[1].ChildNodes["p"].ChildNodes["a"].Attributes["title"].Value);
elemento.duration = xd.ChildNodes[1].ChildNodes[1].ChildNodes["span"].ChildNodes["span"].InnerText;
videos.Add(elemento);
Console.WriteLine(videos.Count - 1 + "===>" + elemento.title);
}
}
}
pagedataa.videos = videos;
return(pagedataa);
}
/// <summary>
/// Check for new version.
/// </summary>
public void CheckSourceforge()
{
UpdateListText("Checking version...");
bool needsUpdate = true;
try
{
string versionLatest;
// Check for latest version
string downloadUrl = "https://sourceforge.net/projects/archivist/files/archivist2/";
UpdateTotalStatus(10, 100);
HtmlAgilityPack.HtmlWeb web = new HtmlAgilityPack.HtmlWeb();
HtmlAgilityPack.HtmlDocument doc = web.Load(downloadUrl);
UpdateTotalStatus(90, 100);
HtmlAgilityPack.HtmlNode textspoilerNode = doc.DocumentNode.SelectSingleNode("//div[@class=\"download-bar\"]/strong/a/span");
if (textspoilerNode != null)
{
versionLatest = textspoilerNode.InnerText.Substring(textspoilerNode.InnerText.IndexOf("_") + 1);
int lastPos = -1;
if ((lastPos = versionLatest.LastIndexOf(".rar")) > 0)
{
versionLatest = versionLatest.Substring(0, lastPos);
}
if ((lastPos = versionLatest.LastIndexOf(".exe")) > 0)
{
versionLatest = versionLatest.Substring(0, lastPos);
}
}
else
{
throw new Exception("Error reading file list.");
}
UpdateTotalStatus(95, 100);
string versionCurrent = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString();
UpdateListText("Current Version: " + versionCurrent);
UpdateListText("Latest Version: " + versionLatest);
needsUpdate = versionCurrent != versionLatest;
UpdateTotalStatus(100, 100);
}
catch (Exception e)
{
UpdateTotalStatus(0, 100);
UpdateListText(e.ToString(), true);
}
if (needsUpdate)
{
UpdateListText("There is a new version available.");
UpdateListText("Opening https://sourceforge.net/projects/archivist/", true);
System.Diagnostics.Process.Start("https://sourceforge.net/projects/archivist/");
}
else
{
UpdateListText("You have the latest version installed.", true);
}
}