public void RetreiveItems(HtmlAgilityPack.HtmlNode Item)
{
string buyingFormat = "";
string id = "";
string price = "";
string title = "";
string shippingPrice = "";
//product id
id = Item.GetAttributeValue("listingid", "");
//product title
title = Item.Descendants("h3").Where(node => node.GetAttributeValue("class", "").
Equals("lvtitle")).FirstOrDefault().InnerText.Trim(elementsToTrim);
//product price
price = "" + Regex.Match(Item.Descendants("li").
Where(node => node.GetAttributeValue("class", "").
Equals("lvprice prc")).FirstOrDefault().
InnerText.Trim(elementsToTrim), @"\d+.\d+");
//shippping format
shippingPrice = Item.Descendants("li").
Where(node => node.GetAttributeValue("class", "").
Equals("lvshipping")).FirstOrDefault().
InnerText.Trim(elementsToTrim);
//buying format
buyingFormat = Item.Descendants("li").
Where(node => node.GetAttributeValue("class", "").
Equals("lvformat")).FirstOrDefault().
InnerText.Trim(elementsToTrim);
PrintAllData(id, title, buyingFormat, price, shippingPrice);
}
private async Task LoadContent(DataItem _item)
{
string contentStr = string.Empty;
try
{
contentStr = await Define.DownloadStringAsync(_item.Link);
}
catch (Exception ex)
{
//throw;
}
if (contentStr != string.Empty)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(contentStr);
HtmlAgilityPack.HtmlNode htmlNode = htmlDoc.GetElementbyId("ContentContainer");
while (htmlNode.Descendants("script").Count() > 0)
{
htmlNode.Descendants("script").ElementAt(0).Remove();
}
while (htmlNode.Descendants("meta").Count() > 0)
{
htmlNode.Descendants("meta").ElementAt(0).Remove();
}
//contentStr = "<p><i>This blog post was authored by Andrew Byrne (<a href=\"http://twitter.com/AndrewJByrne\" target=\"_blank\">@AndrewJByrne</a>), a Senior Content Developer on the Windows Phone Developer Content team.</i> <p><i></i> <p><i>- Adam</i></p> <hr> <p> <table cellspacing=\"1\" cellpadding=\"2\" width=\"722\" border=\"0\"> <tbody> <tr> <td valign=\"top\" width=\"397\"> <p>The Windows Phone Developer Content team is continually adding new code samples that you can download from MSDN. In this post, we introduce you to the 10 latest samples that we’ve posted on MSDN. Each sample includes a readme file that walks you through building and running the sample, and includes links to relevant, related documentation. We hope these samples help you with your app development and we look forward to providing more samples as we move forward. You can check out all of our <a href=\"http://code.msdn.microsoft.com/wpapps/site/search?f%5B0%5D.Type=Contributors&f%5B0%5D.Value=Windows%20Phone%20SDK%20Team&f%5B0%5D.Text=Windows%20Phone%20SDK&sortBy=Date\" target=\"_blank\">samples on MSDN</a>.</p></td> <td valign=\"top\" width=\"320\"> <p><a href=\"http://blogs.windows.com/cfs-file.ashx/__key/communityserver-blogs-components-weblogfiles/00-00-00-53-84-metablogapi/clip_5F00_image002_5F00_1765A66A.png\"><img title=\"clip_image002\" style=\"border-top: 0px; border-right: 0px; background-image: none; border-bottom: 0px; float: none; padding-top: 0px; padding-left: 0px; margin-left: auto; border-left: 0px; display: block; padding-right: 0px; margin-right: auto\" border=\"0\" alt=\"clip_image002\" src=\"http://blogs.windows.com/cfs-file.ashx/__key/communityserver-blogs-components-weblogfiles/00-00-00-53-84-metablogapi/clip_5F00_image002_5F00_thumb_5F00_7B083E7C.png\" width=\"121\" height=\"213\"></a></p></td></tr></tbody></table></p> <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306704\" target=\"_blank\">Stored Contacts Sample</a></h3> <p>This sample illustrates how to use the <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.phone.personalinformation.contactstore(v=vs.105).aspx\" target=\"_blank\">ContactStore</a> class and related APIs to create a contact store for your app. This feature is useful if your app uses an existing cloud-based contact store. You can use the APIs you to create contacts on the phone that represent the contacts in your remote store. You can display and modify the contacts in the People Hub on your phone, just like contacts that are added through the built-in experience. You can use the APIs to update and delete contacts you have created on the phone and also to query for any changes the user has made to the contacts locally so you can sync those changes to your remote store. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306701\" target=\"_blank\">Basic Storage Recipes</a></h3> <p>This is a “Windows Runtime Storage 101” sample for Windows Phone developers moving from isolated storage and <b>System.IO</b> to <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.storage.aspx\" target=\"_blank\">Windows.Storage</a> and <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.storage.streams.aspx\" target=\"_blank\">Windows.Storage.Streams</a>. The sample demonstrates how to write to and read files, in addition to how to enumerate directory trees. It also demonstrates how to pass data from one page to the next, and how to persist application state when the app is deactivated. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=301509\" target=\"_blank\">Trial Experience Sample</a></h3> <p>This sample shows you how to design your app to detect its license state when the app launches, and how to detect changes to the license state while running. It comes with a helper class that you can use in your app to wrap <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.applicationmodel.store.licenseinformation.aspx\" target=\"_blank\">LicenseInformation</a> functionality. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=302059\" target=\"_blank\">Windows Runtime Component Sample</a></h3> <p>This sample demonstrates the basics of creating a Windows Phone Runtime component in C++ and consuming it in a XAML app. The sample demonstrates three scenarios: the first scenario illustrates how to call synchronous and asynchronous methods to perform a computation. The second scenario uses the same computation component to demonstrate progress reporting and cancellation of long-running tasks. Finally, the third scenario shows how to use a component to wrap logic that uses <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/jj206944(v=vs.105).aspx\" target=\"_blank\">XAudio2 APIs</a> to play a sound. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306097\" target=\"_blank\">Company Hub Sample</a></h3> <p>This sample demonstrates the construction of an app that is capable of deploying line-of-business (LOB) apps to employees of a corporation. The sample uses an example XML file to define the company XAPs that are available to employees for secure download, and shows you how to dynamically access that file at run time. Then it shows you how to install company apps, enumerate the apps, and then launch the installed company apps. This app is just an example framework and requires additional work beyond the sample to be functional. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306702\" target=\"_blank\">Image Recipes</a></h3> <p>This sample illustrates how to use images in your app efficiently, while giving your users a great experience. It tackles downsampling images, implementing pinch and zoom, and downloading images with a progress display and an option to cancel the download. We’ve taken a recipe approach: each recipe is delivered in a self-contained page in the app so you can focus your attention on the particular part of the sample you are most interested in. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=306026\" target=\"_blank\">Azure Voice Notes</a></h3> <p>This sample uses Windows Phone speech recognition APIs and Windows Azure Mobile Services to record voice notes as text and store the notes in the cloud. It shows how Mobile Services can be used to authenticate a user with their Microsoft Account. It also demonstrates how to use Mobile Services to store, retrieve, and delete data from an Azure database table. The app generates text from speech using the Windows Phone speech recognition APIs and the phone’s predefined dictation grammar. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=299241\" target=\"_blank\">Kid's Corner Sample</a></h3> <p>This sample illustrates how to use the <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.phone.applicationmodel.applicationprofile.modes(v=vs.105).aspx\" target=\"_blank\">ApplicationProfile.Modes</a> property to recognize Kid’s Corner mode. When the app runs, it checks the <a href=\"http://msdn.microsoft.com/en-us/library/windowsphone/develop/windows.phone.applicationmodel.applicationprofile.modes(v=vs.105).aspx\" target=\"_blank\">ApplicationProfile.Modes</a> property. If the value is <b>ApplicationProfileModes.Alternate</b>, you’ll know that the app is running in Kid’s Corner mode. Depending on the content of your app, you may want to change its appearance or features when it runs in Kid’s Corner mode. Some features that you should consider disabling when running in Kid’s Corner mode include in-app purchases, launching the web browser, and the ad control. <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=267468\" target=\"_blank\">URI Association Sample</a></h3> <p>Use this sample to learn how to automatically launch your app via URI association. This sample includes three apps: a URI launcher, and two apps that handle the URI schemes that are built in to the launcher app. You can launch an app that is included with the sample or edit the URI and launch a different app. There is also a button for launching a URI on another phone using Near Field Communication (NFC). <h3><a href=\"http://go.microsoft.com/fwlink/?LinkId=275007\" target=\"_blank\">Speech for Windows Phone: Speech recognition using a custom grammar</a></h3> <p>A grammar defines the words and phrases that an app will recognize in speech input. This sample shows you how to go beyond the basics to create a powerful grammar (based on the grammar schema) with which your app can recognize commands and phrases constructed in different ways. <p> <p>This post is just a glimpse of the latest Windows Phone samples we’ve added to the MSDN code gallery. From launching apps with URI associations, to dictating notes and storing them in the cloud, we hope that there’s something for everyone. We’ll be sure to keep you posted as new samples are added to the collection, so stay tuned. In the meantime, grab the samples, experiment with them, and use the code to light up your apps. You can download all Windows Phone samples at <a href=\"http://code.msdn.microsoft.com/wpapps\" target=\"_blank\">http://code.msdn.microsoft.com/wpapps</a>. <div style=\"clear:both;\"></div><img src=\"http://blogs.windows.com/aggbug.aspx?PostID=588575&AppID=5384&AppType=Weblog&ContentType=0\" width=\"1\" height=\"1\">";
Items[Items.IndexOf(_item)].Content = htmlNode.InnerHtml.Replace("\r", "").Replace("\n", "");
}
}
/// <summary>
///
/// </summary>
/// <param name="node"></param>
/// <param name="element"></param>
/// <param name="attribute"></param>
/// <param name="valuePortion"></param>
/// <returns></returns>
public static HtmlAgilityPack.HtmlNode GetNode(HtmlAgilityPack.HtmlNode node,
string element,
string attribute,
string valuePortion,
bool tryNull = false)
{
if (tryNull)
{
try
{
var _nodes = node.Descendants(element)
.Where(d => d.Attributes.Contains(attribute) && d.Attributes[attribute].Value.Contains(valuePortion));
if (_nodes != null && _nodes.Count() > 0)
{
var _node = _nodes.ToArray()[0];//.SingleOrDefault();
return(_node);
}
}
catch { }
}
else
{
var _nodes = node.Descendants(element)
.Where(d => d.Attributes.Contains(attribute) && d.Attributes[attribute].Value.Contains(valuePortion));
if (_nodes != null && _nodes.Count() > 0)
{
var _node = _nodes.ToArray()[0];//.SingleOrDefault();
return(_node);
}
}
return(null);
}
private static IEnumerable <HtmlAgilityPack.HtmlNode> DescendantsSelector(this HtmlAgilityPack.HtmlNode source, string elementName, bool self)
{
if (self)
{
return(elementName == "*" ? source.DescendantsAndSelf() : source.DescendantsAndSelf(elementName));
}
else
{
return(elementName == "*" ? source.Descendants() : source.Descendants(elementName));
}
}
public static HtmlAgilityPack.HtmlNode rmsssScript(this HtmlAgilityPack.HtmlNode node)
{
if (node.Descendants("script") != null)
{
foreach (var script in node.Descendants("script").ToArray())
{
script.Remove();
}
}
return(node);
}
protected override string GetPosition(HtmlAgilityPack.HtmlNode div)
{
return(div.Descendants("a").Where(
d => d.Attributes.Contains("class") &&
d.Attributes["class"].Value.Contains("rua-b-vacancy-title") || d.Attributes["class"].Value.Contains("jqKeywordHighlight")
).Select(d => d.InnerText).First());
}
private void LoadTaskCreateAndModifyTime(UTrackTask task)
{
string url = string.Format("https://ustr-vm-0315.na.uis.unisys.com:8443/uTrack/TaskAdmin?tid={0}", task.ID);
HtmlAgilityPack.HtmlNode htmlNode = FeatureData(url);
var allRow = htmlNode.Descendants().Where(t => t.Attributes.Contains("class") && t.Attributes["class"].Value == "row").ToList();
if (allRow.Any())
{
var createLabelText = SpiderUtil.TrimHtmlTag(allRow[10].InnerText);
var modifiedLabelText = SpiderUtil.TrimHtmlTag(allRow[11].InnerText);
var createdLable = createLabelText.Substring(createLabelText.Length - 10, 10);
var modifiedLabel = modifiedLabelText.Substring(modifiedLabelText.Length - 10, 10);
task.CreatedOwner = createLabelText.Substring(0, createLabelText.Length - 10).Replace("Created:", string.Empty).Trim();
task.ModifiedOwner = modifiedLabelText.Substring(0, modifiedLabelText.Length - 10).Replace("Modified:", string.Empty).Trim();
if (string.IsNullOrEmpty(task.ModifiedOwner))
{
task.ModifiedOwner = task.CreatedOwner;
}
task.Created = DateTime.Parse(createdLable);
task.Modified = DateTime.Parse(modifiedLabel);
}
}
/// <summary>
/// Get all transfers data as list of strings.
/// </summary>
/// <param name="htmlDocument">HTML document string representation.</param>
/// <returns>String list of data.</returns>
public List <String> GetTransfersData(string htmlDocument)
{
// Data container.
List <string> dataToSave = new List <string>();
// Load downloaded website.
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(htmlDocument);
// Find proper HTML element with transfers 'table'.
HtmlAgilityPack.HtmlNode div = htmlDoc.DocumentNode.SelectSingleNode("//div[@class='panes']");
// Get all players with data, each data element as separate list element.
if (div != null)
{
dataToSave = div.Descendants("li")
.Select(a => a.InnerText)
.ToList();
}
else
{
throw new Exception("Could not find div with transfers!");
}
return(dataToSave);
}
private void webBrowser1_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
if (webBrowser1.Url.AbsoluteUri != e.Url.AbsoluteUri)
{
return;
}
int asdf = 0;
asdf++; // 디버그할때 걸리도록 함
// webBrowser1.Document.Encoding
// webBrowser1.DocumentText
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
Stream docStream = this.webBrowser1.DocumentStream;
StreamReader streamReader = new StreamReader(docStream,
System.Text.Encoding.GetEncoding(webBrowser1.Document.Encoding));
string strHtml = streamReader.ReadToEnd(); // 대충 인코딩하고 strHtml에 담는다는 내용
// 대충 필요한 부분만 잘라내는 내용
HtmlAgilityPack.HtmlNode parseNode = doc.DocumentNode.
SelectSingleNode("//div[@class='box_type_l']").SelectSingleNode("//table[@class='type_2']");
List <List <string> > parsedTbl = parseNode.Descendants("tr") //tr기준으로 자름
.Skip(1) // To Skip Table Header Row 헤더 스킵하려고
.Where(tr => tr.Elements("td").Count() > 1) // element를 td 기준으로 잘라냄
.Select(tr => tr.Elements("td")
.Select(td => td.InnerText.Trim()) // InnerHtml : 표 안에있는 글자 <-> OuterHtml
.ToList()).ToList();
}
public void ReplaceTagsWithOtherTag(HtmlAgilityPack.HtmlNode elem, string originaltag, string destinationtag)
{
foreach (HtmlAgilityPack.HtmlNode e in elem.Descendants(originaltag).ToList())
{
e.Name = destinationtag;
}
}
protected override string GetVacancyUrl(HtmlAgilityPack.HtmlNode row)
{
var vacancyHref = row.Descendants("a").Where(
r => r.Attributes.Contains("class") && r.Attributes["class"].Value.Contains("title"))
.Select(r => r.Attributes["href"].Value).SingleOrDefault();
return(BaseUrl + vacancyHref);
}
public static IEnumerable <HtmlAgilityPack.HtmlNode> GetElementWithClass(this HtmlAgilityPack.HtmlNode htmlNode, string classname)
{
var cssClassAttribute = "class";
var elements = htmlNode.Descendants().Where(node => node.Attributes[cssClassAttribute] != null &&
node.Attributes[cssClassAttribute].Value.Contains(classname)).ToList();
return(elements);
}
/// <summary>
///
/// </summary>
/// <param name="node"></param>
/// <param name="element"></param>
/// <param name="attribute"></param>
/// <param name="valuePortion"></param>
/// <returns></returns>
public static IEnumerable <HtmlAgilityPack.HtmlNode> GetNodeCollection(HtmlAgilityPack.HtmlNode node,
string element,
string attribute,
string valuePortion)
{
var _nodes = node.Descendants(element)
.Where(d => d.Attributes.Contains(attribute) && d.Attributes[attribute].Value.Contains(valuePortion));
return(_nodes);
}
private void LoadStoryTaskInfomation(Story story)
{
string url = string.Format("https://ustr-vm-0315.na.uis.unisys.com:8443/uTrack/TaskList?fid={0}", story.ID);
HtmlAgilityPack.HtmlNode htmlNode = FeatureData(url);
var taskList = htmlNode.Descendants().Where(t => t.Name == "tr").ToList();
bool alreadySkippedHeader = false;
foreach (var taskItem in taskList)
{
if (!alreadySkippedHeader)
{
alreadySkippedHeader = true;
continue;
}
var taskItemColumn = taskItem.ChildNodes.Where(t => t.Name == "td").ToList();
var Id = SpiderUtil.TrimHtmlTag(taskItemColumn[0].ChildNodes.Where(t => t.Name == "a").ToList().First().InnerText);
if (string.IsNullOrEmpty(Id))
{
Id = SpiderUtil.TrimHtmlTag(taskItemColumn[0].ChildNodes.First().InnerText);
}
var name = SpiderUtil.TrimHtmlTag(taskItemColumn[1].ChildNodes.First().InnerText);
var type = SpiderUtil.TrimHtmlTag(taskItemColumn[2].Attributes["title"].Value);
var priority = int.Parse(SpiderUtil.TrimHtmlTag(taskItemColumn[3].InnerText));
var owner = SpiderUtil.TrimHtmlTag(taskItemColumn[4].ChildNodes.Last().InnerText);
var status = SpiderUtil.TrimHtmlTag(taskItemColumn[5].Attributes["title"].Value);
decimal estimate = decimal.Parse(SpiderUtil.TrimHtmlTag(taskItemColumn[6].InnerText));
decimal workDone = decimal.Parse(SpiderUtil.TrimHtmlTag(taskItemColumn[7].InnerText));
decimal workTodo = decimal.Parse(SpiderUtil.TrimHtmlTag(taskItemColumn[8].InnerText));
UTrackTask task = new UTrackTask();
task.ID = Id;
task.Description = name;
task.Type = ConvertToTaskType(type);
task.Priority = priority;
task.Owner = owner;
task.Status = (Unisys.Trend.DataModel.TaskStatus)Enum.Parse(typeof(Unisys.Trend.DataModel.TaskStatus), status.Replace(" ", ""));
task.Estimate = estimate;
task.WorkDone = workDone;
task.WorkToDo = workTodo;
if (story.Status == StoryStatus.Done || story.Status == StoryStatus.Pending || story.Status == StoryStatus.InProcess)
{
LoadTaskCreateAndModifyTime(task);
}
story.Tasks.Add(task);
}
}
/// <summary>
///
/// </summary>
/// <param name="htmlDocument"></param>
/// <param name="_errMsg"></param>
/// <returns></returns>
public bool CheckPageIfError(string htmlDocument, ref string _errMsg)
{
HtmlAgilityPack.HtmlDocument _doc = new HtmlAgilityPack.HtmlDocument();
_doc.LoadHtml(htmlDocument);
//http://www.yellowpages.com.au/search/listings?clue=kobo&locationClue=lyoo&selectedViewMode=LIST&emsLocationId=
/*<div id="zeroResultsMessage">
* We couldn't find any results for <strong>kobo</strong> in
* <strong>Lue, NSW 2850</strong> and/or nearby areas.
* </div>*/
HtmlAgilityPack.HtmlNode _node = HtmlUtil.GetNode(_doc.DocumentNode, "div", "id", "zeroResultsMessage");
if (_node != null)
{
_errMsg = _node.InnerText;
_errMsg = _errMsg.Replace("<strong>", "").Replace("</strong>", "");
return(true);
}
if (_node == null)
{
/*
* <div id="fourOFourMessage">
* <h2>
* Sorry we couldn't find that page.<br>
* Perhaps the page doesn't exist or address was mistyped.
* </h2>
* You could try:
* <ul>
* <li>to do a business search in the search fields at the top of the page</li>
* <li>go to the <a href="http://www.yellowpages.com.au">Yellow Pages<sup>®</sup></a> home page</li>
* </ul>
* </div>
*/
_node = HtmlUtil.GetNode(_doc.DocumentNode, "div", "id", "fourOFourMessage");
if (_node != null)
{
var _Ns = _node.Descendants("h2").ToArray();
if (_Ns != null && _Ns.Count() > 0)
{
var _n = _Ns[0];
_errMsg = _n.InnerText;
return(true);
}
}
}
return(false);
}
/// <summary>
///
/// </summary>
/// <param name="node"></param>
/// <param name="element"></param>
/// <param name="attribute"></param>
/// <param name="valuePortion"></param>
/// <returns></returns>
public static string GetInnerText(HtmlAgilityPack.HtmlNode node,
string element,
string attribute,
string valuePortion,
string defaultValue = "")
{
var _node = node.Descendants(element)
.Where(d => d.Attributes.Contains(attribute) && d.Attributes[attribute].Value.Contains(valuePortion))
.FirstOrDefault();
if (_node == null)
{
return(defaultValue);
}
return(_node.InnerText);
}
static public DialogueMissionLocation MissionLocationFromDialogue(this HtmlAgilityPack.HtmlNode node)
{
var SecurityLevelMilli =
node?.Descendants()?.Select(descendant => Number.NumberParseDecimalMilli(descendant?.InnerText?.Trim()))
?.WhereNotDefault()
?.FirstOrDefault();
var NameNode = node?.SelectSingleNode(".//a");
var Name = NameNode?.InnerText?.Trim();
return(new DialogueMissionLocation()
{
Name = Name,
SecurityLevelMilli = (int?)SecurityLevelMilli,
SystemName = Name?.AsLocation()?.SystemName,
});
}
public static TimeSpan GetBuildDuration(HtmlAgilityPack.HtmlNode node, Classificator.ServerVersionEnum version)
{
var duration = node.Descendants("div").FirstOrDefault(x => x.HasClass("duration"));
if (duration != null)
{
switch (version)
{
case Classificator.ServerVersionEnum.T4_4:
return(TimeParser.ParseDuration(duration.InnerText));
case Classificator.ServerVersionEnum.T4_5:
var dur = duration.Descendants("span").FirstOrDefault(x => x.HasClass("value"));
if (dur != null)
{
return(TimeParser.ParseDuration(dur.InnerText));
}
break;
}
}
return(new TimeSpan());
}
/// <summary>
/// Prepare tables
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
private void PrepareTables(HtmlAgilityPack.HtmlDocument wordDoc)
{
List <HtmlAgilityPack.HtmlNode> tables = wordDoc.DocumentNode.Descendants().
Where(node => node.OriginalName.Equals("table")).ToList <HtmlAgilityPack.HtmlNode>();
for (int i = 0; i < tables.Count; i++)
{
HtmlAgilityPack.HtmlNode table = tables[i];
table.Attributes.RemoveAll();
table.SetAttributeValue("border", "1");
table.SetAttributeValue("cellspacing", "0");
table.SetAttributeValue("cellpadding", "0");
table.SetAttributeValue("style", "border-collapse:collapse;border:none;");
table.SetAttributeValue("width", "100%");
List <HtmlAgilityPack.HtmlNode> rows = table.Descendants().
Where(node => node.OriginalName.Equals("tr")).ToList <HtmlAgilityPack.HtmlNode>();
for (int j = 0; j < rows.Count; j++)
{
HtmlAgilityPack.HtmlNode row = rows[j];
row.Attributes.RemoveAll();
List <HtmlAgilityPack.HtmlNode> columns = row.Descendants().
Where(node => node.OriginalName.Equals("td")).ToList <HtmlAgilityPack.HtmlNode>();
for (int k = 0; k < columns.Count; k++)
{
HtmlAgilityPack.HtmlNode column = columns[k];
column.Attributes.RemoveAll();
column.SetAttributeValue("valign", "top");
string width = string.Format("{0}%", 100 / columns.Count);
column.SetAttributeValue("width", width);
}
}
}
}
private bool ItemIsInPriceRange(HtmlAgilityPack.HtmlNode item)
{
string itemPrice = Regex.Match(item.Descendants("li").
Where(node => node.GetAttributeValue("class", "").
Equals("lvprice prc")).FirstOrDefault().
InnerText.Trim(elementsToTrim), @"\d+.\d+").ToString();
if (tbMinPrice.Text != "" && tbMaxPrice.Text != "" && !Regex.Match(tbMaxPrice.Text, "[a-zA-Z]").Success&& !Regex.Match(tbMinPrice.Text, "[a-zA-Z]").Success || Regex.Match(tbMaxPrice.Text, "[$&+,:;=?@#|'<>.^*()%!-]").Success)
{
tbMaxPrice.Text = Regex.Replace(tbMaxPrice.Text, @"\s+", "");
tbMinPrice.Text = Regex.Replace(tbMinPrice.Text, @"\s+", "");
if ((double.Parse(itemPrice) >= Convert.ToInt32(tbMinPrice.Text) && double.Parse(itemPrice) <= Convert.ToInt32(tbMaxPrice.Text)))
{
return(true);
}
else
{
return(false);
}
}
return(true);
}
/// <summary>
///
/// </summary>
/// <param name="htmNode"></param>
/// <returns></returns>
public Models.Advertisement EscrapeAdInfo(HtmlAgilityPack.HtmlNode htmNode)
{
Models.Advertisement _adInfo = null;
if (htmNode != null)
{
try
{
string _businessName = HtmlUtil.GetInnerText(htmNode, "span", "class", "listingTitle");
_businessName = Models.Advertisement.Resolve(_businessName);
string _phone = HtmlUtil.GetInnerText(htmNode, "div", "class", "phoneNumber");
string _fax = "[NA]"; //HtmlUtil.GetInnerText(_n, "div", "class", "phoneNumber");
string _fullAddress = HtmlUtil.GetInnerText(htmNode, "div", "class", "address");
string _streetBlk = string.Empty;
string _locality = string.Empty;
string _region = string.Empty;
string _postalCode = string.Empty;
Parsers.SplitAddresses(_fullAddress, ref _streetBlk, ref _locality, ref _region, ref _postalCode);
HtmlAgilityPack.HtmlNode __n = null;
string _website = "[NA]";
__n = HtmlUtil.GetNode(htmNode, "ul", "class", "ypgListingLinks");
if (__n != null)
{
try
{
__n = HtmlUtil.GetNode(htmNode, "li", "class", "noPrint");
__n = __n.Descendants("a").ToArray()[0];
_website = __n.Attributes["href"].Value.Replace("/gourl/", string.Empty);
}
catch { }
}
string _adLink = "[NA]";
__n = HtmlUtil.GetNode(htmNode, "h3", "class", "listingTitleLine");
if (__n != null)
{
__n = __n.Descendants("a").ToArray()[0];
_adLink = __n.Attributes["href"].Value;
}
//--------------------------------------------------------
_adInfo = new Models.Advertisement()
{
BusinessName = _businessName,
Phone = _phone,
Fax = _fax,
FullAddress = _fullAddress,
StreetBlk = _streetBlk,
Locality = _locality,
Region = _region,
PostalCode = _postalCode,
Website = _website,
AdvertiserLink = string.Format("{0}{1}", this.directoryProviderSetting.BaseUrl, _adLink),
};
}
catch (Exception ex)
{
if (this.frameworkExceptionInvoke != null)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.directoryProviderSetting.ServicedCountry.ToString(), "EscrapeAdInfo"), ex);
this.frameworkExceptionInvoke(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
}
return(_adInfo);
}
public List <HtmlNode> Descendants(string name)
{
return(_htmlNode.Descendants(name)?.Select(node => new HtmlNode(node)).ToList());
}
private void UpdateGridBDDThread()
{
List <string> _gatherableItemNames = new List <string>();
List <string> _gridOKItemNames = new List <string>();
Service_Misc.LogText(_logTextBox, "Updating database...");
MiqoCraftCore.MiqoCraftCore.DownloadGrids();
try
{
List <MiqoItemPage> result = new List <MiqoItemPage>();
CookieCollection logMiqobotCookies = Miqobot.LogInForum();
CookieCollection oCookies = new CookieCollection();
HttpStatusCode oCode = HttpStatusCode.NotFound;
HtmlAgilityPack.HtmlDocument answer = Service_Misc.GetWebPageFromRequest("GET https://miqobot.com/forum/forums/topic/index-gathering-grids/ HTTP/1.1|Host: miqobot.com|Connection: keep-alive|Cache-Control: max-age=0|Upgrade-Insecure-Requests: 1|User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36|Sec-Fetch-Mode: navigate|Sec-Fetch-User: ?1|Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3|Sec-Fetch-Site: same-origin|Referer: https://miqobot.com/forum/forums/forum/grids-and-presets/|Accept-Encoding: gzip, deflate, br|Accept-Language: fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7|Cookie: wordpress_test_cookie=WP+Cookie+check; _ga=GA1.2.1771485810.1566089776||",
logMiqobotCookies, ref oCookies, ref oCode);
if (null == answer)
{
Service_Misc.LogText(_logTextBox, "Failed to update database. No answer from miqobot forum.");
return;
}
HtmlAgilityPack.HtmlNode firstAnswerNode = Service_Misc.GetFirstChildNode(answer.DocumentNode, "div", "topic-tag-gathering");
List <HtmlAgilityPack.HtmlNode> listItemNodes = firstAnswerNode.Descendants("li").ToList();
DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath());
DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid"));
if (!cacheDirectory.Exists)
{
Service_Misc.LogText(_logTextBox, "Failed to compute database status, CacheGrid directory does not exist.");
return;
}
foreach (HtmlAgilityPack.HtmlNode node in listItemNodes)
{
if (null == node)
{
continue;
}
string nodeInnerTextLower = node.InnerText.ToLower();
if (nodeInnerTextLower.Contains("Mining Your Own Business".ToLower()))
{
break;
}
if (nodeInnerTextLower.Contains("lv."))
{
//Found node !
string level = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[0].Trim();
string itemName = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[1].Trim().Replace("(hidden)", "").Trim();
_gatherableItemNames.Add(itemName);
if (File.Exists(Path.Combine(cacheDirectory.FullName, itemName + " Grid.txt")))
{
_gridOKItemNames.Add(itemName);
}
}
}
lock (_locker)
{
MiqoCraftOptions options = new MiqoCraftOptions();
options.Load(OptionLocation.GlobalOption);
options.ListGatherableItems = _gatherableItemNames;
options.ListGridOKItems = _gridOKItemNames;
options.Save();
}
UpdateBDDStatusFromOptions();
{
Service_Misc.LogText(_logTextBox, "Database updated !");
return;
}
}
catch
{
Service_Misc.LogText(_logTextBox, "Failed to compute database status, CacheGrid directory does not exist.");
return;
}
}
public static IEnumerable <HtmlAgilityPack.HtmlNode> DescendantsElements(this HtmlAgilityPack.HtmlNode source, string name = null)
{
return(source.Descendants().Where((i) => i.NodeType == HtmlAgilityPack.HtmlNodeType.Element && (name != null ? i.Name == name : true)));
}
protected override string GetPosition(HtmlAgilityPack.HtmlNode row)
{
return(row.Descendants("a").Where(
r => r.Attributes.Contains("class") && r.Attributes["class"].Value.Contains("title"))
.Select(r => r.InnerText).SingleOrDefault());
}
public static IEnumerable <HtmlAgilityPack.HtmlNode> DescendantsC(this HtmlAgilityPack.HtmlNode node, string clas)
{
return(node.Descendants().Where((HtmlAgilityPack.HtmlNode h) => h.HasClass(clas)));
}
private void DownloadMissingItemGrids()
{
List <MiqoItemPage> result = new List <MiqoItemPage>();
CookieCollection logMiqobotCookies = Miqobot.LogInForum();
CookieCollection oCookies = new CookieCollection();
HttpStatusCode oCode = HttpStatusCode.NotFound;
HtmlAgilityPack.HtmlDocument answer = Service_Misc.GetWebPageFromRequest("GET https://miqobot.com/forum/forums/topic/index-gathering-grids/ HTTP/1.1|Host: miqobot.com|Connection: keep-alive|Cache-Control: max-age=0|Upgrade-Insecure-Requests: 1|User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36|Sec-Fetch-Mode: navigate|Sec-Fetch-User: ?1|Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3|Sec-Fetch-Site: same-origin|Referer: https://miqobot.com/forum/forums/forum/grids-and-presets/|Accept-Encoding: gzip, deflate, br|Accept-Language: fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7|Cookie: wordpress_test_cookie=WP+Cookie+check; _ga=GA1.2.1771485810.1566089776||",
logMiqobotCookies, ref oCookies, ref oCode);
if (null == answer)
{
VPThreading.SetText(_progressLabel, "Failed to update database. No answer from miqobot forum.");
return;
}
HtmlAgilityPack.HtmlNode firstAnswerNode = Service_Misc.GetFirstChildNode(answer.DocumentNode, "div", "topic-tag-gathering");
List <HtmlAgilityPack.HtmlNode> listItemNodes = firstAnswerNode.Descendants("li").ToList();
DirectoryInfo exeDirectory = new DirectoryInfo(Service_Misc.GetExecutionPath());
DirectoryInfo cacheDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "CacheGrid"));
DirectoryInfo analyzeDirectory = new DirectoryInfo(Path.Combine(exeDirectory.FullName, "DownloadedGrids"));
if (!analyzeDirectory.Exists)
{
analyzeDirectory.Create();
}
if (!cacheDirectory.Exists)
{
VPThreading.SetText(_progressLabel, "Failed to compute database status, CacheGrid directory does not exist.");
return;
}
int indexProg = 0;
foreach (HtmlAgilityPack.HtmlNode node in listItemNodes)
{
indexProg++;
VPThreading.SetProgress(_progressBar, indexProg * 100 / listItemNodes.Count);
if (null == node)
{
continue;
}
string nodeInnerTextLower = node.InnerText.ToLower();
if (nodeInnerTextLower.Contains("Mining Your Own Business".ToLower()))
{
break;
}
if (nodeInnerTextLower.Contains("lv."))
{
//Found node !
string level = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[0].Trim();
string itemName = nodeInnerTextLower.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries)[1].Trim().Replace("(hidden)", "").Trim();
//Check if grid exist
string gridItemName = itemName + " Grid";
//Looking into cache directory
FileInfo cacheGridFile = new FileInfo(Path.Combine(cacheDirectory.FullName, gridItemName + ".txt"));
if (cacheGridFile.Exists)
{
VPThreading.SetText(_progressLabel, "Item already has a grid.");
continue;
}
VPThreading.SetText(_progressLabel, "Trying to find a new grid : " + itemName);
//Looking for all links
List <MiqoItemPage> listLinks = Miqobot.GetURLItem(itemName, logMiqobotCookies, answer);
foreach (MiqoItemPage page in listLinks)
{
//List<string> ListGrids = Miqobot.GetAllGridsFromForum(itemName, logMiqobotCookies, page);
//if (null == ListGrids) continue;
//foreach(string grid in ListGrids)
//{
// try
// {
// string pathGrid = Path.Combine(analyzeDirectory.FullName, gridItemName + "---" + gridIndex + ".txt");
// if (File.Exists(pathGrid)) File.Delete(pathGrid);
// File.WriteAllText(pathGrid, grid);
// gridIndex++;
// }
// catch
// {
// }
//}
DownloadFromURL(page.URL, itemName, logMiqobotCookies);
}
}
}
VPThreading.SetProgress(_progressBar, 100);
VPThreading.SetText(_progressLabel, "Done!");
}
/// <summary>
///
/// </summary>
/// <param name="adInf"></param>
/// <returns></returns>
public Models.Advertisement EscrapeAdInfoExtend(Models.Advertisement adInf)
{
if (adInf != null && !string.IsNullOrEmpty(adInf.AdvertiserLink))
{
try
{
string _htmlDoc = HtmlUtil.GetPageDocument(adInf.AdvertiserLink);
HtmlAgilityPack.HtmlDocument htmDocAg = new HtmlAgilityPack.HtmlDocument();
htmDocAg.LoadHtml(_htmlDoc);
string _googMap = "[NA]";
string _keywords = "[NA]";
var _n = HtmlUtil.GetNode(htmDocAg.DocumentNode, "meta", "name", "keywords");
if (_n != null)
{
_keywords = _n.Attributes["content"].Value;
}
string _description = "[NA]";
HtmlAgilityPack.HtmlNode _metaDesc = HtmlUtil.GetNode(htmDocAg.DocumentNode, "meta", "name", "description");
if (_metaDesc != null)
{
_description = _metaDesc.Attributes["content"].Value;
}
string _emailAdd = "[NA]";
/*
* <a id="mainEmailAddressLink" class="emailBusinessLink" rel="nofollow" href="/onlineSolution_emailBusiness.do?listingId=14074960&classification=MAIN&context=businessTypeSearch&referredBy=YOL" title="Contact Turner Freeman Lawyers">
* <img class="emailAddressIcon" src="/ui/standard/bpp/email_icon.png" alt="Main Email Address">
* <span>[email protected]</span>
* </a>
*/
HtmlAgilityPack.HtmlNode _emailAdNode = HtmlUtil.GetNode(htmDocAg.DocumentNode, "a", "id", "mainEmailAddressLink");
if (_emailAdNode != null)
{
_emailAdd = _emailAdNode.Descendants("span").ToArray()[0].InnerText;
}
string _locations = "[NA]";
//string _dateAdded = "[NA]";
adInf.GoogleMap = _googMap;
adInf.Keywords = _keywords;
adInf.Description = _description;
adInf.EmailAddress = _emailAdd;
adInf.Locations = _locations;
//adInf.DateAdded = _dateAdded;
}
catch (Exception ex)
{
if (this.frameworkExceptionInvoke != null)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.directoryProviderSetting.ServicedCountry.ToString(), "EscrapeAdInfoExtend"), ex);
this.frameworkExceptionInvoke(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
}
return(adInf);
}
public static HtmlAgilityPack.HtmlNode ElementC(this HtmlAgilityPack.HtmlNode node, string clas, int layerdepth = 1)
{
return(node.Descendants(layerdepth).First((HtmlAgilityPack.HtmlNode h) => h.HasClass(clas)));
}
/// <summary>
///
/// </summary>
/// <param name="adInf"></param>
/// <returns></returns>
public Models.Advertisement EscrapeAdInfoExtend(Models.Advertisement adInf)
{
if (adInf != null && !string.IsNullOrEmpty(adInf.AdvertiserLink))
{
try
{
string _htmlDoc = HtmlUtil.GetPageDocument(adInf.AdvertiserLink);
HtmlAgilityPack.HtmlDocument htmDocAg = new HtmlAgilityPack.HtmlDocument();
htmDocAg.LoadHtml(_htmlDoc);
string _latitude = "[NA]";
string _longitude = "[NA]";
HtmlAgilityPack.HtmlNode _mapDatNode = HtmlUtil.GetNode(htmDocAg.DocumentNode, "div", "id", "ypgMapContainer");
if (_mapDatNode != null)
{ //latLong = new VELatLong(43.8087172232, -79.5469648855); map.CreateAndLoadMap
string _mapDat = _mapDatNode.InnerText;
int _mrkrStart = _mapDat.IndexOf("VELatLong", 0);
int _mrkrEnd = _mapDat.IndexOf("map.CreateAndLoadMap", 0);
_mapDat = _mapDat.Substring(_mrkrStart, _mrkrEnd - _mrkrStart);
_mapDat = _mapDat
.Replace("VELatLong(", "")
.Replace("map.CreateAndLoadMap", "")
.Replace(");", "")
.Trim();
string[] _coords = _mapDat.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
_latitude = _coords[0];
_longitude = _coords[1];
}
string _googMap = "[NA]";
string _keywords = "[NA]";
HtmlAgilityPack.HtmlNode _metaKeywords = HtmlUtil.GetNode(htmDocAg.DocumentNode, "meta", "name", "keywords");
if (_metaKeywords != null)
{
_keywords = _metaKeywords.Attributes["content"].Value;
}
string _description = "[NA]";
HtmlAgilityPack.HtmlNode _metaDesc = HtmlUtil.GetNode(htmDocAg.DocumentNode, "meta", "name", "description");
if (_metaDesc != null)
{
_description = _metaDesc.Attributes["content"].Value;
}
string _rating = "[NA]";
string _emailAdd = "[NA]";
HtmlAgilityPack.HtmlNode _emailAdNode = HtmlUtil.GetNode(htmDocAg.DocumentNode, "div", "class", "busCardLeftLinks");
if (_emailAdNode != null)
{
try
{
_emailAdd = _emailAdNode.Descendants("a").ToArray()[0].Attributes["content"].Value;
}
catch { }
}
string _locations = "[NA]";
//string _dateAdded = "[NA]";
adInf.Latitude = _latitude;
adInf.Longtitude = _longitude;
adInf.GoogleMap = _googMap;
adInf.Keywords = _keywords;
adInf.Description = _description;
adInf.Rating = _rating;
adInf.EmailAddress = _emailAdd;
adInf.Locations = _locations;
//adInf.DateAdded = _dateAdded;
}
catch (Exception ex)
{
if (this.frameworkExceptionInvoke != null)
{
Exception _ex = new Exception(string.Format("Exception in {0}.{1}(?)", this.directoryProviderSetting.ServicedCountry.ToString(), "EscrapeAdInfoExtend"), ex);
this.frameworkExceptionInvoke(new Handlers.EventHandlers.FrameworkExceptionEventArgs(_ex));
}
}
}
return(adInf);
}
