private void ValidateTable(ValidationEventArgs e, HtmlAgilityPack.HtmlNode table)
{
var columns = table.SelectNodes("//th");
if (columns == null || columns.Count == 0)
{
Fail(e, "Could not locate any columns in the table!");
}
else
{
int columnIndex = FindColumnIndexByName(columns, ColumnName);
if (columnIndex == -1)
{
Fail(e, String.Format("Could not find a column named '{0}'!", ColumnName));
}
else
{
bool foundTheValue = DoesValueExistInColumn(table, columnIndex, ExpectedValue);
if (foundTheValue == true)
{
Pass(e, "Found the column value!");
}
else
{
Fail(e, String.Format("Could not find a cell with the value '{0}'", ExpectedValue));
}
}
}
}
/// <summary>
/// Uzupełnienie komentarza danymi zalet
/// </summary>
/// <param name="comment"></param>
/// <param name="node"></param>
private void fillAdvantages(CommentDb comment, HtmlAgilityPack.HtmlNode node)
{
HtmlAgilityPack.HtmlNodeCollection bodyNodes = node.SelectNodes(containerPath + "ul[@class=\"pros-n-cons\"]//li[1]");
if (bodyNodes != null)
{
foreach (HtmlAgilityPack.HtmlNode commentNode in bodyNodes)
{
comment.Advantages += commentNode.InnerText;
}
}
}
/// <summary>
/// Uzupełnienie komentarza danymi zalet
/// </summary>
/// <param name="comment"></param>
/// <param name="node"></param>
private void fillAdvantages(CommentDb comment, HtmlAgilityPack.HtmlNode node)
{
HtmlAgilityPack.HtmlNodeCollection bodyNodes = node.SelectNodes(".//span[@class=\"pros-cell\"]//li");//("//*[@id=\"body\"]/div[2]/div/div/div[2]/div[3]/div[2]/ol/li/div/div[1]/p");// //body//div[@id='body']class=\"product - review - body\"");
if (bodyNodes != null)
{
foreach (HtmlAgilityPack.HtmlNode commentNode in bodyNodes)
{
comment.Advantages += commentNode.InnerText;
}
}
}
/// <summary>
/// Extracts the episode information out of a episode node from ShowRSS.info
/// </summary>
/// <param name="node">The node containing the episode</param>
/// <returns>An <see cref="Episode"/> with the information from the html node</returns>
public static Episode Extract(HtmlAgilityPack.HtmlNode node)
{
var s = node.SelectNodes("a[@href]").First();
string seriestitle = s.ChildNodes[0].InnerHtml;
string episodetitle = s.ChildNodes[1].InnerHtml;
string datestring = node.ParentNode.PreviousSibling.InnerHtml;
string datestringcrop = datestring.Substring(0, datestring.Length - 1);
DateTime date = DateTime.Parse(datestringcrop, CultureInfo.CreateSpecificCulture("en-US"));
return new Episode(seriestitle, episodetitle, date);
}
private Vehicle(HtmlAgilityPack.HtmlNode htmlData)
{
// parse the html node to get the info out
string licensePlate = htmlData.SelectSingleNode(".//input[@type='text']/@value")?.GetAttributeValue("value", "");
string vehicleTypeVal = htmlData.SelectSingleNode(".//select/option[@selected='selected']/@value")?.GetAttributeValue("value", "");
var hiddenData = htmlData.SelectNodes(".//input[@type='hidden']");
var hiddenId = hiddenData.Where(input => input.Attributes["name"]?.Value.Contains("VehicleUid") == true)?.First()?.GetAttributeValue("value", "");
var hiddenLicPlate = hiddenData.Where(input => input.Attributes["name"]?.Value.Contains("LicensePlateHiddenField") == true)?.First()?.GetAttributeValue("value", "");
_myType = (VehicleType)int.Parse(vehicleTypeVal);
_myLicensePlate = licensePlate;
_myHiddenLicensePlate = hiddenLicPlate;
_myVehicleId = hiddenId;
WebData = new VehicleWebData(htmlData);
_isFromWebsite = true;
}
public override List<dynamic> SelectList(HtmlAgilityPack.HtmlNode element)
{
List<dynamic> result = new List<dynamic>();
var nodes = element.SelectNodes(_xpath);
if (nodes != null)
{
foreach (var node in nodes)
{
if (!HasAttribute())
{
result.Add(node);
}
else
{
var attr = node.Attributes[_attribute];
if (attr != null)
{
result.Add(attr.Value?.Trim());
}
}
}
}
return result;
}
/// <summary>
/// Uzupełnienie komentarza danymi ilości gwiazdek
/// </summary>
/// <param name="comment"></param>
/// <param name="node"></param>
private void fillStar(CommentDb comment, HtmlAgilityPack.HtmlNode node)
{
HtmlAgilityPack.HtmlNodeCollection bodyNodes = node.SelectNodes(".//span[@class=\"review-score-count\"]");//("//*[@id=\"body\"]/div[2]/div/div/div[2]/div[3]/div[2]/ol/li/div/div[1]/p");// //body//div[@id='body']class=\"product - review - body\"");
if (bodyNodes != null)
{
foreach (HtmlAgilityPack.HtmlNode commentNode in bodyNodes)
{
string score = commentNode.InnerText;
string[] scoreCount = score.Split('/');
comment.Stars = Convert.ToDouble(scoreCount[0]);
}
}
}
/// <summary>
/// Uzupełnia tekst komentarza
/// </summary>
/// <param name="comment"></param>
/// <param name="node"></param>
private void fillCommentContent(CommentDb comment, HtmlAgilityPack.HtmlNode node)
{
HtmlNodeCollection bodyNodes = node.SelectNodes(".//div[@class=\"opinion-container\"]//p");//("//*[@id=\"body\"]/div[2]/div/div/div[2]/div[3]/div[2]/ol/li/div/div[1]/p");// //body//div[@id='body']class=\"product - review - body\"");
if (bodyNodes != null)
{
foreach (HtmlAgilityPack.HtmlNode commentNode in bodyNodes)
{
comment.Comment += commentNode.InnerText;
}
}
}
protected virtual HtmlAgilityPack.HtmlNodeCollection GetDishInfoList(HtmlAgilityPack.HtmlNode dishTypeNode)
{
var dishesPath = DishesPath();
if (string.IsNullOrEmpty(dishesPath))
{
return null;
}
return dishTypeNode.SelectNodes(dishesPath);
}
private BasicInfo ProcessGameData(HtmlAgilityPack.HtmlNode game, DateTime gameDate, string webID)
{
// 建立比賽資料，時間以台灣為主
BasicInfo gameInfo = new BasicInfo(this.AllianceID, this.GameType, gameDate, webID);
gameInfo.Away = game.SelectSingleNode("div/div[@class='team visitor']/div[@class='team-capsule']/p[@class='team-name']/span/a").InnerText;
gameInfo.Home = game.SelectSingleNode("div/div[@class='team home']/div[@class='team-capsule']/p[@class='team-name']/span/a").InnerText;
gameInfo.Status = game.SelectSingleNode("div/div[@class='game-header']/div/p").InnerText;
gameInfo.Status = gameInfo.Status.ToLower();
// 取代文字
gameInfo.Status = gameInfo.Status.Replace("halftime", "中場休息");
gameInfo.Status = gameInfo.Status.Replace("half", "中場休息");
gameInfo.Status = gameInfo.Status.Replace("qtr", "");
gameInfo.Status = gameInfo.Status.Replace("1st", "");
gameInfo.Status = gameInfo.Status.Replace("2nd", "");
gameInfo.Status = gameInfo.Status.Replace("3rd", "");
gameInfo.Status = gameInfo.Status.Replace("4th", "");
gameInfo.Status = gameInfo.Status.Replace("ot", "");
gameInfo.Status = gameInfo.Status.Replace("2ot", "");
gameInfo.Status = gameInfo.Status.Replace("of", "");
gameInfo.Status = gameInfo.Status.Replace("end", "結束");
gameInfo.Status = gameInfo.Status.Replace(" ", "");//移除空白字元
gameInfo.Status = gameInfo.Status.Trim();
if (gameInfo.Status == "00:00") gameInfo.Status = "結束";
//目前有顯示的分數欄位
HtmlAgilityPack.HtmlNode scoreHeader = game.SelectSingleNode("div/div[@class='game-header']/ul[@class='score header']");
if (scoreHeader != null)//是否已經有比分
{
string linescoreHeader = game.SelectSingleNode("div/div[@class='game-header']/ul[@class='score header']").InnerText;
if (linescoreHeader.Length > 7)//資料不完整，缺乏前面局數比分 2 3 4 OT OT T
{
//第1個數字才是有顯示的比分
int lossInning = 1;
int.TryParse(linescoreHeader.Substring(0, 1), out lossInning);
if (lossInning > 1)
{
lossInning--;
GetGameScore(gameInfo, lossInning);//無暫存取得遺漏比分
}
}
List<int> FullAwayBoard = new List<int>();//完整比分
List<int> FullHomeBoard = new List<int>();//完整比分
if (this.DownScore.ContainsKey(webID) && !string.IsNullOrEmpty(this.DownScore[webID].Data))
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(this.DownScore[webID].Data);
HtmlAgilityPack.HtmlNodeCollection HomeAway = doc.GetElementbyId("gamepackageTop").SelectNodes("div[@class='line-score clear']/div[@class='line-score-container']/table/tr");
if (HomeAway != null && HomeAway.Count == 3)
{
foreach (HtmlAgilityPack.HtmlNode scores in HomeAway)
{
if (scores.Attributes["class"] != null)//忽略periods
continue;
HtmlAgilityPack.HtmlNodeCollection score = scores.SelectNodes("td");
if (score != null)
{
bool isAway = false;
if (FullAwayBoard.Count == 0)
isAway = true;
int period = score.Count - 1;
for (int i = 1; i < period; i++)//第0個值為隊名
{
string tmpNum = score[i].InnerHtml.Replace("\n", "").Replace("\t", "").Trim();
int tryInt = 0;
int.TryParse(tmpNum, out tryInt);
if (isAway)
FullAwayBoard.Add(tryInt);
else
FullHomeBoard.Add(tryInt);
}
}
}
}
}
//首頁比分計算
foreach (HtmlAgilityPack.HtmlNode numDoc in game.SelectNodes("div/div[@class='team visitor']/ul[@class='score']/li"))
{
int num = 0;
if (int.TryParse(numDoc.InnerText, out num))
{
if (gameInfo.AwayBoard.Count < 4)//沒有OT
gameInfo.AwayBoard.Add(numDoc.InnerText);
else
{
int otDiff = FullAwayBoard.Count - gameInfo.AwayBoard.Count;//OT數差別
if (FullAwayBoard.Count == 0 || otDiff == 0)
gameInfo.AwayBoard.Add(numDoc.InnerText);
else
{
int sum = 0;
for (int i = 0; i < otDiff - 1; i++)
{
sum += FullAwayBoard[4 + i];//最後一個OT不累加
gameInfo.AwayBoard.Add(FullAwayBoard[4 + i].ToString());//OT1 OT2 OT3 ...
}
gameInfo.AwayBoard.Add((num - sum).ToString());//剩下的OT分數
}
}
}
}
foreach (HtmlAgilityPack.HtmlNode numDoc in game.SelectNodes("div/div[@class='team home']/ul[@class='score']/li"))
{
int num = 0;
if (int.TryParse(numDoc.InnerText, out num))
{
if (gameInfo.HomeBoard.Count < 4)//沒有OT
gameInfo.HomeBoard.Add(numDoc.InnerText);
else
{
int otDiff = FullHomeBoard.Count - gameInfo.HomeBoard.Count;//OT數差別
if (FullHomeBoard.Count == 0 || otDiff == 0)//尚未取得完整比分資料 或者是最後一個比分則為總分
gameInfo.HomeBoard.Add(numDoc.InnerText);
else
{
int sum = 0;
for (int i = 0; i < otDiff - 1; i++)
{
sum += FullHomeBoard[4 + i];//最後一個OT不累加
gameInfo.HomeBoard.Add(FullHomeBoard[4 + i].ToString());//OT1 OT2 OT3 ...
}
gameInfo.HomeBoard.Add((num - sum).ToString());//剩下的OT分數
}
}
}
}
// 總分
if (gameInfo.AwayBoard.Count > 0)
{
gameInfo.AwayPoint = gameInfo.AwayBoard[gameInfo.AwayBoard.Count - 1];
gameInfo.AwayBoard.RemoveAt(gameInfo.AwayBoard.Count - 1);
gameInfo.HomePoint = gameInfo.HomeBoard[gameInfo.HomeBoard.Count - 1];
gameInfo.HomeBoard.RemoveAt(gameInfo.HomeBoard.Count - 1);
}
}
if (gameInfo.Status.IndexOf("final") != -1)
{
gameInfo.GameStates = "E";
}
else if (gameInfo.Quarter > 0)
{
gameInfo.GameStates = "S";
}
return gameInfo;
}
public VehicleWebData(HtmlAgilityPack.HtmlNode vehicleData)
{
var allInputs = vehicleData.SelectNodes(".//input");
var select = vehicleData.SelectSingleNode(".//select");
LicensePlateTextBox = allInputs.Where(input => input.Attributes["name"]?.Value?.Contains("LicensePlateTextBox") == true).First()?.GetAttributeValue("name", "");
LicensePlateHiddenField = allInputs.Where(input => input.Attributes["name"]?.Value?.Contains("LicensePlateHiddenField") == true).First()?.GetAttributeValue("name", "");
VehicleUidHiddenField = allInputs.Where(input => input.Attributes["name"]?.Value?.Contains("VehicleUidHiddenField") == true).First()?.GetAttributeValue("name", "");
VehicleTypeHiddenField = allInputs.Where(input => input.Attributes["name"]?.Value?.Contains("VehicleTypeUidHiddenField") == true).First()?.GetAttributeValue("name", "");
VehicleTypeDropDown = select?.GetAttributeValue("name", "");
}
private void GetPagesRecursive(HtmlAgilityPack.HtmlNode document, IDictionary<int, string> pages)
{
var chapterPages = document.SelectNodes(@"//div[@class=""main-body""]//div[@class=""btn-group""][2]/ul[@class=""dropdown-menu""]/li/a");
if (chapterPages == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
int addedCount = 0;
foreach (var pageLink in chapterPages)
{
int pageNumber = 0;
var url = GetFullUrl(pageLink.Attributes["href"].Value);
if (pages.Any(kvp => kvp.Value == url)) // skip duplicate urls
continue;
if (!Int32.TryParse(Regex.Match(pageLink.InnerText, @"\d+").Value, out pageNumber))
_log.Error("Unable to parse page number '" + pageLink.InnerText + "'");
if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead
pageNumber = Random;
pages.Add(pageNumber, url);
addedCount++;
}
if (addedCount > 0)
{
var pageRecord = pages.OrderByDescending(kvp => kvp.Key).Skip(1).FirstOrDefault();
if (pageRecord.Equals(default(KeyValuePair<int, string>)))
return;
var nextDocument = WebHelper.GetHtmlDocument(pageRecord.Value);
GetPagesRecursive(nextDocument, pages);
}
}
public ResultType_enum FillCustomer(Customer customer, HtmlAgilityPack.HtmlNode customerNode, InternetRequestEngine internetRequestEngine,
out string message)
{
try
{
message = "";
HtmlAgilityPack.HtmlNode nodeTmp = null;
#region Определение параметров
text = ".//table";
//text += "/tbody"; !!!newBrowser
text += "/tr";
text += "/td[@class=\"descriptTenderTd\"]";
text += "/dl";
text += "/dt";
text += "/a";
nodeTmp = customerNode.SelectSingleNode(text);
// Название заказчика
if (nodeTmp.Attributes.Contains("title"))
customer.Name = Globals.DecodeInternetSymbs(nodeTmp.Attributes["title"].Value.Trim());
// Определение ID 94 и 223 законов
// Есди существует атрибут <href>, значит указана одна ссылка на один закон
LawType_enum lawType = LawType_enum.None;
Regex regex = new Regex(@"http://.*?Id=\d*");
MatchCollection matchColl = null;
if (nodeTmp.Attributes.Contains("href"))
{
matchColl = regex.Matches(nodeTmp.Attributes["href"].Value);
}
else
{
matchColl = regex.Matches(nodeTmp.Attributes["onclick"].Value);
}
foreach (Match match in matchColl)
{
string lawID = GetIDLaw(match.Value, out lawType);
switch (lawType)
{
case (LawType_enum._44_94):
customer.Law_44_94_ID = lawID;
break;
case (LawType_enum._223):
customer.Law_223_ID = lawID;
break;
default:
break;
}
//break;
}
// Уровень организации
text = ".//table";
//text += "/tbody"; !!!newBrowser
text += "/tr";
text += "/td[@class=\"tenderTd\"]";
text += "/dl";
text += "/dt";
nodeTmp = customerNode.SelectSingleNode(text);
CustomerLevel_enum customerLevel = CustomerLevel_enum.None;
if (nodeTmp != null) {
switch(nodeTmp.InnerText.ToLower().Trim()) {
case ("федеральный уровень"):
customerLevel = CustomerLevel_enum.Federal;
break;
case ("уровень субъекта рф"):
customerLevel = CustomerLevel_enum.Subject;
break;
case ("муниципальный уровень"):
customerLevel = CustomerLevel_enum.Municipal;
break;
case ("иное"):
customerLevel = CustomerLevel_enum.Other;
break;
}
}
customer.CustomerLevel = mvm.wc.CustomerLevels.FirstOrDefault(m => m.CustomerLevelCode == customerLevel.ToString());
// ИНН (VATIN)
text = ".//table";
//text += "/tbody"; !!!newBrowser
text += "/tr";
text += "/td[@class=\"descriptTenderTd\"]";
text += "/dl";
text += "/dd[@class=\"nameOrganization\"]";
text += "/span";
HtmlAgilityPack.HtmlNodeCollection collectionVatin = customerNode.SelectNodes(text);
foreach (HtmlAgilityPack.HtmlNode nodeVatin in collectionVatin)
{
int vatinIntPos = nodeVatin.InnerText.IndexOf("ИНН:");
if (vatinIntPos > -1)
{
customer.Vatin = nodeVatin.InnerText.Substring(vatinIntPos + 4, nodeVatin.InnerText.Length - (vatinIntPos + 4)).Trim();
break;
}
}
// Адрес
text = ".//table";
//text += "/tbody"; !!!newBrowser
text += "/tr";
text += "/td[@class=\"descriptTenderTd\"]";
text += "/dl";
text += "/dd";
HtmlAgilityPack.HtmlNodeCollection adressDDColl = customerNode.SelectNodes(text);
if (adressDDColl.Count == 3)
{
customer.Address = Globals.CutAddress(adressDDColl[2].InnerText.Trim());
}
#endregion
return ResultType_enum.Done;
}
catch (Exception ex)
{
message = ex.Message + "\n\n" + ex.StackTrace;
return ResultType_enum.Error;
}
}
/// <summary>
/// 指定リストよりHtmlNodeCollectionを取得します
/// </summary>
/// <param name="top">HtmlNode</param>
/// <returns>HtmlNodeCollection</returns>
private HtmlAgilityPack.HtmlNodeCollection GetHtmlTargetCollection(List<HtmlEachType> target, HtmlAgilityPack.HtmlNode top)
{
string searchString = "/";
foreach (HtmlEachType eachType in target)
{
searchString += "/";
if (eachType.CategoryString == string.Empty)
{
searchString += eachType.AttributeString;
}
else
{
searchString += eachType.AttributeString + "[@" + eachType.CategoryString + @"=""" + eachType.KeyString + @"""]";
}
}
return top.SelectNodes(searchString);
}
private object Extract(string name, ConfigSection config, HtmlAgilityPack.HtmlNode parentNode, List<HtmlAgilityPack.HtmlNode> logicalParents)
{
this.RemoveUnwantedTags(config, parentNode);
// We will try to extract text for this item because it does not have children
var containers = new JArray();
if (config.XPathRules != null && config.XPathRules.Count > 0)
{
foreach (var xpath in config.XPathRules)
{
// TODO: Add try catch Exception
var nodes = parentNode.SelectNodes(xpath);
if (nodes != null && nodes.Count > 0)
{
var newLogicalParents = logicalParents.GetRange(0, logicalParents.Count);
newLogicalParents.Add(parentNode);
foreach (var node in nodes)
{
if (config.Children != null && config.Children.Count > 0)
{
var container = new JObject();
this.ExtractChildren(config: config, parentNode: node, container: container, logicalParents: newLogicalParents);
containers.Add(container);
}
else if (config.Transformations != null && config.Transformations.Count > 0)
{
var obj = this.RunTransformations(config.Transformations, node, newLogicalParents);
if (obj != null)
{
containers.Add(obj);
}
}
else if (node.InnerText != null)
{
containers.Add(HtmlEntity.DeEntitize(node.InnerText).Trim());
}
}
}
}
}
else
{
var container = new JObject();
this.ExtractChildren(config: config, parentNode: parentNode, container: container, logicalParents: logicalParents);
containers.Add(container);
}
if (!config.ForceArray && containers.Count == 0)
{
return new JObject();
}
else if (!config.ForceArray && containers.Count == 1)
{
return containers.First;
}
else
{
return containers;
}
}
