|
public SelectSingleNode ( string xpath ) : |
||
| xpath | string | The XPath expression. May not be null. |
| return | ||
public Segment GetSegment(HtmlNode segmentDiv, string day)
{
var imgDiv = segmentDiv.SelectSingleNode("./div[@class='programListContentImgArea']");
var txtDiv = segmentDiv.SelectSingleNode("./div[@class='programListContentTextArea scheduleColumnTextArea']");
var anchor = txtDiv.GetChildById("hypProgramTxt");
var authorNode = txtDiv.GetChildById("lblAuthor");
var imageNode = imgDiv.SelectSingleNode(".//img");
//TODO: Move url building (below) into UrlBuilder class
var segment = new Segment
{
DayDescription = day,
Title = anchor.DecodeHtml(),
SegmentDetailUrl = configurationManager.BookTvDomain + anchor.GetAttributeValue("href", ""),
Time = txtDiv.GetChildById("lblAirTime").DecodeHtml(),
Duration = txtDiv.GetChildById("lblLength").DecodeHtml(),
Author = (authorNode == null) ? "" : authorNode.DecodeHtml(),
ImageUrl = configurationManager.BookTvDomain + imageNode.Attributes["src"].Value,
};
segment.DurationInMinutes = durationParser.GetDurationInMinutes(segment.Duration);
segment.Date = dayParser.Parse(segment.DayDescription);
return segment;
}
private ProductBasicData BuildProductBasicData(HtmlNode productNode)
{
log.DebugFormat("[BuildProductBasicData] OuterHtml= {0}.", productNode.OuterHtml);
//throw new NotImplementedException();
ProductBasicData returnValue = new ProductBasicData();
//should be transfered to barcode, but last digitals of the barcode is productId
returnValue.Barcode = productNode.Attributes.First(c => c.OriginalName == "DdPiD").Value;
returnValue.ProductId = productNode.Attributes.First(c => c.OriginalName == "DdPiD").Value;
returnValue.pbcatid = productNode.Attributes.First(c => c.OriginalName == "pbcatid").Value;
returnValue.qty = productNode.Attributes.First(c => c.OriginalName == "qty").Value;
returnValue.iq = productNode.Attributes.First(c => c.OriginalName == "iq").Value;
returnValue.inb = productNode.Attributes.First(c => c.OriginalName == "inb").Value;
//returnValue.ImageSource = productNode.SelectSingleNode("//img[@src]").Attributes.FirstOrDefault(c=> c.Name == "src").Value;
returnValue.ImageSource = productNode.SelectNodes("child::*/child::div/child::img").First().Attributes.First(c => c.Name == "src").Value;
returnValue.EffectivePrice = productNode.SelectSingleNode("child::*/child::div/child::div/child::div/child::span").InnerText;
//returnValue.EffectivePrice = productNode.SelectSingleNode("//span[@id='spnEffectivePrice']").InnerText;
//returnValue.Description = HttpUtility.HtmlDecode(productNode.SelectSingleNode("//div[@class='ProdBoxSupplierText']").InnerText);
returnValue.Description = HttpUtility.HtmlDecode(productNode.SelectSingleNode("child::*/child::div/child::div/child::a").InnerText);
returnValue.ProductName = HttpUtility.HtmlDecode(productNode.SelectSingleNode("child::*/child::div/child::div/child::a").InnerText);
log.DebugFormat("[BuildProductBasicData] fetched product={0}.", returnValue.ToString());
return returnValue;
}
private static void AddPackage(SteamApp app, HtmlNode packageNode)
{
var package = app.AddNewPackage();
var packageTitleNode = packageNode.SelectSingleNode($"//{PackageTitle}");
package.Title = packageTitleNode.InnerHtml.Replace("Buy ", "").Trim();
var priceNodes = packageNode.SelectNodes($"//div[@class='{PackagePriceXPath}']");
if (priceNodes != null)
{
var priceNode = priceNodes[0];
package.CurrentPrice = ParseNodeWithCurrencyToDecimal(priceNode);
package.OriginalPrice = package.CurrentPrice;
}
else
{
var originalPriceNode = packageNode.SelectSingleNode($"//div[@class='{PackageOriginalPriceXPath}']");
package.OriginalPrice = ParseNodeWithCurrencyToDecimal(originalPriceNode);
var discountPriceNode = packageNode.SelectSingleNode($"//div[@class='{PackageDiscountPriceXPath}']");
package.CurrentPrice = ParseNodeWithCurrencyToDecimal(discountPriceNode);
}
}
public Trade(HtmlNode code)
{
htmlCode = code;
// fetch the trade ID (well, actually its the url)
id = htmlCode.SelectSingleNode("//a[contains(@href,'/trade/')]").GetAttributeValue("href", "");
// fetch the name, donator type is used to find it
HtmlNode node = htmlCode.SelectSingleNode("//span[@class='regular' or starts-with(@class, 'donator')]");
name = (node != null) ? node.InnerText : null;
// the trade's description
HtmlNode descriptionNode = htmlCode.SelectSingleNode("//div[@class='notes expandable show']");
description = (descriptionNode != null) ? descriptionNode.InnerText : "";
/*
* here we construct the item objects
*
*/
HtmlNode itemOfferBundleHtml = htmlCode.SelectSingleNode("(div[@class='four-column'])[1]");
HtmlNode itemRequestBundleHtml = htmlCode.SelectSingleNode("(div[@class='four-column'])[2]");
HtmlNodeCollection itemOfferHtml = itemOfferBundleHtml.SelectNodes("div[starts-with(@class, 'item')]");
HtmlNodeCollection itemRequestHtml = itemRequestBundleHtml.SelectNodes("div[starts-with(@class, 'item')]");
offer = new List<Item>();
request = new List<Item>();
for (int i = 0; i < itemOfferHtml.Count; i++) { offer.Add(new Item(itemOfferHtml[i])); }
for (int i = 0; i < itemRequestHtml.Count; i++) { request.Add(new Item(itemRequestHtml[i])); }
}
internal Match MapHtmlNodeToMatch(HtmlNode root, MatchPath matchPath, int currentCount)
{
Match match = new Match();
match.Id = HtmlEntity.DeEntitize(root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Id, currentCount)).Attributes[matchPath.IdAttribute].Value).Replace(matchPath.IdReplace, "");
HtmlNode heroNode = root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Hero, currentCount));
if (heroNode != null)
{
string heroReference = root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Hero, currentCount)).Attributes[HtmlAttributes.Hero.Attribute.Value].Value.Replace(HtmlAttributes.Hero.Replace.Value, "");
match.Hero = mainController.HeroController.GetHero(heroReference);
}
match.Result = mainController.MapStringToEnum<Results>(root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Result, currentCount)).InnerText);
match.TimeAgo = DateTime.Parse(root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.TimeAgo, currentCount)).Attributes[MainController.HTML_ATTRIBUTE_DATETIME].Value);
match.Type = mainController.MapStringToEnum<Types>(root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Type, currentCount)).InnerText);
match.Mode = mainController.MapStringToEnum<Modes>(root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Mode, currentCount)).InnerText);
HtmlNode skillBracketNode = root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Skillbracket, currentCount));
if (skillBracketNode != null)
match.Skillbracket = mainController.MapStringToEnum<Skillbrackets>(HtmlEntity.DeEntitize(skillBracketNode.InnerText));
match.Duration = mainController.ConvertStringToTimeSpan(root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Duration, currentCount)).InnerText);
match.Kda = mainController.ConvertStringToKda(root.SelectSingleNode(mainController.CombinePathWithListCount(matchPath.Kda, currentCount)).InnerText);
return match;
}
private static ArticleInfo ParseArticleInfoDiv(HtmlNode articleDiv)
{
var linkToArticle = articleDiv.SelectSingleNode("h3/a");
var dateDiv = articleDiv.SelectSingleNode("div[@class='headline-date']");
var commentCountNode = articleDiv.SelectSingleNode("h3/a[@class='commentCount']");
var articleInfo = new ArticleInfo();
articleInfo.Url = linkToArticle.Attributes["href"].Value;
if (articleInfo.Url.Contains(@"/video/"))
{
throw new CommonParsingException("Delfi TV article");
}
articleInfo.Id.ExternalId = articleInfo.Url.GetQueryParameterValueFromUrl("id");
articleInfo.Title = linkToArticle.InnerText;
articleInfo.DatePublished = DelfiWordyDateParser.Parse(dateDiv.InnerText);
articleInfo.DateScraped = DateTime.UtcNow.AddHours(2);
articleInfo.Id.Portal = Portal.Delfi;
articleInfo.CommentCount = commentCountNode == null ? 0 : Convert.ToInt32(commentCountNode.InnerText.TrimStart('(').TrimEnd(')'));
var articleId = Convert.ToInt32(articleInfo.Url.GetQueryParameterValueFromUrl("id"));
if (articleId == 0) throw new CommonParsingException("Article id not found");
return articleInfo;
}
private string VariableNameResolver(HtmlNode row)
{
var isOptional = row.SelectSingleNode(@".//td[contains(@class,'description')]/span[@class='optional']");
var variable = row.SelectSingleNode(@".//td[@class=""name"" ]").InnerText + (isOptional != null ? "?" : "");
return variable;
}
private static string getMoney(HtmlNode node)
{
try
{
string money = "";
string gold = HtmlEntity.DeEntitize(node.SelectSingleNode("descendant::span[@class='mgold']").InnerText).TrimStart();
string silver = HtmlEntity.DeEntitize(node.SelectSingleNode("descendant::span[@class='msilver']").InnerText).TrimStart();
string bronze = HtmlEntity.DeEntitize(node.SelectSingleNode("descendant::span[@class='mbronze']").InnerText).TrimStart();
if (gold != "")
money += gold;
if (silver != "")
if (silver.Length > 1)
money += silver;
else
if (gold == "")
money += silver;
else
money += '0' + silver;
if (bronze.Length > 1)
money += bronze;
else
money += '0' + bronze;
return money;
}
catch (NullReferenceException e)
{
return "";
}
}
internal Stat MapHtmlNode(HtmlNode root, int currentCount)
{
string matchesPlayed = root.SelectSingleNode(string.Format(PlayerPath.MostPlayedHeroes.MatchesPlayed.Value, currentCount)).InnerText;
string winRate = root.SelectSingleNode(string.Format(PlayerPath.MostPlayedHeroes.Winrate.Value, currentCount)).InnerText;
string kdaRatio = root.SelectSingleNode(string.Format(PlayerPath.MostPlayedHeroes.Kda.Value, currentCount)).InnerText;
return new Stat(matchesPlayed.Replace(",", ""), winRate, kdaRatio.Replace(".", ","));
}
public static Remark ParseRemark(HtmlNode remarkNode)
{
var remark = new Remark();
//发现物
var discoveryNode = remarkNode.SelectSingleNode("a[@title!='']");
if (discoveryNode!=null)
{
var levelNode = discoveryNode.PreviousSibling.PreviousSibling;
var typeNode = levelNode.PreviousSibling.PreviousSibling;
remark.DiscoveryType = Enum.Parse(typeof(DisType), typeRegex.Match(typeNode.Attributes["src"].Value).Groups["type"].Value).ToString();
remark.DiscoveryLevel = Int32.Parse(levelNode.InnerText.Substring(0, 1));
remark.DiscoveryExp = Int32.Parse(discoveryNode.Attributes["title"].Value.Remove(0,5));
remark.Discovery = discoveryNode.InnerText;
}
//奖励物
var awardNode = remarkNode.SelectSingleNode("span[@style='color:#804000;']");
if (awardNode != null)
remark.AwardItem = awardNode.InnerText;
//相关任务
var relativeNodes = remarkNode.SelectNodes("descendant::a[@style='color:#C000C0;' or @style='color:DarkBlue;']");
if (relativeNodes != null)
{
foreach (HtmlNode relativeNode in relativeNodes)
{
IList<int> questList = null;
IList<string> foundNameList = null;
if (relativeNode.InnerText.StartsWith("前:"))
{
foundNameList = remark.PreFoundName;
questList = remark.PreQuestID;
}
else
{
questList = remark.FollowQuestID;
}
var match = questRegex.Match(relativeNode.Attributes["href"].Value);
if (relativeNode.InnerText.StartsWith("前:港口-") == false)
questList.Add(Int32.Parse(match.Groups["id"].Value));
else
foundNameList.Add(relativeNode.InnerText.Replace("前:港口-",""));
}
}
//接受城市
//last br next a
var cityNodes = remarkNode.SelectNodes("descendant::a[@class='MisCity']");
if (cityNodes != null)
{
cityNodes.All(node =>
{
if (node.InnerText == "南美开拓港" || node.InnerText == "东南亚开拓港" ||
node.InnerText == "掠夺地图" || node.InnerText == "沉船资讯")
return true;
remark.FromCityList.Add(node.InnerText);
return true;
});
}
return remark;
}
private string VariableTypeResolver(HtmlNode row)
{
var tdType = row.SelectSingleNode(@".//td[@class=""type"" ]");
//EntityCollection contains(entity) as no type information.
var typeNodes = tdType.SelectNodes(@".//span[@class='param-type']");
if (typeNodes == null)
return "any";
var types = typeNodes.Select(Program.TypeReader).ToArray().Distinct();
if (!types.Skip(1).Any() && types.First() == "Object")
{
var props = GetSignatureTypes(row.SelectSingleNode(@".//td[contains(@class,'description')]"));
if (props.Keys.Any())
{
// var type = "opt_" + Program.CalculateMD5Hash(string.Join("", props.Keys.OrderBy(k => k)));
var type = this._name + "Options";
var dependencies = new List<string>();
var writer = Program.GetWriter(type,_source);
if (writer != null)
{
props = props.ToDictionary(k => k.Key, v => Program.extractDependencies(dependencies, v.Value));
Program.WriteDependencies(type, dependencies, writer, null, null, _source);
writer.WriteLine($"interface {type}");
writer.WriteLine("{");
foreach (var prop in props)
{
writer.WriteLine($"\t{prop.Key}: {prop.Value};");
}
writer.WriteLine("}");
writer.WriteLine($"export = {type}");
}
return "Cesium." + type;
}
}
return string.Join("|", types);
}
protected override string ExtractAuthor(HtmlNode node)
{
if (node.SelectSingleNode("div/div[2]/div/div/span") != null)
{
return node.SelectSingleNode("div/div[2]/div/div/span").InnerText.Split('-')[0].Trim();
}
return string.Empty;
}
private string getDescription(HtmlNode node)
{
var node2 = node.SelectSingleNode(".//div[@class='hoverbox-details']/p");
if (node2 == null)
node2 = node.SelectSingleNode(".//div[@class='list-item-main1']");//klipit
if (node2 != null)
return node2.InnerText;
return String.Empty;
}
private string GetAuthorName(HtmlNode docNode)
{
var authorNode = docNode.SelectSingleNode(".//strong[@itemprop='author']");
if (authorNode == null)
{
authorNode = docNode.SelectSingleNode(".//div[@class='rtl-info']/strong");
}
return authorNode != null ? authorNode.InnerText.Trim() : null;
}
protected override string ExtractSnippet(HtmlNode node)
{
if (node.SelectSingleNode("div/div[2]") != null)
{
return node.SelectSingleNode("div/div[2]").InnerText;
}
return string.Empty;
}
private string getDescription(HtmlNode node)
{
var node2 = node.SelectSingleNode(".//p[@class='expanding-card__description']");//kausi
if (node2 == null)
node2 = node.SelectSingleNode(".//div[@class='list-item-main1']");//klipit
if (node2 != null)
return node2.InnerText;
return String.Empty;
}
protected override string ExtractTitle(HtmlNode node)
{
if (node.SelectSingleNode("div/div[1]/h3/a") != null)
{
return node.SelectSingleNode("div/div[1]/h3/a").InnerText;
}
return string.Empty;
}
public override string ReadContent(HtmlNode node)
{
var content = node.SelectSingleNode("//div[@id='mediaarticlebody']");
if (content == null)
content = node.SelectSingleNode("//div[@itemprop='articleBody']");
RemoveScripts(content);
RemoveTags(GetNodeByXpathAndClass(node, "//div", "yog-ycb"));
return content.InnerHtml;
}
public virtual void Parser(HtmlNode node)
{
userURL = node.SelectSingleNode("div[@class='log-content']/div[@class='log-body']/a[last()]").Attributes["href"].Value;
userName = node.SelectSingleNode("div[@class='log-content']/div[@class='log-body']/a[last()]").InnerText;
stampName = node.SelectSingleNode("div[@class='log-content']/div[@class='log-details log-target log-target-stamp']/div[@class='log-target-info']/a").InnerText;
stampURL = node.SelectSingleNode("div[@class='log-content']/div[@class='log-details log-target log-target-stamp']/div[@class='log-target-info']/a").Attributes["href"].Value;
stampImageURL = node.SelectSingleNode("div[@class='log-content']/div[@class='log-details log-target log-target-stamp']/div[@class='log-target-thumbnail']/a/img").Attributes["data-src"].Value;
longago = new NicorepoItemSubLongago(node);
nicoru = new NicorepoItemSubNicoru(node);
}
public override string ReadContent(HtmlNode node)
{
var content = node.SelectSingleNode("//div[@class='article-body']");
if (content == null)
content = node.SelectSingleNode("//div[@class='slide-description']");
RemoveScripts(content);
RemoveTags(GetNodeByXpathAndClass(node,"//div","poll_module"));
return CleanHtml(content.InnerHtml);
}
public override string ReadAuthor(HtmlNode node)
{
var author = node.SelectSingleNode("//a[@rel='author']");
if (author != null)
return author.InnerText;
else
{
author = node.SelectSingleNode("//meta[@itemprop='author']");
}
return string.Empty;
}
private static IUsage CreateUsageNode(HtmlNode node)
{
var typeNode = node.SelectSingleNode( ".//em/text()" );
var textNode = node.SelectSingleNode( ".//strong/text()" );
return new Usage
{
Text = textNode.InnerText,
Type = GetWordType( typeNode ),
Synonyms = ExtractSynonyms( node )
};
}
public HtmlAgilityPackTableXpath(string html, int skip)
{
htmlTableAttDic = new Dictionary<string, int>();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
this.skip = skip;
doc.LoadHtml(html);
rootDomNode = doc.DocumentNode;
GetHtmlAllTableXpath(rootDomNode);
maxValue = rootDomNode.SelectSingleNode(maxKey).GetAttributeValue("id", "");
secondValue = rootDomNode.SelectSingleNode(secondKey).GetAttributeValue("id", "");
}
public Departure(HtmlNode departure)
{
var time = departure.SelectSingleNode(TimetabledDepartureXPath).ChildNodes[0].InnerText;
TimetabledDeparture = new DepartureTime(time);
if (departure.SelectSingleNode(PlatformXPath) != null)
{
Platform = departure.SelectSingleNode(PlatformXPath).ChildNodes[2].InnerText;
}
}
public override string ReadContent(HtmlNode node)
{
var content = node.SelectSingleNode("//div[@class='bg-opaque pad-16 article']");
if (content == null)
content = node.SelectSingleNode("//article");
RemoveScripts(content);
RemoveTags(content, "//div");
return CleanHtml(content.InnerHtml);
}
public TorrentLeechEntry(HtmlNode node, BrowserClient client)
{
_client = client;
Title = node.SelectSingleNode(".//span[@class='title']/a").InnerText;
Friendly = Title.TorrentName();
var size = node.SelectSingleNode(".//td[5]").InnerText;
double number = double.Parse(size.RegexMatch(@"\d+").Value);
if(size.Contains("GB"))
number *= 1024;
Size = number;
Torrent = "http://torrentleech.org" + node.SelectSingleNode(".//td[@class='quickdownload']/a").Attributes["href"].Value;
}
private static string GetAuthorName(HtmlNode docNode)
{
var node = docNode.SelectSingleNode("//div[@class='delfi-author-name']") ??
docNode.SelectSingleNode("//div[@class='delfi-source-name']");
if (node == null)
{
var t = 5;
}
return node == null ? null : node.InnerText;
}
public override string ReadContent(HtmlNode node)
{
var content = node.SelectSingleNode("//div[@class='fs-article']");
if (content == null)
content = node.SelectSingleNode("//div[@id='fsn_v3_main']");
if (content == null)
return null;
RemoveScripts(content);
RemoveTags(content, "//div[@id='story-top-container']");
RemoveTags(content, "//div[@class='parentWrapper']");
return CleanHtml(content.InnerHtml);
}
private BsonDocument ParseMessageViewNode(HtmlNode messageViewNode)
{
var mainContentNode = messageViewNode.SelectSingleNode("div//div[contains(@class,'lia-quilt-column-main-right')]");
var messageAuthorNode = messageViewNode.SelectSingleNode("div//div[contains(@class, 'lia-message-author')]");
var idNode = messageViewNode.SelectSingleNode("div//div[@data-message-id]");
ThrowExceptionIfNull(idNode, "can not find the id node");
ThrowExceptionIfNull(mainContentNode, "can not find the content node");
var subjectNode = mainContentNode.SelectSingleNode("div//div[contains(@class, 'lia-message-subject')]");
ThrowExceptionIfNull(subjectNode, "can not find the subject node");
var titleNode = subjectNode.SelectSingleNode("h1");
var solutionNode = subjectNode.SelectSingleNode("span[contains(@class, 'solution')]");
var messagePostDateNode = mainContentNode.SelectSingleNode("div//span[contains(@class,'lia-message-posted-on')]");
var bodyNode = mainContentNode.SelectSingleNode("div//div[contains(@class, 'lia-message-body-content')]");
var userNameNode = messageAuthorNode.SelectSingleNode("div//a[contains(@class, 'lia-user-name-link')]/span");
var userRegisterDateNode = messageAuthorNode.SelectSingleNode("div//span[contains(@class, 'DateTime')]");
var author = new BsonDocument
{
{ "name", userNameNode.InnerText},
{ "registeredOn", ParseDateTimeNode(userRegisterDateNode)}
};
var message = new BsonDocument
{
{ "id", idNode.Attributes["data-message-id"].Value },
{ "body", bodyNode == null ? string.Empty : bodyNode.InnerHtml},
{ "isAnswer", solutionNode != null },
{ "createdOn", ParseDateTimeNode(messagePostDateNode)},
{ "author", author }
};
if(titleNode != null)
{
message.Add("title", titleNode.InnerText);
}
return message;
}
public TorrentLeechEntry(HtmlNode node, BrowserClient client)
{
_client = client;
var titleNode = node.SelectSingleNode(".//span[@class='title']/a");
Title = titleNode.InnerText;
Id = int.Parse(titleNode.Attributes["href"].Value.RegexMatch(@"\d+").Value);
Friendly = Title.TorrentName();
var size = node.SelectSingleNode(".//td[5]").InnerText;
double number = double.Parse(size.RegexMatch(@"\d+").Value);
if (size.Contains("GB"))
number *= 1024;
Size = number;
Torrent = "http://www.torrentleech.org/rss/download/{0}/ed2597d8977cde9da218/{1}".Template(Id,Title.RegexReplace(@"\s", "."));
}
private int AddSubcats(HtmlNode node, RssLink parentCat)
{
var subs = node.SelectNodes(".//article");
foreach (var sub in subs)
{
RssLink subcat = new RssLink()
{
ParentCategory = parentCat
};
subcat.Name = HttpUtility.HtmlDecode(sub.SelectSingleNode(".//a[@title]").Attributes["title"].Value.Trim());
subcat.Url = FormatDecodeAbsolutifyUrl(parentCat.Url, sub.SelectSingleNode(".//a[@href]").Attributes["href"].Value, null, UrlDecoding.None);
subcat.Thumb = getThumb(sub.SelectSingleNode(".//picture/img"));
parentCat.SubCategories.Add(subcat);
}
var np = node.SelectSingleNode(".//a[@href and text()='More shows']");
nextPageAvailable = false;
if (np != null)
{
string url = CreateUrl(parentCat.Url, np.Attributes["href"].Value);
var npCat = new NextPageCategory()
{
Url = url, ParentCategory = parentCat
};
parentCat.SubCategories.Add(npCat);
}
parentCat.SubCategoriesDiscovered = true;
return(parentCat.SubCategories.Count);
}
private static void ParseEventDiv(List <DateTimeOffset> weekDays, int columnIndex, List <NewsEvent> list1, HAP.HtmlNode div)
{
var name = div.SelectNode("a").InnerText.Trim();
var country = name.Split(':').FirstOrDefault();
if (string.IsNullOrWhiteSpace(country))
{
country = "ALL";
}
else
{
name = name.Substring(country.Length + 1);
}
var childNodes = div.ChildNodes;
var dates = (from node in childNodes
where node.NodeType == HAP.HtmlNodeType.Text
select node.InnerText.Decode()).ToArray();
if (!dates.Any())
{
throw new NewsParserException("No text nodes found in column " + (columnIndex + 1) + " for " + name);
}
var date = dates.FirstOrDefault(s => !string.IsNullOrWhiteSpace(s));
if (date == null)
{
return;
}
//throw new NewsParserException("No dates found in column " + (columnIndex + 1) + " for " + name);
var level = div.SelectSingleNode(starXPath("Star")) != null
? NewsEventLevel.H
: div.SelectSingleNode(starXPath("djStar")) != null ? NewsEventLevel.M
: NewsEventLevel.L;
var n = new NewsEvent()
{
Country = country,
Name = name,
Time = weekDays[columnIndex].Add(ParseEventDate(date).TimeOfDay),
Type = name.Contains("Speaks") ? NewsEventType.Speech : NewsEventType.Report,
Level = level
};
list1.Add(n);
}
private void buttonParseHtml_Click(object sender, EventArgs e)
{
OpenFileDialog openFileDialog = new OpenFileDialog();
openFileDialog.Filter = "HTML File (*.html;)|*.html";
openFileDialog.Multiselect = false;
if (openFileDialog.ShowDialog() == DialogResult.OK)
{
if (String.IsNullOrEmpty(openFileDialog.FileName))
{
return;
}
string strHtml = String.Empty;
using (StreamReader reader = new StreamReader(openFileDialog.FileName, Encoding.UTF8))
{
strHtml = reader.ReadToEnd();
reader.Close();
}
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(strHtml); //加载html
foreach (var err in doc.ParseErrors)
{
Console.WriteLine(err.Code);
}
//Console.WriteLine(doc.Text);
HtmlAgilityPack.HtmlNode rootNode = doc.DocumentNode; //获取文档的根节点
//Console.WriteLine(rootNode.OuterHtml);
#if true
string xpath = @"//table";
HtmlAgilityPack.HtmlNode node = rootNode.SelectSingleNode(xpath); //获取表格
foreach (var script in node.Descendants("script").ToArray())
{
script.Remove();
}
foreach (var style in node.Descendants("style").ToArray())
{
style.Remove();
}
string innerText = node.OuterHtml; //到这里就是纯纯的表格了
var trNodes = node.SelectNodes("tr");
foreach (var trnod in trNodes) //遍历行
{
var tdNodes = trnod.SelectNodes("td");
for (int i = 0; i < tdNodes.Count; i++) //遍历列
{
Console.WriteLine(tdNodes[i].InnerText);
}
}
#endif
}
}
public static HAP.HtmlNode SelectNode(this HAP.HtmlNode parent, string xPath)
{
var node = parent.SelectSingleNode(xPath);
if (node == null)
{
throw new NewsParserException(new { xPath });
}
return(node);
}
/// <summary>
/// 检测服务器是否存在错误
/// </summary>
/// <returns></returns>
public static bool TestServiceError(Html.HtmlNode doucmentNode)
{
var node = doucmentNode.SelectSingleNode("/html[1]/body[1]/strong[1]/span[1]");
if (node != null && node.InnerText.IndexOf("网站忙") != -1)
{
return(true);
}
return(false);
}
public DAL.Entities.Article ParseArticle(HtmlAgilityPack.HtmlNode rootNode, ScrapingExpressions expressions)
{
var title = rootNode.SelectSingleNode(expressions.TitleXPath);
var post = rootNode.SelectSingleNode(expressions.PostXPath);
var date = rootNode.SelectSingleNode(expressions.DateXPath);
if (title == null || post == null || date == null)
{
Console.WriteLine("Failed to parse document");
return(null);
}
return(new DAL.Entities.Article()
{
Contents = post.InnerText,
Title = title.InnerText,
CreationDate = Program.ParseDateTime(date.InnerText),
});
}
public static BsonDocument AnalizeGeneralListInformation(HtmlAgilityPack.HtmlNode node, Checker checker, BufferBlock <string> imageTargetBlock)
{
var bson = new BsonDocument();
var linkNode = node.SelectSingleNode(".//div[@class='h16']/a");
if (linkNode != null)
{
if (checker(linkNode.Attributes["href"].Value))
{
return(null);
}
var link = linkNode.Attributes["href"].Value;
bson.Add("link", link);
bson.Add("title", linkNode.InnerText);
}
var dateNode = node.SelectSingleNode(".//div[@class='date']");
if (dateNode != null)
{
bson.Add("date", dateNode.InnerText);
}
var contentNode = node.SelectSingleNode(".//div[@class='p']");
if (contentNode != null)
{
bson.Add("content", contentNode.InnerText);
}
var imageNode = node.SelectSingleNode(".//img");
if (imageNode != null)
{
var imgUrl = imageNode.Attributes["src"].Value;
bson.Add("img", GetSubUrl(imgUrl));
bson.Add("compressImg", WebImageSaver.Instance.GetComressImageName(GetSubUrl(imgUrl)));
imageTargetBlock.Post(GetIhChina.MainPage + imgUrl);
}
Console.WriteLine(bson.ToString());
return(bson);
}
protected override float?getRating(HtmlAgilityPack.HtmlNode node)
{
var ratingNode = node.SelectSingleNode(ratingXPath);
if (ratingNode != null)
{
float rating = 0;
if (!float.TryParse(ratingNode.Attributes["content"].Value, out rating))
{
}
return(rating);
}
return(null);
}
private List <VideoInfo> GetVids(HtmlNode node, string parentUrl)
{
List <VideoInfo> videos = new List <VideoInfo>();
var vids = node.SelectNodes(".//article");
foreach (var vid in vids)
{
VideoInfo video = new VideoInfo();
if (vid.SelectSingleNode(".//h2[contains(@class,'h3')]") == null)
{
video.Title = HttpUtility.HtmlDecode(vid.SelectSingleNode(".//a[@title]").Attributes["title"].Value.Trim());
video.VideoUrl = FormatDecodeAbsolutifyUrl(parentUrl, vid.SelectSingleNode(".//a[@href]").Attributes["href"].Value, null, UrlDecoding.None);
}
else
{
video.Title = vid.SelectSingleNode(".//h2[contains(@class,'h3')]").InnerText.Trim();
video.VideoUrl = FormatDecodeAbsolutifyUrl(parentUrl, vid.SelectSingleNode(".//a[@class='teaser__link' and @href]").Attributes["href"].Value, null, UrlDecoding.None);
if (vid.SelectSingleNode(".//p[contains(@class,'teaser__description')]") != null)
{
video.Description = vid.SelectSingleNode(".//p[contains(@class,'teaser__description')]").InnerText.Trim();
}
else
{
video.Description = vid.SelectSingleNode(".//h3[contains(@class,'teaser__subtitle')]").InnerText.Trim();
}
}
var moNode = vid.SelectSingleNode(".//span[@data-month]");
var daNode = vid.SelectSingleNode(".//span[@data-date]");
if (moNode != null && daNode != null)
{
video.Airdate = moNode.InnerText.Trim() + ' ' + daNode.InnerText.Trim();
}
video.Thumb = getThumb(vid.SelectSingleNode(".//picture/img"));
videos.Add(video);
}
var np = node.SelectSingleNode(".//a[@href and contains(text(),'More ')]");
nextPageAvailable = false;
if (np != null)
{
nextPageAvailable = true;
nextPageUrl = CreateUrl(parentUrl, np.Attributes["href"].Value);
}
return(videos);
}
public static BookInfo AmazonSearchBook(string title, string author)
{
BookInfo result = null;
string authorTrim = "";
Regex regex = new Regex(@"( [A-Z]\.)", RegexOptions.Compiled);
Match match = Regex.Match(author, @"( [A-Z]\.)", RegexOptions.Compiled);
if (match.Success)
{
foreach (Match m in regex.Matches(author))
{
authorTrim = author.Replace(m.Value, m.Value.Trim());
}
}
else
{
authorTrim = author;
}
if (title.IndexOf(" (") >= 0)
{
title = title.Substring(0, title.IndexOf(" ("));
}
string searchUrl = @"http://www.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Ddigital-text&field-keywords=" +
Uri.EscapeDataString(title + " " + authorTrim + " kindle edition");
HAP.HtmlDocument searchDoc = new HAP.HtmlDocument();
searchDoc.LoadHtml(HttpDownloader.GetPageHtml(searchUrl));
HAP.HtmlNode node = searchDoc.DocumentNode.SelectSingleNode("//li[@id='result_0']");
//At least attempt to verify it might be the same book?
//Ignore case of title
if (node != null && node.InnerText.IndexOf(title, StringComparison.OrdinalIgnoreCase) >= 0)
{
string foundASIN = node.GetAttributeValue("data-asin", "");
node = node.SelectSingleNode(".//div/div/div/div[@class='a-fixed-left-grid-col a-col-right']/div/a");
if (node != null)
{
result = new BookInfo(node.InnerText, author, foundASIN);
result.amazonUrl = node.GetAttributeValue("href", ""); // Grab the true link for good measure
}
}
return(result);
}
internal static int GetReviewLastPageNumber(HtmlAgilityPack.HtmlNode html)
{
/*
* <ul class="a-pagination">
* <li class="a-disabled">
* <span class="a-declarative" data-action="reviews:page-action" data-reviews:page-action="{"allowLinkDefault":"1"}">
* ←<span class="a-letter-space"></span><span class="a-letter-space"></span>Previous
* </span>
* </li>
* <li class="a-selected page-button" data-reftag="cm_cr_pr_btm_link">
* <a href="/We-Yevgeny-Zamyatin/product-reviews/0140185852/ref=cm_cr_pr_btm_link_1?ie=UTF8&pageSize=50&sortBy=recent">
* 1
* </a>
* </li>
* ...
* <li class="page-button" data-reftag="cm_cr_pr_btm_link">
* <a href="/We-Yevgeny-Zamyatin/product-reviews/0140185852/ref=cm_cr_pr_btm_link_5?ie=UTF8&pageNumber=5&pageSize=50&sortBy=recent">
* 5
* </a>
* </li>
* <li class="a-last">
* <a href="/We-Yevgeny-Zamyatin/product-reviews/0140185852/ref=cm_cr_pr_paging_btm_2?ie=UTF8&pageNumber=2&pageSize=50&sortBy=recent">
* Next<span class="a-letter-space"></span><span class="a-letter-space"></span>→
* </a>
* </li>
* </ul>
*/
int pageNum = 0;
var lastPageNode = html.SelectSingleNode("//ul/li[@class='page-button' and position() = (last()-1)]/a");
if (lastPageNode != null)
{
Match m = pageNumberFromUrl.Match(lastPageNode.GetAttributeValue("href", ""));
if (m.Groups.Count > 1)
{
int.TryParse(m.Groups[1].Value, out pageNum);
}
}
return(pageNum);
}
public string ParseNewsItem(HtmlAgilityPack.HtmlNode item)
{
// to visualize, news items have this structure:
// <p>сегодня, 12:30</p
// <a href="/relative-uri">
// <span>title</span>
// </a>
try
{
var link = item.SelectSingleNode("a");
var uriText = link.GetAttributeValue("href", "");
return(uriText);
}
catch (NullReferenceException ex)
{
// something wrong with markup, log
Console.WriteLine("Error parsing item {0}, malformed markup: {1}, {2}", item.XPath, ex.Message, ex.StackTrace);
}
return(null);
}
Boolean PageHasData(string documentText)
{
try
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(documentText);
HtmlAgilityPack.HtmlNode bloodDonor = htmlDocument.DocumentNode.SelectSingleNode("//*[@id='regmain']");
HtmlAgilityPack.HtmlNode userLink = bloodDonor.SelectSingleNode("//*[@id='regPage']/div/div/div");
// Of Label
// LabelElement.InnerHTML.Equals("of")
// linkPageCount = aElement.InnHTML
var n1 = userLink.Descendants().ToArray();
var link = userLink.Descendants().Count();
totalPages = Int32.Parse(n1[(link - 5)].InnerText);
if (totalPages > 0)
{
return(true);
}
else
{
return(false);
}
}
catch (Exception ex)
{
totalPages = 0;
return(false);
}
}
//static int nLevels = 64;
//static Size testPadding = new Size(32, 32);
//static double hitThreshold = 0;
//static int groupThreshold = 2;
//static double scaleStep = 1.05;
//static bool useMeanShiftGrouping = false;
public static List <PSM4TxSample> LoadSamples(string folder)
{
//Load all samples from all the folders
List <PSM4TxSample> samples = new List <PSM4TxSample>(); List <string> missing_xml_samples = new List <string>();
string[] files = Directory.EnumerateFiles(folder)
.Where(file => file.ToLower().EndsWith(".bmp") || file.ToLower().EndsWith(".jpg"))
.ToArray();
for (int k = 0; k < files.Length; k++)
{
string img_file = files[k].ToLower();
string xml_file = img_file.EndsWith("bmp") ? img_file.Replace(".bmp", "_data.xml") : img_file.Replace(".jpg", "_data.xml");
if (!File.Exists(xml_file))
{
missing_xml_samples.Add(img_file); continue;
}
PSM4TxSample sample = new PSM4TxSample(img_file);
// .// Means descendants, which includes children of children (and so forth).
// ./ Means direct children.
//If a XPath starts with a / it becomes relative to the root of the document;
//to make it relative to your own node start it with ./.
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.Load(xml_file);
HtmlAgilityPack.HtmlNodeCollection shape_nodes = doc.DocumentNode.SelectNodes("//shape");
foreach (HtmlAgilityPack.HtmlNode shape_node in shape_nodes)
{
HtmlAgilityPack.HtmlNode blocktext_node = shape_node.SelectSingleNode(".//blocktext");
HtmlAgilityPack.HtmlNode text_node = blocktext_node.SelectSingleNode("./text");
HtmlAgilityPack.HtmlNode data_node = shape_node.SelectSingleNode(".//data");
HtmlAgilityPack.HtmlNode extent_node = data_node.SelectSingleNode("./extent");
string s_x = extent_node.GetAttributeValue("X", "");
string s_y = extent_node.GetAttributeValue("Y", "");
string s_w = extent_node.GetAttributeValue("Width", "");
string s_h = extent_node.GetAttributeValue("Height", "");
if (text_node.InnerText.ToLower() == "isolator")
{
sample.SetIsolator(s_x, s_y, s_w, s_h); //(787.18896484375, 1370.0)
}
else if (text_node.InnerText.ToLower() == "arrayblock")
{
sample.SetArrayBlock(s_x, s_y, s_w, s_h); //(1052.86828613281, 1201.8359375)
}
else if (text_node.InnerText.ToLower() == "aperture")
{
sample.SetAperture(s_x, s_y, s_w, s_h); //(209.156982421875, 1885.03271484375)
}
}
samples.Add(sample);
}
using (StreamWriter sw = new StreamWriter("missing_xml.txt"))
{
for (int i = 0; i < missing_xml_samples.Count; i++)
{
sw.WriteLine(missing_xml_samples[i]);
}
}
return(samples);
}
void ExtractUsers(string documentText)
{
try
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(documentText);
HtmlAgilityPack.HtmlNode bloodDonor = htmlDocument.DocumentNode.SelectSingleNode("//*[@id='regmain']");
HtmlAgilityPack.HtmlNode linkPage = htmlDocument.DocumentNode.SelectSingleNode("//*[@id='regPage']/div/div/div/a[4]");
//linkPageCount = linkPage.InnerLength;
HtmlAgilityPack.HtmlNode userTable = htmlDocument.DocumentNode.SelectSingleNode("//*[@id='regPage']/div/table");
HtmlAgilityPack.HtmlNodeCollection userTableCount = userTable.SelectNodes("./tr");
for (int k = 2; k <= userTableCount.Count(); k++)
{
HtmlAgilityPack.HtmlNode moreBtn = userTable.SelectSingleNode("./tr[" + k + "]/td[6]/a");
string linkBtn = moreBtn.GetAttributeValue("onclick", null);
string[] mainLink = linkBtn.Split('b');
string moreLink = mainLink[1];
using (WebClient client = new WebClient())
{
var link = client.DownloadString("http://bloodhelpers.com/b" + moreLink);
HtmlAgilityPack.HtmlDocument userDocument = new HtmlAgilityPack.HtmlDocument();
userDocument.LoadHtml(link);
HtmlAgilityPack.HtmlNode selectTable = userDocument.DocumentNode.SelectSingleNode("//*[@id='search']/form/table");
//User Name
HtmlAgilityPack.HtmlNode userName = selectTable.SelectSingleNode("./tr[1]/td[2]");
string Name = userName.InnerText;
//User Email
HtmlAgilityPack.HtmlNode userEmail = selectTable.SelectSingleNode("./tr[2]/td[2]");
string[] Mail = userEmail.InnerHtml.Split('=');
string addMail = Mail[1] + "=" + Mail[2];
string Email = addMail.Replace("border", "");
//fuction to convert png image into jpg
//User BloodGroup
HtmlAgilityPack.HtmlNode userBloodGroup = selectTable.SelectSingleNode("./tr[3]/td[2]");
string BloodGroup = userBloodGroup.InnerText;
//User Gender
HtmlAgilityPack.HtmlNode userGender = selectTable.SelectSingleNode("./tr[4]/td[2]");
string Gender = userGender.InnerText;
//User Age
HtmlAgilityPack.HtmlNode userAge = selectTable.SelectSingleNode("./tr[5]/td[2]");
int Age = Int32.Parse(userAge.InnerText.Replace("Years", ""));
//User City
HtmlAgilityPack.HtmlNode userCity = selectTable.SelectSingleNode("./tr[6]/td[2]");
string City = userCity.InnerText;
//User Mobile Number
HtmlAgilityPack.HtmlNode userMobile = selectTable.SelectSingleNode("./tr[7]/td[2]");
string[] num = userMobile.InnerHtml.Split('=');
string addNum = num[1] + "=" + num[2];
string mobileNumber = addNum.Replace("border", "");
//function to convert png image into jpg
//User Land Line Number
HtmlAgilityPack.HtmlNode userLandLine = selectTable.SelectSingleNode("./tr[8]/td[2]");
string[] landNum = userLandLine.InnerHtml.Split('=');
string addLandNum = landNum[1] + "=" + landNum[2];
string landLineNum = addLandNum.Replace("border", "");
//Function to convert png image into jpg
//User Last Donation Date
HtmlAgilityPack.HtmlNode userLastDonationDate = selectTable.SelectSingleNode("./tr[9]/td[2]");
string LastDonationDate = userLastDonationDate.InnerText;
//Store Data in DataBase
StoreUserData(Name, Email, BloodGroup, Gender, Age, City, mobileNumber, landLineNum, LastDonationDate);
}
}
}
catch
{
}
}
public void start_conversion()
{
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
string html = wc.DownloadString(url.Text);
agi.HtmlDocument doc = new agi.HtmlDocument();
doc.LoadHtml(html);
agi.HtmlNode id_main = doc.GetElementbyId("main");
string title = id_main.SelectSingleNode("//h1").InnerText; //제목
string re_title = "";
String[] result_title;
if (title.Contains("» "))
{
result_title = title.Split(' ');
/*for (int r = 0; r < result_title.Length; r++)
* {
* if (result_title[r] != "»")
* {
* if (r != 0) re_title += " ";
* //title = result_title[r];
* re_title += result_title[r];
* } else
* {
* break;
* }
* }*/
re_title += result_title[0];
}
else
{
re_title = title;
}
//result_context.Text = title + "\r\n";
int now_num = 1;
int data_i = 1;
String[] now_split = new String[] { };
var result_date = "";
//실제날짜 추출
var ani_date = doc.DocumentNode.SelectSingleNode("//div[@class='tidHeader']//table/tr/td[1]").InnerHtml;
String result_ani_date = ani_date.ToString().Split('月')[0];
//result_context.Text = ani_date.ToString().Split('月')[0] + "\r\n";
for (;;)
{
result_date = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText; //날짜 분기
if (data_i == 3)
{
if (result_date.Contains('～'))
{
now_split = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText.Split('～');
break;
}
}
if (data_i == 4)
{
if (result_date.Contains('～'))
{
now_split = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText.Split('～');
break;
}
}
if (data_i == 5)
{
if (result_date.Contains('～'))
{
now_split = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText.Split('～');
break;
}
}
data_i++;
}
String[] now_split2 = now_split[0].Split('-');
if (Convert.ToInt32(now_split2[1]) == 4)
{
now_num = 2;
}
if (Convert.ToInt32(now_split2[1]) == 7)
{
now_num = 3;
}
if (Convert.ToInt32(now_split2[1]) == 10)
{
now_num = 4;
}
String nowyear = now_split2[0];
ArrayList ch_ = new ArrayList();
foreach (var cell in doc.DocumentNode.SelectNodes("//td[@class='ch']"))
{
if (cell.InnerText != " ")
{
ch_.Add(cell.InnerText);
}
}
ArrayList now_date = new ArrayList();
int ch_num = 1;
var team_date = "";
foreach (var cell in doc.DocumentNode.SelectNodes("//table[@class='schedule']/tbody/tr"))
{
foreach (var cell2 in cell.SelectNodes(".//td").Count.ToString())
{
//result_context.Text += Convert.ToInt32(cell2);
if (Convert.ToInt32(cell2) == 53)
{
if (cell.SelectSingleNode(".//td[3]").InnerText != " ")
{
//result_context.Text += doc.DocumentNode.SelectSingleNode("//table[@class='schedule']/tbody/tr[" + ch_num + "]/td[1]").InnerText + "\r\n";
team_date = doc.DocumentNode.SelectSingleNode("//table[@class='schedule']/tbody/tr[" + ch_num + "]/td[1]").InnerText.Split(' ')[0];
now_date.Add(doc.DocumentNode.SelectSingleNode("//table[@class='schedule']/tbody/tr[" + ch_num + "]/td[1]").InnerText.Split(' ')[0]);
}
}
else if (Convert.ToInt32(cell2) == 52)
{
if (cell.SelectSingleNode(".//td[2]").InnerText != " ")
{
//result_context.Text += team_date + "\r\n";
now_date.Add(team_date);
}
}
else if (Convert.ToInt32(cell2) == 50)
{
team_date = doc.DocumentNode.SelectSingleNode("//table[@class='schedule']/tbody/tr[" + ch_num + "]/td[1]").InnerText.Split(' ')[0] + "\r\n";
}
ch_num++;
}
}
ArrayList subtitle_all = new ArrayList();
ArrayList subtitle_num = new ArrayList();
ArrayList subtitle_content = new ArrayList();
ArrayList numsss = new ArrayList();
int numssd = 0;
foreach (var cell in doc.DocumentNode.SelectNodes("//td[@class='subtitle']"))
{
if (cell.InnerText != " ")
{
//MessageBox.Show(cell.InnerText.Substring(0, 1));
if (cell.InnerText.Substring(0, 1) == "^")
{
subtitle_all.Add(cell.InnerText);
subtitle_num.Add(cell.InnerText.Split('#')[1].Split(' ')[0]);
if (1 < cell.InnerText.Split(' ').Length)
{
string cells_text = "";
for (int c = 1; c < cell.InnerText.Split(' ').Length; c++)
{
if (c != 1)
{
cells_text += " ";
}
cells_text += cell.InnerText.Split(' ')[c].Replace("!", "！").Replace("?", "？");
}
subtitle_content.Add(cells_text);
}
else
{
subtitle_content.Add(cell.InnerText.Split(' ')[1]);
}
numsss.Add(numssd);
numssd++;
}
else if (cell.InnerText.Split(' ')[0].Contains("#"))
{
//result_context.Text += cell.InnerText + "\r\n";
subtitle_all.Add(cell.InnerText);
subtitle_num.Add(cell.InnerText.Split(' ')[0].Replace("#", ""));
if (1 < cell.InnerText.Split(' ').Length)
{
string cells_text = "";
for (int c = 1; c < cell.InnerText.Split(' ').Length; c++)
{
if (c != 1)
{
cells_text += " ";
}
cells_text += cell.InnerText.Split(' ')[c].Replace("!", "！").Replace("?", "？");
}
subtitle_content.Add(cells_text);
}
else
{
subtitle_content.Add(cell.InnerText.Split(' ')[1]);
}
numsss.Add(numssd);
numssd++;
}
else
{
numssd++;
continue;
}
}
}
int notget = 0;
ArrayList flag_ = new ArrayList();
foreach (var cell in doc.DocumentNode.SelectNodes("//td[@class='flag nobr']"))
{
if ((int)numsss[0] <= (int)notget)
{
//result_context.Text += cell.InnerText.Replace("！", "").Replace("●", "").Replace("再", "").Replace("Update", "").Replace(" ", "").Trim() + "\r\n";
flag_.Add("[" + cell.InnerText.Replace("！", "").Replace("●", "").Replace("注", "").Replace("再", "").Replace("Update", "").Replace(" ", "").Trim() + "] ");
}
notget++;
}
ArrayList date_sort = new ArrayList();
ArrayList date_text_sort = new ArrayList();
int now_info = 0;
for (int i = 0; i < subtitle_all.Count; i++)
{
Application.DoEvents();
//MessageBox.Show(subtitle_num[i].ToString());
int nums = Convert.ToInt32(subtitle_num[i]);
String resert_num = "";
if (nums < 10)
{
resert_num = "0" + Convert.ToString(subtitle_num[i]);
}
else
{
resert_num = Convert.ToString(subtitle_num[i]);
}
string clb_string = "";
for (int x = 0; x <= brocas.CheckedItems.Count - 1; x++)
{
clb_string = clb_string + brocas.CheckedItems[x].ToString();
}
String flag = "";
if (Convert.ToString(flag_[i]) == "[] ")
{
flag = "";
}
else
{
flag = Convert.ToString(flag_[i]);
}
if (result_context.Text.Contains("第" + resert_num + "話 ") || ch_[i].ToString().Contains(clb_string))
{
continue;
}
else
{
if (all_radio.Checked)
{ //전체
string sub_content = "";
if (subtitle_content[i].ToString() != "")
{
sub_content = "〔 " + subtitle_content[i].ToString() + " 〕";
}
result_context.Text += "(" + nowyear + "Q" + now_num + ") " + re_title + " - 第" + resert_num + "話 " + flag + sub_content;
result_context.Text += clb_string + " 1280x720 x264 \r\n";
}
else if (now_radio.Checked)
{ //현재
DateTime time_1 = new DateTime(Convert.ToInt32(result_ani_date.Split('年')[0]), Convert.ToInt32(result_ani_date.Split('年')[1]), Convert.ToInt32(now_date[i]), 0, 0, 0);
//result_context.Text += now_date_text.Text.Split('-')[0]+ now_date_text.Text.Split('-')[1]+ now_date_text.Text.Split('-')[2];
int year = Convert.ToInt32(DateTime.Now.ToString("yyyy"));
int month = Convert.ToInt32(DateTime.Now.ToString("MM"));
int day = Convert.ToInt32(DateTime.Now.ToString("dd"));
DateTime time_2 = new DateTime(year, month, day, 0, 0, 0);
if (DateTime.Compare(time_1, time_2) <= 0)
{
now_info = 1;
string sub_content = "";
if (subtitle_content[i].ToString() != "")
{
sub_content = "〔 " + subtitle_content[i].ToString() + " 〕";
}
//date_sort.Add(Convert.ToInt32(DateTime.Now.ToString("dd")) - Convert.ToInt32(now_date[i])+ "(" + nowyear + "Q" + now_num + ") " + re_title + " - 第" + resert_num + "話 " + flag + sub_content + clb_string + " 1280x720 x264");
result_context.Text += "(" + nowyear + "Q" + now_num + ") " + re_title + " - 第" + resert_num + "話 " + flag + " " + sub_content;
result_context.Text += clb_string + " 1280x720 x264 \r\n";
}
}
else //미래
{
DateTime time_1 = new DateTime(Convert.ToInt32(result_ani_date.Split('年')[0]), Convert.ToInt32(result_ani_date.Split('年')[1]), Convert.ToInt32(now_date[i]), 0, 0, 0);
//result_context.Text += now_date_text.Text.Split('-')[0]+ now_date_text.Text.Split('-')[1]+ now_date_text.Text.Split('-')[2];
int year = Convert.ToInt32(DateTime.Now.ToString("yyyy"));
int month = Convert.ToInt32(DateTime.Now.ToString("MM"));
int day = Convert.ToInt32(DateTime.Now.ToString("dd"));
DateTime time_2 = new DateTime(year, month, day, 0, 0, 0);
if (DateTime.Compare(time_1, time_2) > 0)
{
string sub_content = "";
if (subtitle_content[i].ToString() != "")
{
sub_content = "〔 " + subtitle_content[i].ToString() + " 〕";
}
result_context.Text += "(" + nowyear + "Q" + now_num + ") " + re_title + " - 第" + resert_num + "話 " + flag + sub_content;
result_context.Text += clb_string + " 1280x720 x264 \r\n";
}
}
}
}
if (now_info == 1)
{
string[] tempArray = result_context.Lines;
result_context.Text = tempArray[result_context.Lines.Length - 2];
now_info = 0;
}
if (result_context.Text == "")
{
MessageBox.Show("추출결과가 없습니다.");
}
//result_context.Text += subtitle_.Count;
}
public void start_conversion_time()
{
result_context.Text = "";
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
string html = wc.DownloadString(url.Text);
agi.HtmlDocument doc = new agi.HtmlDocument();
doc.LoadHtml(html);
agi.HtmlNode id_main = doc.GetElementbyId("main");
//제목 타이틀 추출
string title = id_main.SelectSingleNode("//h1").InnerText; //제목
string re_title = "";
String result_title;
//MessageBox.Show(title.Split('年')[0].ToString());
if (title.Contains("» "))
{
result_title = "";
if (title.Contains("年"))
{
result_title = title.Split('年')[0].Substring(0, title.Split('年')[0].LastIndexOf(' ')).ToString();
}
if (title.Contains("&"))
{
result_title = title.Split('&')[0].ToString().Trim();
}
/*for (int r = 0; r < result_title.Length; r++)
* {
* if (result_title[r] != "»")
* {
* if (r != 0) re_title += " ";
* //title = result_title[r];
* re_title += result_title[0];
* }
* else
* {
* break;
* }
* }*/
re_title = result_title;
}
else
{
re_title = title;
}
//re_title==========================================
//날짜분기추출
int now_num = 1;
int data_i = 1;
String[] now_split = new String[] { };
var result_date = "";
//실제날짜 추출
var ani_date = doc.DocumentNode.SelectSingleNode("//table[@class='section basic']//tr/td[1]").InnerHtml;
String result_ani_date = ani_date.ToString().Split('月')[0];
//result_context.Text = ani_date.ToString().Split('月')[0] + "\r\n";
for (;;)
{
result_date = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText; //날짜 분기
if (data_i == 3)
{
if (result_date.Contains('～'))
{
now_split = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText.Split('～');
break;
}
}
if (data_i == 4)
{
if (result_date.Contains('～'))
{
now_split = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText.Split('～');
break;
}
}
if (data_i == 5)
{
if (result_date.Contains('～'))
{
now_split = doc.DocumentNode.SelectSingleNode("//table[@class='data']//tr[" + data_i + "]/td[1]").InnerText.Split('～');
break;
}
}
data_i++;
}
String[] now_split2 = now_split[0].Split('-');
if (Convert.ToInt32(now_split2[1]) == 4)
{
now_num = 2;
}
if (Convert.ToInt32(now_split2[1]) == 7)
{
now_num = 3;
}
if (Convert.ToInt32(now_split2[1]) == 10)
{
now_num = 4;
}
String nowyear = now_split2[0];
//nowyear+now_num==============================
//flag_ 추출
ArrayList flag_ = new ArrayList();
foreach (var cell in doc.DocumentNode.SelectNodes("//table[@id='ProgList']/tr/td[6]"))
{
//MessageBox.Show(cell.InnerText);
//result_context.Text += cell.InnerText.Replace("！", "").Replace("●", "").Replace("再", "").Replace("Update", "").Replace(" ", "").Trim() + "\r\n";
flag_.Add(" [" + cell.InnerText.Replace("！", "").Replace("●", "").Replace("注", "").Replace("再", "").Replace("Update", "").Replace(" ", "").Trim() + "]");
}
//flag_============================
//ch_ 추출 ==============================
ArrayList ch_ = new ArrayList();
foreach (var cell in doc.DocumentNode.SelectNodes("//table[@id='ProgList']/tr/td[5]"))
{
if (cell.InnerText != " ")
{
if (cell.SelectSingleNode(".//div[@class='peNotice']") != null)
{
if (cell.SelectSingleNode(".//div[@class='peNotice']").InnerText != "")
{
ch_.Add(cell.InnerText.Replace(cell.SelectSingleNode(".//div[@class='peNotice']").InnerText, "").Replace("!", "！").Replace("?", "？"));
}
else
{
ch_.Add(cell.InnerText.Replace("!", "！").Replace("?", "？"));
}
}
else if (cell.SelectSingleNode(".//div[@class='peComment']") != null)
{
if (cell.SelectSingleNode(".//div[@class='peComment']").InnerText != "")
{
ch_.Add(cell.InnerText.Replace(cell.SelectSingleNode(".//div[@class='peComment']").InnerText, "").Replace("!", "！").Replace("?", "？"));
}
else
{
ch_.Add(cell.InnerText.Replace("!", "！").Replace("?", "？"));
}
}
else
{
ch_.Add(cell.InnerText.Replace("!", "！").Replace("?", "？"));
}
}
}
//ch_ ========================================
//날짜==============
ArrayList day_date = new ArrayList();
ArrayList time_date = new ArrayList();
foreach (var cell in doc.DocumentNode.SelectNodes("//table[@id='ProgList']/tr/td[2]"))
{
day_date.Add(cell.InnerText.Split('(')[0]);
time_date.Add(cell.InnerText.Split(')')[1].Replace(" ", "").Replace(" ", ""));
}
//t_date=================
string clb_string = "";
for (int x = 0; x <= brocas.CheckedItems.Count - 1; x++)
{
clb_string = clb_string + brocas.CheckedItems[x].ToString();
}
int cell_num = 0;
//ArrayList Rs_Array_LIst = new ArrayList();
Dictionary <int, string> Rs_Array_LIst = new Dictionary <int, string>();
foreach (var cell in doc.DocumentNode.SelectNodes("//table[@id='ProgList']/tr/td[4]"))
{
System.Windows.Forms.Application.DoEvents();
string numbers = cell.InnerText;
//MessageBox.Show(time_date[cell_num].ToString());
string minute_date = "";
if (time_date[cell_num].ToString().Split(':')[1].Contains('↑'))
{
minute_date = time_date[cell_num].ToString().Split(':')[1].Split('↑')[0];
}
else if (time_date[cell_num].ToString().Split(':')[1].Contains('↓'))
{
minute_date = time_date[cell_num].ToString().Split(':')[1].Split('↓')[0];
}
else
{
minute_date = time_date[cell_num].ToString().Split(':')[1];
}
int hours = 0;
if (Convert.ToInt32(time_date[cell_num].ToString().Split(':')[0]) < 24)
{
hours = Convert.ToInt32(time_date[cell_num].ToString().Split(':')[0]);
}
else
{
hours = Convert.ToInt32(time_date[cell_num].ToString().Split(':')[0]) - 24;
}
string flag = "";
if (flag_[cell_num].ToString() == " []")
{
flag = " ";
}
else
{
flag = flag_[cell_num].ToString() + " ";
}
//MessageBox.Show(Convert.ToInt32(day_date[cell_num].ToString().Split('-')[0])+" "+ Convert.ToInt32(day_date[cell_num].ToString().Split('-')[1])+" "+ Convert.ToInt32(day_date[cell_num].ToString().Split('-')[2])+" "+ Convert.ToInt32(time_date[cell_num].ToString().Split(':')[0])+" "+ Convert.ToInt32(minute_date));
DateTime time_1 = new DateTime(Convert.ToInt32(day_date[cell_num].ToString().Split('-')[0]), Convert.ToInt32(day_date[cell_num].ToString().Split('-')[1]), Convert.ToInt32(day_date[cell_num].ToString().Split('-')[2]), hours, Convert.ToInt32(minute_date), 0);
int year = Convert.ToInt32(DateTime.Now.ToString("yyyy"));
int month = Convert.ToInt32(DateTime.Now.ToString("MM"));
int day = Convert.ToInt32(DateTime.Now.ToString("dd"));
int hour = Convert.ToInt32(DateTime.Now.ToString("HH"));
int minute = Convert.ToInt32(DateTime.Now.ToString("mm"));
DateTime time_2 = new DateTime(year, month, day, hour, minute, 0);
int result = DateTime.Compare(time_1, time_2);
if (result > 0)
{
cell_num++;
continue;
}
if (numbers == "")
{
cell_num++;
continue;
}
if (Convert.ToInt32(numbers) < 10)
{
numbers = "0" + cell.InnerText;
}
if (result_context.Text.Contains(" - 第" + numbers + "話 " + ch_[cell_num]))
{
cell_num++;
continue;
}
string ch_str = "〔 " + ch_[cell_num] + " 〕 ";
if (ch_str == "〔 〕 ")
{
ch_str = "";
}
try
{
Rs_Array_LIst.Add(Convert.ToInt32(numbers), "(" + nowyear + "Q" + now_num + ") " + re_title + " - 第" + numbers + "話" + flag + "" + ch_str + clb_string + " 1280x720 x264" + "\r\n");
}
catch { }
//result_context.AppendText("(" + nowyear + "Q" + now_num + ") " + re_title + " - 第" + numbers + "話 "+ ch_str + clb_string+" 1280x720 x264"+ "\r\n");
cell_num++;
}
//Rs_Array_LIst.Keys.Sort();
foreach (String re_line in Rs_Array_LIst.Values)
{
Application.DoEvents();
result_context.AppendText(re_line);
}
}
public static BookModels GetBookDetails(string url)
{
var book = new BookModels();
book.AmazonUrl = url;
var webGet = new HtmlWeb();
var htmlDoc = webGet.Load(url);
htmlDoc.OptionFixNestedTags = true;
// ParseErrors is an ArrayList containing any errors from the Load statement
if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0)
{
// Handle any parse errors as required
}
if (htmlDoc.DocumentNode != null)
{
HtmlAgilityPack.HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
if (bodyNode != null)
{
var image = bodyNode.SelectSingleNode("//img[@id='main-image']");
if (image == null)
{
image = bodyNode.SelectSingleNode("//img[@id='imgBlkFront']");
}
if (image != null)
{
book.CoverUrl = image.Attributes["src"].Value;
book.CoverUrl = book.CoverUrl.Substring(0, book.CoverUrl.IndexOf("._")) + ".jpg";
}
var title = bodyNode.Descendants("span")
.Where(x => x.Id == "btAsinTitle")
.Select(s => s.InnerText);
book.Title = title.FirstOrDefault();
if (string.IsNullOrEmpty(book.Title))
{
title = bodyNode.Descendants("h1")
.Where(x => x.Id == "title")
.Select(s => s.InnerText);
book.Title = title.FirstOrDefault();
}
var price = bodyNode.SelectSingleNode("//b[@class='priceLarge']");
if (price != null)
{
book.Price = Convert.ToDecimal(price.InnerText.Trim().Replace("$", string.Empty).Replace("\n", string.Empty));
}
var description = bodyNode.SelectSingleNode("//div[@id='postBodyPS']")
.InnerText;
book.Description = description;
}
}
return(book);
}
/// <summary>
/// Search Shelfari for series info, scrape series page, and return next title in series.
/// </summary>
/// <param name="searchHtmlDoc">Book's Shelfari page, pre-downloaded</param>
private string GetNextInSeriesTitle2(HtmlAgilityPack.HtmlDocument searchHtmlDoc)
{
bool hasSeries = false;
string series = "";
string seriesShort = "";
string seriesURL = "";
int currentSeriesIndex = 0;
int currentSeriesCount = 0;
string nextTitle = "";
//Check if book's Shelfari page contains series info
HtmlAgilityPack.HtmlNode node = searchHtmlDoc.DocumentNode.SelectSingleNode("//span[@class='series']");
if (node != null)
{
//Series name and book number
series = node.InnerText.Trim();
//Convert book number string to integer
Int32.TryParse(series.Substring(series.LastIndexOf(" ") + 1), out currentSeriesIndex);
//Parse series Shelfari URL
seriesURL = node.SelectSingleNode("//span[@class='series']/a[@href]")
.GetAttributeValue("href", "");
seriesShort = node.FirstChild.InnerText.Trim();
//Add series name and book number to log, if found
searchHtmlDoc.LoadHtml(HttpDownloader.GetPageHtml(String.Format(seriesURL)));
//Parse number of books in series and convert to integer
node = searchHtmlDoc.DocumentNode.SelectSingleNode("//h2[@class='f_m']");
string test = node.FirstChild.InnerText.Trim();
Match match = Regex.Match(test, @"\d+");
if (match.Success)
{
Int32.TryParse(match.Value, out currentSeriesCount);
}
hasSeries = true;
//Check if there is a next book
if (currentSeriesIndex < currentSeriesCount)
{
//Add series name and book number to log, if found
main.Log(String.Format("This is book {0} of {1} in the {2} Series...",
currentSeriesIndex, currentSeriesCount, seriesShort));
foreach (HtmlAgilityPack.HtmlNode seriesItem in
searchHtmlDoc.DocumentNode.SelectNodes(".//ol/li"))
{
node = seriesItem.SelectSingleNode(".//div/span[@class='series bold']");
if (node != null)
{
if (node.InnerText.Contains((currentSeriesIndex + 1).ToString()))
{
node = seriesItem.SelectSingleNode(".//h3/a");
//Parse title of the next book
nextTitle = node.InnerText.Trim();
//Add next book in series to log, if found
main.Log(String.Format("The next book in this series is {0}!", nextTitle));
return(nextTitle);
}
}
}
}
if (hasSeries)
{
return("");
}
}
return("");
}
/// <summary>
/// Search Shelfari page for possible series info, returning the next title in the series without downloading any other pages.
/// TODO: Un-yuckify all the return paths without nesting a ton of ifs
/// </summary>
/// <param name="searchHtmlDoc">Book's Shelfari page, pre-downloaded</param>
private string GetNextInSeriesTitle(HtmlAgilityPack.HtmlDocument searchHtmlDoc)
{
//Added estimated reading time and page count from Shelfari, for now...
HtmlAgilityPack.HtmlNode pageNode = searchHtmlDoc.DocumentNode.SelectSingleNode("//div[@id='WikiModule_FirstEdition']");
if (pageNode == null)
{
return("");
}
HtmlAgilityPack.HtmlNode node1 = pageNode.SelectSingleNode(".//div/div");
if (node1 == null)
{
return("");
}
//Parse page count and multiply by average reading time
Match match1 = Regex.Match(node1.InnerText, @"Page Count: ((\d+)|(\d+,\d+))");
if (match1.Success)
{
double minutes = int.Parse(match1.Groups[1].Value, NumberStyles.AllowThousands) * 1.2890625;
TimeSpan span = TimeSpan.FromMinutes(minutes);
main.Log(String.Format("Typical time to read: {0} hours and {1} minutes ({2} pages)"
, span.Hours, span.Minutes, match1.Groups[1].Value));
curBook.pagesInBook = match1.Groups[1].Value;
curBook.readingHours = span.Hours.ToString();
curBook.readingMinutes = span.Minutes.ToString();
}
//Added highlighted passage from Shelfari, dummy info for now...
HtmlAgilityPack.HtmlNode members = searchHtmlDoc.DocumentNode.SelectSingleNode("//ul[@class='tabs_n tn1']");
int highlights = 0;
if (members != null)
{
Match match3 = Regex.Match(members.InnerText, @"Reviews \(((\d+)|(\d+,\d+))\)");
if (match3.Success)
{
curBook.popularPassages = match3.Groups[1].Value.ToString();
}
match3 = Regex.Match(members.InnerText, @"Readers \(((\d+)|(\d+,\d+))\)");
if (match3.Success)
{
curBook.popularHighlights = match3.Groups[1].Value.ToString();
highlights = int.Parse(match3.Groups[1].Value, NumberStyles.AllowThousands);
}
string textPassages = curBook.popularPassages == "1"
? String.Format("{0} passage has ", curBook.popularPassages)
: String.Format("{0} passages have ", curBook.popularPassages);
string textHighlights = curBook.popularHighlights == "1"
? String.Format("{0} time", curBook.popularHighlights)
: String.Format("{0} times", curBook.popularHighlights);
main.Log(String.Format("Popular Highlights: {0}been highlighted {1}"
, textPassages, textHighlights));
}
//If no "highlighted passages" found from Shelfari, add to log
if (highlights == 0)
{
main.Log("Popular Highlights: No highlighted passages have been found for this book");
curBook.popularPassages = "";
curBook.popularHighlights = "";
}
//Check if book series is available and displayed in Series & Lists on Shelfari page.
HtmlAgilityPack.HtmlNode seriesNode = searchHtmlDoc.DocumentNode.SelectSingleNode("//div[@id='WikiModule_Series']/div");
if (seriesNode != null)
{
//If multiple Series found, find and use standard series.
foreach (HtmlAgilityPack.HtmlNode seriesType in
seriesNode.SelectNodes(".//div"))
{
if (seriesType.InnerText.Contains("(standard series)", StringComparison.OrdinalIgnoreCase) && !seriesType.InnerText.Contains("(Reading Order)", StringComparison.OrdinalIgnoreCase))
{
Match match = Regex.Match(seriesType.InnerText, @"This is book (\d+) of (\d+)");
if (!match.Success)
{
continue;
}
curBook.seriesName = seriesType.ChildNodes["a"].InnerText.Trim();
main.Log("About the series: " + seriesType.InnerText.Replace(". (standard series)", ""));
if (!match.Success || match.Groups.Count != 3)
{
return("");
}
curBook.seriesPosition = match.Groups[1].Value;
curBook.totalInSeries = match.Groups[2].Value;
HtmlAgilityPack.HtmlNode seriesInfo = seriesNode.SelectSingleNode(".//p");
//Parse preceding book
if (seriesInfo != null && seriesInfo.InnerText.Contains("Preceded by ", StringComparison.OrdinalIgnoreCase))
{
match = Regex.Match(seriesInfo.InnerText, @"Preceded by (.*),", RegexOptions.IgnoreCase);
if (match.Success && match.Groups.Count == 2)
{
previousTitle = match.Groups[1].Value;
}
else
{
match = Regex.Match(seriesInfo.InnerText, @"Preceded by (.*)\.", RegexOptions.IgnoreCase);
if (match.Success && match.Groups.Count == 2)
{
previousTitle = match.Groups[1].Value;
}
}
main.Log("Preceded by: " + previousTitle);
//Grab Shelfari Kindle edition link for this book
previousShelfariUrl = seriesInfo.ChildNodes["a"].GetAttributeValue("href", "") +
"/editions?binding=Kindle";
}
// Check if book is the last in the series
if (!curBook.seriesPosition.Equals(curBook.totalInSeries))
{
//Parse following book
if (seriesInfo != null && seriesInfo.InnerText.Contains("followed by ", StringComparison.OrdinalIgnoreCase))
{
match = Regex.Match(seriesInfo.InnerText, @"followed by (.*)\.", RegexOptions.IgnoreCase);
if (match.Success && match.Groups.Count == 2)
{
main.Log("Followed by: " + match.Groups[1].Value);
//Grab Shelfari Kindle edition link for this book
nextShelfariUrl = seriesInfo.ChildNodes["a"].GetAttributeValue("href", "") + "/editions?binding=Kindle";
return(match.Groups[1].Value);
}
}
}
//Stop after first standard series is found maybe
//add popup (already started implimentaton) in
//future to pick which standard series you
//want to use, not sure if worthwhile though.
//eg: http://www.shelfari.com/books/37598923
break;
}
}
}
return("");
}
private void textBox1_KeyDown(object sender, KeyEventArgs e)
{
if (e.KeyCode == Keys.Enter)
{
String serial = "INVALID_SERIAL";
if (textBox1.Text.Length > 0)
{
serial = textBox1.Text;
}
try
{
webBrowser1.AllowNavigation = true;
string parsedHTML = "";
string movieTitle = "UNKNOWN";
string imgSrc = "UNKNOWN";
//string html = new WebClient().DownloadString("https://www.google.com/search?q=" + serial);
//parsedHTML = parseHTML(html);
//richTextBox1.Text = html;
//webBrowser1.DocumentText = html;
HtmlWeb web = new HtmlWeb();
document = web.Load("https://www.google.com/search?q=" + serial);
HtmlAgilityPack.HtmlNode bodyNode = document.DocumentNode.SelectSingleNode("//td[@id='rhs_block']");
richTextBox1.Text = document.DocumentNode.InnerHtml;
if (bodyNode.InnerText.Length > 1)
{
webBrowser1.DocumentText = bodyNode.InnerHtml;
foreach (var image in bodyNode.SelectNodes(".//img"))
{
var src = image.GetAttributeValue("src", null);
if (src != null)
{
imgSrc = src;
break;
}
}
textBox2.Enabled = false;
normalQuery = true;
try
{
movieTitle = bodyNode.SelectSingleNode("//div[@class='_B5d']").InnerText;
}
catch (Exception ex)
{
textBox2.Enabled = true;
}
}
else
{
normalQuery = false;
textBox2.Enabled = true;
webBrowser1.Navigate("https://www.google.com/search?q=" + serial);// = new WebClient().DownloadString("https://www.google.com/search?q=" + serial); }
if (list.Count == 0)
{
list.Add(new String[] { serial + " (1)", movieTitle, imgSrc });
listView1.Items.Add(serial + " (1)");
listBox2.Items.Add(movieTitle);
}
else
{
for (int i = 0; i < list.Count; ++i)
{
if (list[i][0].ToString().Contains(serial))
{
int count = Int32.Parse(list[i][0].ToString().Substring(list[i][0].ToString().IndexOf('(')).Substring(1, list[i][0].ToString().Substring(list[i][0].ToString().IndexOf('(')).Length - 2));
list[i][0] = list[i][0].ToString().Substring(0, list[i][0].ToString().IndexOf('(')) + "(" + ++count + ")";
listView1.Items.Clear();
listBox2.Items.Clear();
foreach (String[] j in list)
{
listView1.Items.Add(j[0]);
listBox2.Items.Add(j[1]);
}
i = list.Count;
break;
}
else if (i == list.Count - 1)
{
list.Add(new String[] { serial + " (1)", movieTitle, imgSrc });
listView1.Items.Add(serial + " (1)");
listBox2.Items.Add(movieTitle);
i = list.Count;
break;
}
}
}
}
}
catch (Exception ex)
{
textBox1.Text = "INVALID SERIAL";
}
textBox1.Text = "";
e.SuppressKeyPress = true;
}
}
public void Crawler()
{
int previousPageGallNum = 1000000000;
Console.WriteLine(initDate.ToString() + endDate.ToString());
string url = gallUrl + "&page=";
var client = new WebClient();
client.Encoding = System.Text.Encoding.UTF8;
//Dictionary value => count, replyNum, gallCount, gallRecommend
Dictionary <UserInfo, int[]> userDic = new Dictionary <UserInfo, int[]>();
int currentPage = this.initPage;
while (true)
{
string text;
try
{
text = client.DownloadString(url + currentPage.ToString());
if (string.IsNullOrEmpty(text))
{
continue;
}
}
catch
{
continue;
}
hap.HtmlDocument textHap = new hap.HtmlDocument();
textHap.LoadHtml(text);
hap.HtmlNodeCollection nicks = textHap.DocumentNode.SelectNodes("//tr[@class='ub-content us-post']");
//Console.WriteLine(nicks.Count);
//Console.WriteLine("==================" + currentPage.ToString() + "==================");
try
{
foreach (hap.HtmlNode nick in nicks)
{
int gallNum, replyNum, gallCount, gallRecommend;
DateTime gallDate; string subject;
gallNum = GetOnlyInt(nick.SelectSingleNode("./td[@class='gall_num']").InnerText);
gallDate = DateTime.ParseExact(nick.SelectSingleNode("./td[@class='gall_date']").Attributes["title"].Value,
"yyyy-MM-dd HH:mm:ss", null);
Console.WriteLine(gallNum.ToString() + " " + gallDate.ToString());
if (gallNum >= previousPageGallNum)
{
Console.WriteLine(previousPageGallNum.ToString() + " " + gallNum.ToString());
Console.WriteLine("번호 에러");
continue;
}
if (DateTime.Compare(gallDate, initDate) < 0 || DateTime.Compare(gallDate, endDate) > 0)
{
Console.WriteLine("날짜 에러");
continue;
}
hap.HtmlNode user = nick.SelectSingleNode("./td[@class='gall_writer ub-writer']");
UserInfo tempUserInfo = new UserInfo(user.Attributes["data-nick"].Value);
if (user.Attributes["data-uid"].Value == "")
{
tempUserInfo.setFluidNick(user.Attributes["data-ip"].Value);
}
else
{
tempUserInfo.setFixedNick(user.Attributes["data-uid"].Value);
}
//replyNum and subject are in <td class='gall_tit ub-word'></td>
hap.HtmlNode subjectNode = nick.SelectSingleNode("./td[2]");
try
{
if (subjectNode.Attributes["class"].Value == "gall_subject")
{
subjectNode = nick.SelectSingleNode("./td[3]");
}
subject = subjectNode.SelectSingleNode("./a[1]").InnerText;
if (subjectNode.SelectNodes("./a").Count == 2)
{
replyNum = GetOnlyInt(subjectNode.SelectSingleNode("./a[@class='reply_numbox']/span").InnerText);
}
else
{
replyNum = 0;
}
}
catch
{
subject = "NullSubjectException";
replyNum = 0;
}
// Console.WriteLine("댓글: " + replyNum.ToString());
gallCount = GetOnlyInt(nick.SelectSingleNode("./td[@class='gall_count']").InnerText);
gallRecommend = GetOnlyInt(nick.SelectSingleNode("./td[@class='gall_recommend']").InnerText);
//Dictionary value => count, replyNum, gallCount, gallRecommend
if (userDic.ContainsKey(tempUserInfo))
{
userDic[tempUserInfo][0] += 1;
userDic[tempUserInfo][1] += replyNum;
userDic[tempUserInfo][2] += gallCount;
userDic[tempUserInfo][3] += gallRecommend;
}
else
{
int[] tempInts = new int[] { 1, replyNum, gallCount, gallRecommend };
userDic.Add(tempUserInfo, tempInts);
}
UserData tempUserData = new UserData(tempUserInfo);
tempUserData.DataInput(gallNum, replyNum, gallCount, gallRecommend, gallDate, subject);
//gallDatas.Add(tempUserData);
}
}
catch
{
if (ErrorOccured != null)
{
ErrorOccured(text, null);
}
currentPage++;
continue;
}
previousPageGallNum = GetOnlyInt(nicks[nicks.Count - 1].SelectSingleNode("./td[@class='gall_num']").InnerText);
DateTime currentDate = DateTime.ParseExact(nicks[nicks.Count - 1].
SelectSingleNode("./td[@class='gall_date']").Attributes["title"].Value, "yyyy-MM-dd HH:mm:ss", null);
if (currentPage >= endPage || DateTime.Compare(currentDate, initDate) < 0)
{
break;
}
else
{
System.Collections.ArrayList arr = new System.Collections.ArrayList();
string str = currentPage.ToString() + " 페이지, 날짜: " + currentDate.ToString();
arr.Add(str); arr.Add(currentDate); arr.Add(currentPage - initPage);
if (newPageHappened != null)
{
newPageHappened(arr, null);
}
currentPage++;
}
}
//Dictionary value => count, replyNum, gallCount, gallRecommend
foreach (KeyValuePair <UserInfo, int[]> user in userDic)
{
UserInfo tempUser = user.Key;
tempUser.count = user.Value[0];
tempUser.replyNum = user.Value[1];
tempUser.gallCount = user.Value[2];
tempUser.gallRecommend = user.Value[3];
UserRank tempUserRank = new UserRank(tempUser, user.Value[0], user.Value[1], user.Value[2], user.Value[3]);
userList.Add(tempUserRank);
}
var sorted = from userRank in userList
orderby userRank.count descending
select userRank;
userList = sorted.ToList <UserRank>();
if (CrawlingEnded != null)
{
CrawlingEnded(userList, null);
}
string tempDataDir = Directory.GetCurrentDirectory() + "\\temp-data\\";
Directory.CreateDirectory(tempDataDir);
string filename = tempDataDir + gallId + DateTime.Now.ToString("_yyyy-MM-dd_HH-mm-ss");
SaveResult(filename);
}
public static int GetCoverImageUrl(
IWin32Window owner,
string strISSN,
string strYear,
string strIssueNo,
ref CookieContainer cookie,
out string strImageUrl,
out string strError)
{
strError = "";
strImageUrl = "";
string strUrl = "http://xxdy.qikan.com/MagInfo.aspx?issn=" + strISSN + "&year=" + strYear + "&periodNum=" + strIssueNo;
/*
* Connection: Keep-Alive
* Cookie: xxdy=Default|Default_blue|1674-3121|%e4%b8%ad%e5%b0%8f%e5%ad%a6%e5%be%b7%e8%82%b2|2016|8|xxdy|459|||%e5%b0%8f%e5%ad%a6%e5%be%b7%e8%82%b2%e7%bd%91%e7%ab%99%3ahttp%3a%2f%2fxxdy.qikan.com|2016|9|True|ãä¸å°å¦å¾·è²ãæ¯ç±å½å®¶æè²é¨å§æååå¸è大å¦ä¸»åçä¸æ¬å
* ¨é¢åæ ä¸å°å¦å¾·è²å·¥ä½çä¸ä¸æåï¼åæ¶ä½ä¸ºä¸å½æè²å¦ä¼ä¸å°å¦å¾·è²ç 究åä¼ä¼åãæ¬åè´åäºä¸ºä¸å°å¦å¾·è²ç论ç 究ä¸å®è·µå·¥ä½è
* æä¾ææ°çå¾·è²æ¹é©å¨åãæå¨çå¾·è²æ¿çæå¼ãæ°éçå¾·è²ç 究ææãé²æ´»çå¾·è²å®è·µç»éªï¼åäºæ为ä¸å°å¦å¾·è²æ¹é©çâé£åæ âï¼éææ¹é©çâæ头å
* µâï¼å¾·è²å·¥ä½è
* 交æµç»éªãæ¢ç´¢å¾·è²è§å¾çâ大èå°âï¼å¼é¢å¾·è²è¡æ¿é¨é¨ãæç é¨é¨å广大德è²æå¸çâåè°é¨âã%0d%0aç¼ è¾ é¨ï¼020-85215129 85211209%0d%0aå è¡ é¨ï¼020-85215179 85211443ï¼ä¼ çï¼%0d%0açµåé®ç®±ï¼[email protected]ï¼æ稿ï¼%0d%0a [email protected]ï¼è®¢é
* ï¼%0d%0aå客ï¼http://blog.sina.com.cn/s/articlelist_2734759432_0_1.html
*/
if (cookie == null)
{
cookie = new CookieContainer();
}
WebClientEx webClient = new WebClientEx(cookie);
#if NO
{
byte[] byteArray = x.DownloadData(new Uri("http://xxdy.qikan.com"));
}
x = new WebClientEx(cookie);
#endif
webClient.Headers.Add("Accept", "text/html, application/xhtml+xml, image/jxr, */*");
webClient.Headers.Add("Accept-Encoding", "gzip, deflate");
webClient.Headers.Add("Accept-Language", "zh-Hans-CN, zh-Hans; q=0.8, en-US; q=0.5, en; q=0.3");
// Host: xxdy.qikan.com
webClient.Headers.Add("Host", "xxdy.qikan.com");
// x.BaseAddress = "xxdy.qikan.com";
webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393");
try
{
#if NO
byte[] byteArray = x.DownloadData(new Uri(strUrl));
Stream stream = new MemoryStream(byteArray);
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.Load(stream, true);
// ParseErrors is an ArrayList containing any errors from the Load statement
if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0)
{
// Handle any parse errors as required
strError = "parse html error: " + htmlDoc.ParseErrors.ToString();
return(-1);
}
#endif
byte[] byteArray = webClient.DownloadData(new Uri(strUrl));
string strContent = Encoding.UTF8.GetString(byteArray);
// string strContent = x.DownloadString(new Uri(strUrl));
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(strContent);
#if NO
// ParseErrors is an ArrayList containing any errors from the Load statement
if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0)
{
// Handle any parse errors as required
strError = "parse html error: " + htmlDoc.ParseErrors.ToString();
return(-1);
}
#endif
if (htmlDoc.DocumentNode == null)
{
strError = "htmlDoc.DocumentNode == null";
return(-1);
}
/*
* <div class="left1">
* <!--最新封面开始-->
*
*
* <div class="cover1">
* <h1>封面</h1>
* <div class="cover1_box">
* <a href="../../MagInfo.aspx?issn=1674-3121&year=2013&periodNum=7"><img src="http://img.qikan.com.cn/qkimages/xxdy/xxdy201307-l.jpg" width="190" height="270" border="0" alt="2013年第7期" /></a>
* <span class="f14 fBold"><a href="MagInfo.aspx?issn=1674-3121&year=2013&periodNum=7" title="2013年第7期">2013年第7期</a>
* </span>
*
* */
#if NO
HtmlNodeCollection nodes = htmlDoc.DocumentNode.SelectNodes("//img");
foreach (HtmlNode node in nodes)
{
string src1 = node.GetAttributeValue("src", "");
int i = 0;
i++;
}
#endif
HtmlAgilityPack.HtmlNode cover1_box = htmlDoc.DocumentNode.SelectSingleNode("//div[@class='cover1_box']");
if (cover1_box == null)
{
strError = "cover1_box 没有找到";
return(-1);
}
HtmlNode img = cover1_box.SelectSingleNode("*/img");
string src = img.GetAttributeValue("src", "");
strImageUrl = src;
return(1);
}
catch (Exception ex)
{
strError = "异常: " + ex.Message;
return(-1);
}
}
public override void Parse(Response response)
{
//Create a new HTMLAglityPack document
HtmlDocument ContentDocument = new HtmlDocument();
//load the #content of the page into the document
ContentDocument.LoadHtml(response.Css("#content").First().OuterHtml);
HtmlAgilityPack.HtmlNode BodyNode = ContentDocument.DocumentNode;
patternObject.Title = BodyNode.SelectSingleNode("//*[@id=\"firstHeading\"]").InnerHtml;
HtmlAgilityPack.HtmlNode ContentNode = BodyNode.SelectSingleNode("//*[@id=\"mw-content-text\"]");
//remove the "toc" and "jump" and "siteSub" sections to save space and later client-side processing time
if (ContentNode.SelectSingleNode("//*[@id=\"toc\"]") != null)
{
ContentNode.SelectSingleNode("//*[@id=\"toc\"]").Remove();
}
foreach (var node in ContentNode.SelectNodes("//comment()"))
{
node.Remove();
}
ContentNode.PrependChild(BodyNode.SelectSingleNode("//*[@id=\"firstHeading\"]"));
//set the patternObject's title
patternObject.Title = ContentNode.SelectSingleNode("//*[@id=\"firstHeading\"]").InnerHtml;
foreach (var link in ContentNode.SelectNodes("//a/@href"))
{
//skip if this is a redlink (page doesn't exist).
if (link.Attributes["href"].Value.Contains("redlink=1"))
{
continue;
}
//skip if this links to this page
if (link.Attributes["href"].Value.Split('#').First() == response.FinalUrl)
{
continue;
}
//if any of the links ancestor nodes is the "category links" part of the page
if (link.Ancestors().Any(node => node.Id == "catlinks"))
{
if (link.InnerText != "Categories") //if it is not the "categories" special page
{
//add it to the patterns list of categories
patternObject.Categories.Add(link.InnerText);
}
}
else //assume its a normal text-body link
{
//check if we don't already know about this link
patternObject.CreateOrGetPatternLink(link.InnerText);
}
//add relation info if this is a relation link
if (GetNodeReleventPageHeading(link, "h2") != null &&
GetNodeReleventPageHeading(link, "h2").InnerText == "Relations")
{
//get the relation type of this relation and get its inner text
HtmlAgilityPack.HtmlNode RelationHeadingNode = GetNodeReleventPageHeading(link, "h3");
String RelationName = RelationHeadingNode.InnerText;
//if there is a h4 node before the previous h3 node
if (GetNodeReleventPageHeading(link, "h4") != null &&
RelationHeadingNode.InnerStartIndex < GetNodeReleventPageHeading(link, "h4").InnerStartIndex)
{
//assume it is a "with x" sub-category of relation for the "Can Instantiate" section
RelationName = RelationHeadingNode.InnerText + " " + GetNodeReleventPageHeading(link, "h4").InnerText;
}
//add the relevent relation to this link
patternObject.CreateOrGetPatternLink(link.InnerText).Type.Add(RelationName);
}
}
//get a cleaned copy of the #content HTML for giving in the JSON data
patternObject.Content = ProcessPageContentToString(ContentNode);
string Json = JsonConvert.SerializeObject(patternObject);
File.WriteAllText(Pattern.GetFileName(patternObject.Title), Json);
}
/// <summary>
/// return single node using _LabelData.XPath
/// </summary>
/// <returns></returns>
protected HtmlAgilityPack.HtmlNode GetCurrentNode()
{
HtmlAgilityPack.HtmlNode ndTempParent = GetTempParentNode();
return(ndTempParent.SelectSingleNode(_LabelData.XPath));
}
