public async Task<string> ParseHtmlAsync(string tenant, string html)
{
var doc = new HtmlDocument();
doc.LoadHtml(html);
var nodes = doc.DocumentNode.SelectNodes("//include[@article-alias and @category-alias]");
if (nodes == null)
{
return html;
}
foreach (var node in nodes)
{
string alias = node.Attributes["article-alias"].Value;
string categoryAlias = node.Attributes["category-alias"].Value;
var model = await ContentModel.GetContentAsync(tenant, categoryAlias, alias).ConfigureAwait(false);
if (model != null)
{
string contents = model.Contents;
var newNode = HtmlNode.CreateNode(contents);
node.ParentNode.ReplaceChild(newNode, node);
}
}
return doc.DocumentNode.OuterHtml;
}
public HeroDataConverter()
{
agilityPackHelper = new AgilityPackHelper();
HtmlDocument doc = new HtmlDocument();
heroesName = GetHeroesName();
foreach (var heroName in heroesName)
{
doc = LoadHeroHtmlPage(heroName);
skillImages = GetSkillPortraits(doc);
skillNames = GetSkillNames(doc);
skillDescriptions = GetSkillDescriptions(doc);
primaryStatsImages = GetPrimaryStatsImages(doc);
primaryStatsValues = GetPrimaryStatsValues(doc);
biography = GetBiography(doc).Trim() ;
manaCostDictionary = GetManaCost(doc);
coolDownList = GetCoolDown(doc);
abilityCastType = GetAbilityCastType(doc);
skillTargetAffectedType = GetSkillTargetAffectedType(doc);
skillDamageType = GetSkillDamageType(doc);
skillVideo = GetSkillVideo(doc);
skillRemainingValues = GetSkillRemainingValues(doc);
heroCreator.createHero(heroName, biography);
for (int i = 0; i < skillNames.Count; i++)
{
heroCreator.createHeroSkill(skillNames[i], skillDescriptions[i], manaCostDictionary, coolDownList, abilityCastType[i],
skillTargetAffectedType[i], skillDamageType[i], skillVideo);
}
}
}
public static HtmlDocument CreateRequest(string url)
{
using (var client = new HttpClient())
{
try
{
var response = client.GetAsync(url).Result;
if (response.StatusCode == HttpStatusCode.OK)
{
var responseContent = response.Content.ReadAsStringAsync().Result;
var doc = new HtmlDocument();
doc.LoadHtml(responseContent);
return doc;
}
else
{
throw new Exception(response.StatusCode.ToString());
}
}
catch (Exception e)
{
throw new Exception("Error executing query " + url + ". " + e.Message + " " + e.InnerException);
}
}
}
public IEnumerable<HtmlForm> GetForms()
{
var doc = new HtmlDocument();
doc.LoadHtml(Content);
var forms = doc.DocumentNode.SelectNodes("//form");
return forms.Select(HtmlForm.FromNode);
}
private void btnTestCode_Click(object sender, RoutedEventArgs e)
{
var mainPage = GetHtml("http://htmlagilitypack.codeplex.com");
var homepage = new HtmlDocument();
homepage.LoadHtml(mainPage);
var nodes =
homepage.DocumentNode.Descendants("a").Where(x => x.Id.ToLower().Contains("releasestab")).FirstOrDefault
();
var link = nodes.Attributes["href"].Value;
var dc = new HtmlDocument();
try
{
Cursor = Cursors.Wait;
var req = (HttpWebRequest) WebRequest.Create(link);
using (var resp = req.GetResponse().GetResponseStream())
using (var read = new StreamReader(resp))
{
dc.LoadHtml(read.ReadToEnd());
var span =
dc.DocumentNode.Descendants("span").Where(
x => x.Id.ToLower().Contains("releasedownloadsliteral")).FirstOrDefault();
MessageBox.Show(
int.Parse(span.InnerHtml.ToLower().Replace("downloads", string.Empty).Trim()).ToString());
}
}
catch (Exception ex)
{
MessageBox.Show("Error loading file: " + ex.Message, "Error", MessageBoxButton.OK, MessageBoxImage.Error,
MessageBoxResult.OK);
}
}
public static SteamApp ParsePage(int appId, string html)
{
if (string.IsNullOrWhiteSpace(html)) throw new ArgumentNullException(nameof(html));
try
{
var app = SteamApp.NewSteamApp(appId, html);
var htmlDocument = new HtmlDocument();
var htmlCleaned = html.Replace("\"", "'");
htmlDocument.LoadHtml(htmlCleaned);
var documentNode = htmlDocument.DocumentNode;
var titleNode = documentNode.SelectSingleNode($"//div[@class='{AppTitleClass}']");
app.Title = titleNode.InnerHtml.Trim();
var packageNodes = documentNode.SelectNodes($"//div[@class='{PackageClass}']").ToArray();
foreach (var packageNode in packageNodes)
{
AddPackage(app, packageNode);
}
return app;
}
catch (Exception)
{
throw new InvalidAppException(appId);
}
}
public static void ParseRecommends(string html, Action<List<Recommend>> finished)
{
BackgroundWorker bw = new BackgroundWorker();
bw.DoWork += new DoWorkEventHandler((sender, e) =>
{
List<Recommend> allRecommends = new List<Recommend>();
HtmlDocument hDoc = new HtmlDocument();
hDoc.LoadHtml(html);
var tableRows = hDoc.DocumentNode.SelectNodes(Constants.Instance.XPATH_GAME_SHOW_RESULT);
foreach (var node in tableRows)
{
var results = ChildElementsInTableRow(node);
if (results.Count == Constants.Instance.COUNT_GAME_SHOW_RESULT_COLUMNS)
{
var rec = RecommendFromStrings(results);
if (IsValidRecommend(rec))
{
allRecommends.Add(rec);
}
}
}
finished(allRecommends);
});
bw.RunWorkerAsync();
}
static IDictionary<string, string> GetFundamentals(Uri url)
{
string html;
using (var webClient = new WebClient())
html = webClient.DownloadString(url);
var doc = new HtmlDocument
{
OptionFixNestedTags = true
};
doc.LoadHtml(html);
var fundamentals = new Dictionary<string, string>();
var trs = doc.DocumentNode.SelectNodes("//div[@data-ajax-name='EquitySummaryTable']//table[contains(@class, 'horizontalTable')]//tr");
if (trs == null)
return fundamentals;
foreach (var tr in trs)
{
var name = tr.Elements("th").Single().InnerText;
var value = tr.Elements("td").Single().InnerText;
if (value == "--")
{
Log.DebugFormat("Missing: {0} = {1}", name, value);
continue;
}
Log.DebugFormat("Found: {0} = {1}", name, value);
fundamentals.Add(name, value);
}
return fundamentals;
}
private void LoadHtmlTemplate()
{
this.document = new HtmlDocument();
this.document.Load( Assembly.GetExecutingAssembly().GetManifestResourceStream(HtmlTemplate) );
this.table = new HtmlNodeCollection(document.GetElementbyId(LogEventTableId));
}
private string ReplaceTags(string html)
{
var doc = new HtmlDocument();
doc.LoadHtml(html);
var divs = doc.DocumentNode.Descendants("div").Where(d =>
d.Attributes.Contains("class")).ToList(); //&& d.Attributes["class"].Value.Contains("editable-wrapper"));
var editableDivs = new List<HtmlNode>();
foreach (var div in divs)
{
if(div.Attributes["class"].Value.Contains("editable-wrapper"))
{
editableDivs.Add(div);
}
}
foreach (var editableDiv in editableDivs)
{
editableDiv.AppendChild(GetEditButtonElement());
editableDiv.AppendChild(GetSaveButtonElement());
}
return doc.DocumentNode.OuterHtml;
}
public datascraper()
{
string url = @"http://www.bbc.co.uk/sport/football/results/partial/competition-118996114";
HtmlWeb htmlWeb = new HtmlWeb();
HtmlDocument doc = new HtmlDocument{ OptionUseIdAttribute = true };
doc = htmlWeb.Load(url);
HtmlNodeCollection mtchrslts = doc.DocumentNode.SelectNodes("//tr[@id]");
string date;
string ateam;
string hteam;
string score;
string idmess;
string idnum;
string[] teamscores;
string teamscoreh;
string teamscorea;
foreach (HtmlNode matchresult in mtchrslts)
{
idmess = matchresult.SelectSingleNode("//tr[@id]").Id;
idnum = idmess.Replace("match-row-", "");
score = matchresult.SelectSingleNode("//abbr[@title='Score']").InnerText;
teamscores = score.Split('-');
teamscoreh = teamscores[0];
teamscorea = teamscores[1];
hteam = matchresult.SelectSingleNode("//p[(@class='team-home teams')]").InnerText;
ateam = matchresult.SelectSingleNode("//p[(@class='team-away teams')]").InnerText;
date = matchresult.SelectSingleNode("//td[(@class='match-date')]").InnerText;
}
return;
}
public ItemCrawler(Uri url)
{
_htmlDocument = new HtmlDocument();
var html = new WebClient().DownloadString(url.OriginalString);
_htmlDocument.LoadHtml(html);
_document = _htmlDocument.DocumentNode;
}
/// <summary>
/// 非同步刷新最新資訊
/// </summary>
/// <returns></returns>
public async Task RefreshAsync() {
HttpClient client = new HttpClient();
HtmlDocument HTMLDoc = new HtmlDocument();
HTMLDoc.LoadHtml(await client.GetStringAsync(DataSource));
var script = HTMLDoc.DocumentNode.Descendants("script")
.Where(x => x.InnerHtml?.Length > 0).Select(x => x.InnerHtml).ToArray();
var tempAry = script.First()
.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries)
.Select((x, i) => new { index = i, item = x })
.GroupBy(x => Math.Floor(x.index / 4.0));
this.LastPassed = null;
this.Delay = new TimeSpan();
foreach (var item in tempAry) {
string[] temp = item.Select(x=>x.item).ToArray();
if(temp[3] == "TRSearchResult.push('x')") {
this.LastPassed = await Station.GetStationByNameAsync(
innerString(temp[0],"'","'")
);
}
}
var time = new TimeSpan(0, int.Parse(innerString(script.Last(), "=", ";")),0);
this.Delay= time;
}
protected override void Parse(String page)
{
var doc = new HtmlDocument();
doc.LoadHtml(page);
var nodes = doc.DocumentNode.SelectNodes("//ul[@class='list']//li");
if (nodes != null)
{
foreach (var node in nodes)
{
var sale = false; //из списка нельзя получить инфу о скидке, поэтому всегда false
var title = node.SelectSingleNode(".//div[@class='title']").FirstChild.InnerText.Replace("\r\n", " ");
var searchString = DelBadChar(ref title);
var gameUrl = StoreUrl + node.SelectSingleNode(".//a").GetAttributeValue("href", String.Empty);
var cost = node.SelectSingleNode(".//div[@class='price']//span[@class='new']").InnerText;
_entries.Add(new GameEntry()
{
SearchString = searchString,
StoreUrl = StoreUrl,
Title = title,
GameUrl = gameUrl,
Cost = cost,
Sale = sale
});
}
}
}
public NewsItem ParseItem(HtmlDocument doc, string host)
{
var header = ReadTitle(doc.DocumentNode);
var result = new NewsItem();
result.Header = header;
result.Author = ReadAuthor(doc.DocumentNode);
long date;
result.DatePublished = long.TryParse(ReadPublishedDate(doc.DocumentNode),out date)?date:0;
result.MainPic = ReadMainImage(doc.DocumentNode);
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("skybet")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("betfair")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("bet365")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("williamhill")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("paddypower")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("coral.co.uk")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("betvictor")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("doubleclick")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && d.Attributes["href"].Value.Contains("victor.com")));
RemoveTags(doc.DocumentNode.SelectNodes("//a").Where(d => d.Attributes.Contains("href") && !d.Attributes["href"].Value.Contains("http://")));
RemoveTags(doc.DocumentNode.Descendants().Where(d => d.Attributes.Contains("data-bookmaker-url")));
result.Content = ReadContent(doc.DocumentNode);
result.CleanContent = BaseHelper.ScrubHtml(result.Content);
return result;
}
//Получаем общее число страниц
protected override int GetPageCount(HtmlDocument document, string startAddress)
{
HtmlNodeCollection c = document.DocumentNode.SelectNodes("//div[@class='pageNavBar']");
int pageCount = 1;
if (c != null)
foreach (HtmlNode n in c)
{
HtmlNodeCollection cc = n.ChildNodes;
if (cc != null)
{
int i = 0;
foreach (HtmlNode nn in cc)
{
i++;
if (i == 6)
{
pageCount = int.Parse(Regex.Match(nn.InnerText, @"(\d+)").Groups[1].Value);
}
}
}
break;
}
return pageCount;
}
private async Task<String> GetLinkFromWebPage()
{
WebRequest request = WebRequest.CreateHttp(ZERANOE_WEB);
using (WebResponse response = await request.GetResponseAsync())
using (Stream s = response.GetResponseStream())
{
HtmlDocument doc = new HtmlDocument();
doc.Load(s);
HtmlNodeCollection collection = doc.DocumentNode.SelectNodes("/html/body/div[@id='builds-page']/div[@class='grid-460']/a[@class='latest']");
int bits = Environment.Is64BitProcess ? 64 : 32;
Regex regex = new Regex($"Download FFmpeg git-\\S+ {bits}-bit Static");
foreach (var item in collection)
{
if (regex.IsMatch(item.InnerText))
{
string link = item.GetAttributeValue("href", null);
if (link != null)
{
// Link is not absolute (./win64/ffmpeg.7z)
if (link.StartsWith("."))
link = ZERANOE_WEB + link.Substring(2);
return link;
}
}
}
}
return null;
}
public static void ParseTopPerson(string html, Action<List<Person>> finished)
{
BackgroundWorker bw = new BackgroundWorker();
bw.DoWork += new DoWorkEventHandler((sender, e) =>
{
List<Person> TopPerson = new List<Person>();
HtmlDocument hDoc = new HtmlDocument();
hDoc.LoadHtml(html);
var tableRows = hDoc.DocumentNode.SelectNodes(Constants.Instance.XPATH_GAME_TOP_RESULT);
foreach (var node in tableRows)
{
var results = ChildElementsInTableRow(node);
if (results.Count == Constants.Instance.COUNT_GAME_TOP_RESULT_COLUMNS)
{
var person = TopPersonFromStrings(results);
if (person.Total != 0)
{
TopPerson.Add(person);
}
}
}
finished(TopPerson);
});
bw.RunWorkerAsync();
}
/// <summary>
/// New html document instance
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
public HtmlDocument HtmlDocument(string html)
{
var d = new HtmlDocument();
d.LoadHtml(html);
return d;
}
private void ConvertHtml(ExCSS.Stylesheet sheet, string html, Section section)
{
_sheet = sheet;
if (string.IsNullOrEmpty(html))
{
throw new ArgumentNullException("html");
}
if (section == null)
{
throw new ArgumentNullException("section");
}
//section.PageSetup.HeaderDistance = "0.001cm";
section.PageSetup.FooterDistance = Unit.FromCentimeter(0.01);
// Create a paragraph with centered page number. See definition of style "Footer".
var footer = section.Footers.Primary.AddParagraph();
//section.Footers.Primary.
footer.Format.Alignment = ParagraphAlignment.Right;
footer.AddPageField();
footer.AddText(" of ");
footer.AddNumPagesField();
var doc = new HtmlDocument();
doc.LoadHtml(html);
ConvertHtmlNodes(doc.DocumentNode.ChildNodes, sheet, section);
}
public override StoryInfo RequestInfo(string storyUrl)
{
var html = NetworkHelper.GetHtml(storyUrl);
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(html);
var nameNode = htmlDoc.DocumentNode.SelectSingleNode("//*[@id=\"main\"]//h1");
StoryInfo info = new StoryInfo()
{
Url = storyUrl,
Name = nameNode.ChildNodes[1].InnerText.Trim(),
};
var chapterNodes = htmlDoc.DocumentNode.SelectNodes("//*[@class=\"detail_list\"]//li//a");
foreach (HtmlNode chapter in chapterNodes)
{
ChapterInfo chap = new ChapterInfo()
{
Name = chapter.InnerText.Trim(),
Url = chapter.Attributes["href"].Value,
ChapId = ExtractID(chapter.InnerText.Trim())
};
info.Chapters.Add(chap);
}
info.Chapters = info.Chapters.OrderBy(p => p.ChapId).ToList();
return info;
}
public IList<CalendarEvent> Parse(string input)
{
var doc = new HtmlDocument();
doc.LoadHtml(input);
var result = new List<CalendarEvent>();
var nodes = doc.DocumentNode.SelectNodes("//*[contains(@class,'h-event')]");
if (nodes == null) { return result; }
foreach (var e in nodes)
{
result.Add(new CalendarEvent
{
Url = HtmlNodeHelpers.GetAttributeValue(e.SelectSingleNode("//*[contains(@class,'u-url')]"),"href"),
Name = HtmlNodeHelpers.GetInnerText(e.SelectSingleNode("//*[contains(@class,'p-name')]")),
Category = HtmlNodeHelpers.GetInnerText(e.SelectSingleNode("//*[contains(@class,'p-category')]")),
Description = HtmlNodeHelpers.GetInnerText(e.SelectSingleNode("//*[contains(@class,'p-description')]")),
Summary = HtmlNodeHelpers.GetInnerText(e.SelectSingleNode("//*[contains(@class,'p-summary')]")),
Location = HtmlNodeHelpers.GetInnerText(e.SelectSingleNode("//*[contains(@class,'p-location')]")),
Start = DateTime.Parse(HtmlNodeHelpers.GetAttributeValue( e.SelectSingleNode("//*[contains(@class,'dt-start')]"), "datetime")),
End = DateTime.Parse(HtmlNodeHelpers.GetAttributeValue(e.SelectSingleNode("//*[contains(@class,'dt-end')]"), "datetime")),
});
}
return result;
}
private Result ParseResult(HtmlNode resultSet)
{
string[] courseSplit = _currentCourse.Split(new[] { "<br>" }, StringSplitOptions.RemoveEmptyEntries);
string courseName = courseSplit[0];
string eventName = courseSplit[1];
var doc = new HtmlDocument();
doc.LoadHtml(resultSet.OuterHtml);
IEnumerable<HtmlNode> allResults = doc.DocumentNode.QuerySelectorAll("td");
string position = allResults.ElementAt(0).QuerySelector("b").InnerText;
string name = allResults.ElementAt(1).QuerySelector("b").InnerText;
string jockey = allResults.ElementAt(2).QuerySelector("b").InnerText;
string startingPrice = allResults.ElementAt(3).QuerySelector("b").InnerText.Trim();
var price = new Price{DecimalPrice = 0,Denominator=0,Numerator = 0};
try {
price = new Price(startingPrice);
} catch(ArgumentException) {}
return new Result
{
CourseName = courseName,
EventName = eventName,
Position = position,
HorseName = name,
JockeyName = jockey,
StartingPriceDecimal = price.DecimalPrice,
StartingPriceDenominator = price.Denominator,
StartingPriceNumerator = price.Numerator
};
}
public HeroDataConverter()
{
agilityPackHelper = new AgilityPackHelper();
HtmlDocument doc = new HtmlDocument();
heroesName = GetHeroesName();
foreach (var heroName in heroesName)
{
doc = LoadHeroHtmlPage(heroName);
skillImages = GetSkillPortraits(doc);
skillNames = GetSkillNames(doc);
skillDescriptions = GetSkillDescriptions(doc);
primaryStatsImages = GetPrimaryStatsImages(doc);
primaryStatsValues = GetPrimaryStatsValues(doc);
biography = GetBiography(doc).Trim() ;
manaCostList = GetManaCost(doc);
coolDownList = GetCoolDown(doc);
abilityCastType = GetAbilityCastType(doc);
skillTargetAffectedType = GetSkillTargetAffectedType(doc);
skillDamageType = GetSkillDamageType(doc);
skillVideo = GetSkillVideo(doc);
skillRemainingValues = GetSkillRemainingValues(doc);
heroCreator.createHero(heroName, biography);
sCretor.createSkill(heroId, name, description, manaCostList, coolDownList, abilityCastType, targetAffectedType, damageType, videoUrl);
)
}
}
public IEnumerable<ValidationError> ValidateHtml(HtmlDocument document)
{
var records = new List<ValidationError>();
var headingNodes = new List<HtmlNode>();
PopulateHeadingNodeList(document.DocumentNode, headingNodes);
int currentLevel = 0;
foreach (var headingNode in headingNodes)
{
var expectedLevel = currentLevel + 1;
var foundLevel = int.Parse(headingNode.Name.Replace("h", string.Empty));
if (foundLevel > expectedLevel)
{
var message = string.Format("Illogical heading order: Expected to find <h{0}> but found {1} instead", expectedLevel, headingNode.OuterHtml);
records.Add(new ValidationError(message));
break;
}
currentLevel = foundLevel;
}
return records;
}
public bool Add(string html)
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
HtmlNodeCollection tradeSections = doc.DocumentNode.SelectNodes("//div[@class='trade']");
if (tradeSections == null)
{
return false;
}
foreach (HtmlNode t in tradeSections)
{
Trade trade = new Trade(t);
foreach(Trade tradeIdCheck in tradeList)
{
if (tradeIdCheck.id == trade.id)
{
break;
}
}
tradeList.Add(new Trade(t));
}
return true;
}
private HtmlDocument LoadHeroesPage()
{
HtmlDocument doc = new HtmlDocument();
doc.Load(GetStream("http://www.dota2.com/heroes/?l=english"));
return doc;
}
private HtmlDocument LoadHtmlSnippetFromFile()
{
HtmlDocument doc = new HtmlDocument();
doc.Load(GetStream("http://www.dota2.com/hero/Earthshaker/"));
return doc;
}
public HeroesRepository()
{
var skillImages = new List<string>();
var skillNames = new List<string>();
var skillDescriptions = new List<string>();
var primaryStatsImages = new List<string>();
var primaryStatsValues = new List<string>();
string biography;
var manaCosts = new List<string>();
var coolDowns = new List<string>();
var skillBehaviours = new List<string>();
var skillDamageType = new List<string>();
var skillVideos = new List<string>();
//TODO Armazenar quantidade de skills que o heroi tem
agilityPackHelper = new AgilityPackHelper();
// load snippet
HtmlDocument doc = new HtmlDocument();
doc = LoadHtmlSnippetFromFile();
skillImages = GetSkillPortraits(doc);
skillNames = GetSkillNames(doc);
skillDescriptions = GetSkillDescriptions(doc);
primaryStatsImages = GetPrimaryStatsImages(doc);
primaryStatsValues = GetPrimaryStatsValues(doc);
biography = GetBiography(doc);
//TODO Stats(Atributos)
manaCosts = GetManaCost(doc);
coolDowns = GetCoolDown(doc);
skillBehaviours = GetSkillBehaviour(doc);
skillDamageType = GetSkillDamageType(doc);
skillVideos = GetSkillVideo(doc);
}
private HtmlDocument LoadHeroHtmlPage(string heroName)
{
HtmlDocument doc = new HtmlDocument();
doc.Load(GetStream("http://www.dota2.com/hero/" + heroName + "/?l=english"));
return doc;
}
protected async void Page_Load()
{
HttpClient client = new HttpClient();
var doc = new HtmlAgilityPack.HtmlDocument();
var html = await client.GetStringAsync("http://ffxivhunt.com/" + FFXIV_Safari.Properties.Settings.Default.serverselect.ToLower());
doc.LoadHtml(html);
if (this.InvokeRequired)
{
BlinkDelegate del = new BlinkDelegate(Page_Load);
this.Invoke(del);
}
else
{
huntdata.Suspend();
elitemarksA.Clear();
elitemarksS.Clear();
finaloutputA.Clear();
finaloutputS.Clear();
huntdata.Clear();
elitemarksA.Add("Forneus", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_35']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Girtab", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_38']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Melt", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_5']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Ghede Ti Malice", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][4]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_32']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Laideronnette", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_36']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Thousand-cast Theda", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_39']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Wulgaru", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_6']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Mindflayer", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][4]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_33']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Sabotender Bailarina", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_50']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Alectyron", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][5]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_8']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Zanig'oh", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][4]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_44']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Dalvag's Final Flame", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_47']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Maahes", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_41']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Brontes", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_51']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Zona Seeker", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][5]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_9']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Nunyunuwi", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][4]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_45']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Minhocao", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_48']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Lampalagua", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_42']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Hellsclaw", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_20']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Unktehi", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_23']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Vogaal Ja", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_17']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Cornu", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][4]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_29']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Marberry", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][5]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_26']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Nahn", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][6]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_2']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Garlok", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_21']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Croakadile", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_24']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Croque-Mitaine", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_18']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Mahisha", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][4]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_30']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Nandi", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][5]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_27']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Bonnacon", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][1]/div[@class='panel panel-info']/table[@class='table table-condensed table-hover'][6]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_3']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Mirka", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_53']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Lyuba", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_68']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Marraco", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_11']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Kurrea", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover']/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_14']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Kaiser Behemoth", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][3]/td[@class='t140']/a[@id='mob_54']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Safat", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][2]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_12']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Agrippa the Mighty", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][3]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover']/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_15']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Pylraster", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_59']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Lord of the Wyverns", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_75']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Slipkinx Steeljoints", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_62']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Stolas", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_77']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Bune", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_65']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Agathos", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_81']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Senmurv", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][3]/td[@class='t140']/a[@id='mob_60']/span[@class='time']")[0].InnerText);
elitemarksS.Add("The Pale Rider", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][3]/tbody/tr[@class='hot-toggle'][3]/td[@class='t140']/a[@id='mob_63']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Gandarewa", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][1]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][3]/td[@class='t140']/a[@id='mob_66']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Campacti", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_71']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Stench Blossom", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_72']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Enkelados", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][1]/td[@class='t140']/a[@id='mob_56']/span[@class='time']")[0].InnerText);
elitemarksA.Add("Sisiutl", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][2]/td[@class='t140']/a[@id='mob_79']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Leucrotta", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][1]/tbody/tr[@class='hot-toggle'][3]/td[@class='t140']/a[@id='mob_73']/span[@class='time']")[0].InnerText);
elitemarksS.Add("Bird of Paradise", doc.DocumentNode.SelectNodes("/html/body/div[@id='timer']/div[@class='row breath']/div[@class='col-md-3 col-sm-6'][4]/div[@class='panel panel-info'][2]/table[@class='table table-condensed table-hover'][2]/tbody/tr[@class='hot-toggle'][3]/td[@class='t140']/a[@id='mob_57']/span[@class='time']")[0].InnerText);
printData();
}
}
/// <summary>
/// Fill an object and go through it's properties and fill them too.
/// </summary>
/// <typeparam name="T">Type of object to want to fill. It should have atleast one property that defined XPath.</typeparam>
/// <param name="htmlDocument">If htmlDocument includes data , leave this parameter null. Else pass your specific htmldocument.</param>
/// <returns>Returns an object of type T including Encapsulated data.</returns>
public T GetEncapsulatedData <T>(HtmlDocument htmlDocument = null)
{
return((T)GetEncapsulatedData(typeof(T), htmlDocument));
}
public override bool AfterAllGrab(IListSheet listSheet)
{
ExcelWriter resultEW = null;
int fileIndex = 1;
string detailPageUrlColumnName = SysConfig.DetailPageUrlFieldName;
Dictionary <string, string> shopDic = new Dictionary <string, string>();
for (int i = 0; i < listSheet.RowCount; i++)
{
if (resultEW == null || resultEW.RowCount > 500000)
{
if (resultEW != null)
{
resultEW.SaveToDisk();
}
resultEW = this.GetExcelWriter(fileIndex);
fileIndex++;
}
Dictionary <string, string> row = listSheet.GetRow(i);
bool giveUp = "Y".Equals(row[SysConfig.GiveUpGrabFieldName]);
if (!giveUp)
{
string url = row[detailPageUrlColumnName];
string cookie = row["cookie"];
string city = row["city"];
string g = row["g"];
string r = row["r"];
string gName = row["gName"];
string rName = row["rName"];
string pageIndex = row["pageIndex"];
HtmlAgilityPack.HtmlDocument pageHtmlDoc = this.RunPage.GetLocalHtmlDocument(listSheet, i);
HtmlNodeCollection allShopLinkNodes = pageHtmlDoc.DocumentNode.SelectNodes("//div[@id=\"shop-all-list\"]/ul/li/div[@class=\"txt\"]");
if (allShopLinkNodes != null)
{
for (int j = 0; j < allShopLinkNodes.Count; j++)
{
HtmlNode shopLinkNode = allShopLinkNodes[j];
HtmlNodeCollection ratingNodes = shopLinkNode.SelectNodes("./span[@class=\"comment-list\"]/span");
Nullable <decimal> serviceRating = null;
Nullable <decimal> environmentRating = null;
Nullable <decimal> tasteRating = null;
if (ratingNodes != null)
{
foreach (HtmlNode ratingNode in ratingNodes)
{
string ratingText = ratingNode.InnerText.Trim();
decimal ratingValue = decimal.Parse(ratingNode.SelectSingleNode("./b").InnerText.Trim());
if (ratingText.StartsWith("口味"))
{
tasteRating = ratingValue;
}
else if (ratingText.StartsWith("服务"))
{
serviceRating = ratingValue;
}
else if (ratingText.StartsWith("环境"))
{
environmentRating = ratingValue;
}
}
}
string address = "";
HtmlNode addressNode = shopLinkNode.SelectSingleNode("./div[@class=\"tag-addr\"]/span[@class=\"addr\"]");
if (addressNode != null)
{
address = CommonUtil.HtmlDecode(addressNode.InnerText.Trim());
}
HtmlNode nameNode = shopLinkNode.SelectSingleNode("./div[@class=\"tit\"]/a");
HtmlNode reviewNumNode = shopLinkNode.SelectSingleNode("./div[@class=\"comment\"]/a[@class=\"review-num\"]/b");
if (nameNode != null)
{
string shopName = nameNode.Attributes["title"].Value;
string hrefValue = nameNode.Attributes["href"].Value;
string[] hrefs = hrefValue.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
string shopCode = hrefs[hrefs.Length - 1];
Nullable <int> reviewNum = 0;
if (reviewNumNode != null)
{
reviewNum = int.Parse(reviewNumNode.InnerText);
}
if (!shopDic.ContainsKey(shopCode))
{
shopDic.Add(shopCode, "");
string detailPageName = shopCode;
string detailPageUrl = hrefValue;
Dictionary <string, object> f2vs = new Dictionary <string, object>();
f2vs.Add("detailPageUrl", detailPageUrl);
f2vs.Add("detailPageName", detailPageName);
f2vs.Add("cookie", "cy=22; cye=jinan;");
f2vs.Add("city", city);
f2vs.Add("g", g);
f2vs.Add("r", r);
f2vs.Add("gName", gName);
f2vs.Add("rName", rName);
f2vs.Add("pageIndex", pageIndex);
f2vs.Add("shopName", shopName);
f2vs.Add("shopCode", shopCode);
f2vs.Add("reviewNum", reviewNum);
f2vs.Add("serviceRating", serviceRating);
f2vs.Add("environmentRating", environmentRating);
f2vs.Add("tasteRating", tasteRating);
f2vs.Add("address", address);
resultEW.AddRow(f2vs);
}
}
}
}
}
}
resultEW.SaveToDisk();
return(true);
}
internal HtmlAttribute(HtmlDocument ownerdocument)
{
this._ownerdocument = ownerdocument;
}
private bool IsFaulted(HtmlDocument doc)
{
var ind = doc.DocumentNode.SelectSingleNode("//title").InnerHtml.IndexOf("Access Denied", StringComparison.Ordinal);
return(ind != -1);
}
public static DateTime TrainingDuration(HtmlAgilityPack.HtmlDocument html)
{
var training = TroopsParser.GetTroopsCurrentlyTraining(html);
return(training.Count == 0 ? DateTime.MinValue : training.Last().FinishTraining);
}
private String[] pagelist; //각페이지 데이터 정보
//생성자
public naver(String link)
{
this.naverlink = link;
web = new HtmlAgilityPack.HtmlWeb();
document = web.Load(naverlink);
}
private bool GetAllQYZCRYPageUrls(IListSheet listSheet)
{
ExcelWriter qyzzzgEW = this.GetQYZCRYExcelWriter();
string detailPageUrlColumnName = SysConfig.DetailPageUrlFieldName;
Dictionary <string, string> companyDic = new Dictionary <string, string>();
for (int i = 0; i < listSheet.RowCount; i++)
{
Dictionary <string, string> row = listSheet.GetRow(i);
string detailPageUrl = row[SysConfig.DetailPageUrlFieldName];
string detailPageName = row[SysConfig.DetailPageNameFieldName];
string companyId = row["companyId"];
bool giveUp = "Y".Equals(row[SysConfig.GiveUpGrabFieldName]);
if (!giveUp)
{
HtmlAgilityPack.HtmlDocument pageHtmlDoc = this.RunPage.GetLocalHtmlDocument(listSheet, i);
//都包含第一页
Dictionary <string, string> f2vs = new Dictionary <string, string>();
f2vs.Add("detailPageUrl", detailPageUrl);
f2vs.Add("detailPageName", detailPageName);
f2vs.Add("companyId", companyId);
f2vs.Add("pageIndex", "1");
f2vs.Add("formData", "");
qyzzzgEW.AddRow(f2vs);
HtmlNode pageNode = pageHtmlDoc.DocumentNode.SelectSingleNode("//a[@sf=\"pagebar\"]");
if (pageNode != null)
{
string pageData = pageNode.GetAttributeValue("sf:data", "");
if (pageData.Length == 0)
{
throw new Exception("获取分页信息错误. companyId = " + companyId);
}
else
{
JObject rootJo = JObject.Parse(pageData.Substring(1, pageData.Length - 2));
string ps = rootJo.GetValue("ps").ToString();
string tt = rootJo.GetValue("tt").ToString();
string pc = rootJo.GetValue("pc").ToString();
int pageCount = int.Parse(pc);
for (int pIndex = 2; pIndex <= pageCount; pIndex++)
{
Dictionary <string, string> otherPageF2vs = new Dictionary <string, string>();
otherPageF2vs.Add("detailPageUrl", detailPageUrl + "?_=" + pIndex.ToString());
otherPageF2vs.Add("detailPageName", detailPageName + "_" + pIndex.ToString());
otherPageF2vs.Add("companyId", companyId);
otherPageF2vs.Add("pageIndex", pIndex.ToString());
otherPageF2vs.Add("formData", "%24total=" + tt + "&%24reload=0&%24pg=" + pIndex.ToString() + "&%24pgsz=" + ps);
qyzzzgEW.AddRow(otherPageF2vs);
}
}
}
}
}
qyzzzgEW.SaveToDisk();
return(true);
}
public void InsertWeb(DataTable tblDatas)
{
for (int i = 0; i < tblDatas.Rows.Count; i++)
{
string Listsoudoc = "";
driver.Navigate().GoToUrl(tblDatas.Rows[i]["pUrl"].ToString());
IJavaScriptExecutor JS2 = (IJavaScriptExecutor)driver;
JS2.ExecuteScript("window.scrollTo(0, document.body.scrollHeight);");
Thread.Sleep(3000);
HtmlAgilityPack.HtmlDocument listdoc = new HtmlAgilityPack.HtmlDocument();
listdoc.LoadHtml(driver.PageSource);
HtmlNode listgetNode = listdoc.DocumentNode;
//标题
HtmlNode getNodees = listgetNode.SelectSingleNode("//div[@class='article-header']/h3");
// HtmlNode[] T_zhugetNodees = listgetNode.CssSelect("div.title_zuo").ToArray();
//作者
HtmlNode getNodees2 = listgetNode.SelectSingleNode("//div[@class='article-author']/a");
//日期
HtmlNode getNodeestime = listgetNode.SelectSingleNode("//span[@class='date']");
//分类
// HtmlNode getNodees3 = listgetNode.SelectSingleNode("//div[3]/div[2]/div[2]/div[1]/keyword");
//正文
HtmlNode getNodeescount = listgetNode.SelectSingleNode("//div[@class='article-body']");
string stcou = getNodeescount.InnerHtml;
HtmlAgilityPack.HtmlDocument listdoc2 = new HtmlAgilityPack.HtmlDocument();
listdoc2.LoadHtml(stcou);
HtmlNode cgetdco = listdoc2.DocumentNode;
string Pek = cgetdco.InnerHtml;
if (cgetdco.SelectNodes("//strong") != null)
{
HtmlNode[] getNodeescounse = cgetdco.SelectNodes("//strong").ToArray();
for (int p = 0; p < getNodeescounse.Length; p++)
{
Pek = Pek.Replace(getNodeescounse[p].OuterHtml, "");
}
}
if (cgetdco.SelectNodes("//img") != null)
{
HtmlNode[] getNodeescounlink = cgetdco.SelectNodes("//img").ToArray();
for (int k = 0; k < getNodeescounlink.Length; k++)
{
string pkk = getNodeescounlink[k].ParentNode.OuterHtml;
HtmlNode getimg = getNodeescounlink[k];
if (getimg.Attributes["src"] != null)
{
string imgu = getimg.Attributes["src"].Value.ToString();
imgu = imgu.Replace("//img.alicdn.com", "https://img.alicdn.com");
// this.textBox5.Text += getNodeescounlink[i].ParentNode.OuterHtml;
string imgName = imgu.ToString().Substring(imgu.ToString().LastIndexOf("/") + 1);
getimg.Attributes["src"].Value = "/kimages/" + imgName;
SaveImageFromWeb(imgu, "G:\\DTcms\\DTcms\\DTcms.Web\\kimages\\");
Pek = Pek.Replace(pkk, getimg.OuterHtml);
}
// this.textBox5.Text += getNodeescounlink[i].ParentNode.OuterHtml;
// getNodeescounlink[i].Attributes["src"].Value = "/images/" + imgName;
}
}
if (cgetdco.SelectNodes("//a") != null)
{
HtmlNode[] getNodeescounlinka = cgetdco.SelectNodes("//a").ToArray();
for (int a = 0; a < getNodeescounlinka.Length; a++)
{
Pek = Pek.Replace(getNodeescounlinka[a].ParentNode.OuterHtml, "");
}
}
//Pek = Pek.Replace("//img.alicdn.com", "https://img.alicdn.com");
// Pek = Pek.Replace("*****@*****.**", "*****@*****.**");
// this.textBox5.Text = Pek;
string T_Tit = getNodees.InnerText;
ShowInfo(T_Tit);
ShowInfo(getNodeestime.InnerText);
// ShowInfo(getNodees2.InnerText);
ShowInfo(Pek);
//, T_Tit, Pek, getNodeestime.InnerText, getNodees2.InnerText);;
JHinto(6, T_Tit, tblDatas.Rows[i]["Description"].ToString(), Pek, getNodeestime.InnerText, "", tblDatas.Rows[i]["pImg"].ToString());
}
}
public static HtmlNodeCollection SELECT_NODES(this HtmlNode node_input, string xpath)
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(node_input.InnerHtml);
return(doc.DocumentNode.SelectNodes(xpath));
}
/// <summary>
/// Gets a memory stream representing an image from an explicit favicon location.
/// </summary>
/// <param name="fullURI">The URI.</param>
/// <param name="ms">The memory stream (output).</param>
/// <param name="message">Any error message is sent back through this string.</param>
/// <returns></returns>
private Uri getFromFaviconExplicitLocation(Uri fullURI, ref MemoryStream ms, ref string message)
{
HtmlWeb hw = new HtmlWeb();
hw.UserAgent = "Mozilla/5.0 (Windows 6.1; rv:27.0) Gecko/20100101 Firefox/27.0";
HtmlAgilityPack.HtmlDocument hdoc = null;
Uri responseURI = null;
try
{
int counter = 0; // Protection from cyclic redirect
Uri nextUri = fullURI;
do
{
// A cookie container is needed for some sites to work
hw.PreRequest += PreRequest_EventHandler;
// HtmlWeb.Load will follow 302 and 302 redirects to alternate URIs
hdoc = hw.Load(nextUri.AbsoluteUri);
responseURI = hw.ResponseUri;
// Old school meta refreshes need to parsed
nextUri = getMetaRefreshLink(responseURI, hdoc);
counter++;
} while (nextUri != null && counter < 16); // Sixteen redirects would be more than enough.
}
catch (Exception)
{
return(responseURI);
}
if (hdoc == null)
{
return(responseURI);
}
string faviconLocation = "";
try
{
HtmlNodeCollection links = hdoc.DocumentNode.SelectNodes("/html/head/link");
for (int i = 0; i < links.Count; i++)
{
HtmlNode node = links[i];
try
{
HtmlAttribute r = node.Attributes["rel"];
string val = r.Value.ToLower().Replace("shortcut", "").Trim();
if (val == "icon")
{
try
{
faviconLocation = node.Attributes["href"].Value;
// Don't break the loop, because there could be many <link rel="icon"> nodes
// We should read the last one, like web browsers do
}
catch (Exception)
{
}
}
}
catch (Exception)
{
}
}
}
catch (Exception)
{
}
if (String.IsNullOrEmpty(faviconLocation))
{
return(responseURI);
}
return((getFavicon(new Uri(responseURI, faviconLocation), ref ms, ref message)) ? new Uri("http://success") : responseURI);
}
/// <summary>
/// 添加新的类别
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btnAdd_Click(object sender, EventArgs e)
{
int id;
if (!int.TryParse(txtTypeID.Text, out id))
{
new UgMessageBox("نۇر تورىدىكى تۈر نۇمۇرى چۇقۇم سان بۇلىشى كىرەك، سان بولمىسا داۋاملىق مەشغۇلات قىلالمايسىز!!!", Color.Red).ShowDialog();
return;
}
if (string.IsNullOrEmpty(txtTypeTitle.Text.Trim()))
{
new UgMessageBox("نۇر تورىدىكى خەۋەر نامى بوش قالمىسۇن، بۇ ئىككى سانداننى سېلىشتۇرۇش ئۈچۈن ئاساس بۇلىدۇ.", Color.Red).ShowDialog();
return;
}
//检测网络状态
if (!isConnected())
{
new UgMessageBox("تور ئۇلىنىشتا خاتالىق بار،تورغا ئۇلانمىسا سىز قوشقان بۇ تۈرنىڭ نۇر تورىدا مەۋجۇتلۇقىنى بىلگىلى بولمايدۇ، بۇنداق شارائىتتا نىمە قىلىشىڭىزنى سىز ئوبدان بىلىسىز", Color.Red).ShowDialog();
return;
}
//判断该类别是否存在
bool error = false;
//获取 html
//http://www.nur.cn/index.php?a=lists&catid=7
string html = GetHtmlData("http://www.nur.cn/index.php?a=lists&catid=" + id);
if (html == "")
{
error = true;
}
else
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
//获取body
HtmlNode htmlNode = doc.DocumentNode;
HtmlNode bodyNode = htmlNode.Element("html").Element("body");
if (bodyNode == null)
{
error = true;
}
else
{
//获取 div
HtmlNode listNode = bodyNode.Elements("div").FirstOrDefault(n => n.GetAttributeValue("class", "") == "list_box");
if (listNode == null)
{
error = true;
}
else
{
error = false;
}
}
}
if (error)
{
new UgMessageBox("كەچۈرۈڭ!!! نۇر تورىدا بۇنداق تۈر مەۋجۇت ئەمەس، مەۋجۇت بولمىغان نەرسىنى سىز قانداق ساندانغا قۇشاسىز،ئەلۋەتتە قۇشالمايسىز، شۇڭا مەنمۇ بۇنى تۈر تىزىملىكىگە قۇشالمايمەن، خاپا بولماي قايتا سىناپ باقاملا، سىنىغانغا باج ئالمايمەن، خاتىرجەم سىناڭ", Color.Red).ShowDialog();
return;
}
//判断是否在集合中已存在这个类别
if (types.Where(p => p.TypeID == id).Count() > 0)
{
new UgMessageBox("بۇ تۈر ئاللىقاچان تاللانغان، كۆزىڭىزنى يۇغان ئېچىپ ئاۋۇ تىرناقنىڭ ئىچىگە ئوبدان قاراپ بېقىڭ، سىز تولداغان سان بامىكەن يا يوقمىكەن", Color.Red).ShowDialog();
return;
}
//该类别存在,添加到集合中
NurNewsTypeModel nur = new NurNewsTypeModel();
nur.TypeID = id;
nur.Title = txtTypeTitle.Text.Trim();
types.Add(nur);
DataBingToSelectBox();
new UgMessageBox("بۇ تۈر نۇر تورىدا مەۋجۇت بولغانلىقى ئۈچۈن مۇۋاپىقىيەتلىك قوشۇلدى.", Color.Green).ShowDialog();
}
public static IEnumerable <HtmlNode> GetElementsByName(this HtmlDocument document, string name)
{
return(document.DocumentNode.Descendants().Where(n => n.GetAttributeValue("name", "") == name));
}
public static IEnumerable <HtmlNode> GetElementsByClassName(this HtmlDocument document, string className)
{
return(document.DocumentNode.Descendants().Where(n => n.GetAttributeValue("class", "").Split(' ').Contains(className)));
}
/// <summary>
/// Gets a memory stream representing an image from an explicit favicon location.
/// </summary>
/// <param name="url">The URL.</param>
/// <param name="protocol">i.e http or https</param>
/// <param name="ms">The memory stream (output).</param>
/// <param name="message">Any error message is sent back through this string.</param>
/// <returns></returns>
private bool getFromFaviconExplicitLocation(string url, string protocol, string fullURL, ref MemoryStream ms, ref string message)
{
if (protocol != "https")
{
protocol = "http";
}
HtmlWeb hw = new HtmlWeb();
HtmlAgilityPack.HtmlDocument hdoc = null;
Uri responseURI = null;
try
{
Uri nextUri = new Uri(fullURL);
do
{
// HtmlWeb.Load will follow 302 and 302 redirects to alternate URIs
hdoc = hw.Load(nextUri.AbsoluteUri);
responseURI = hw.ResponseUri;
// Old school meta refreshes need to parsed
nextUri = getMetaRefreshLink(responseURI, hdoc);
} while (nextUri != null);
}
catch (Exception)
{
return(false);
}
if (hdoc == null)
{
return(false);
}
string faviconLocation = "";
try
{
HtmlNodeCollection links = hdoc.DocumentNode.SelectNodes("/html/head/link");
for (int i = 0; i < links.Count; i++)
{
HtmlNode node = links[i];
try
{
HtmlAttribute r = node.Attributes["rel"];
if (r.Value.ToLower().CompareTo("shortcut icon") == 0 || r.Value.ToLower().CompareTo("icon") == 0)
{
try
{
faviconLocation = node.Attributes["href"].Value;
break;
}
catch (Exception)
{
}
}
}
catch (Exception)
{
}
}
}
catch (Exception)
{
}
if (string.IsNullOrEmpty(faviconLocation))
{
return(false);
}
faviconLocation = reconcileURI(responseURI, faviconLocation).AbsoluteUri;
return(getFavicon(faviconLocation, ref ms, ref message));
}
private void btnReadUrl_Click(object sender, EventArgs e)
{
string remoteUri = txtUrl.Text.Trim();
if (remoteUri == "")
{
MessageBox.Show("URL is empty.", "Warning", MessageBoxButtons.OK, MessageBoxIcon.Warning);
return;
}
try
{
WebClient webClient = new WebClient();
byte[] dataBuffer = webClient.DownloadData(remoteUri);
MemoryStream memoryStream = new MemoryStream(dataBuffer);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(memoryStream, true);
string text = "";
string title = "";
HtmlNodeCollection headings = doc.DocumentNode.SelectNodes("//h1[@class='firstHeading']");
if (headings != null)
{
title = headings[0].InnerText;
text = title + "\r\n\r\n";
txtFileName.Text = string.Format(@"{0}.txt", replaceNonAscii(title));
}
HtmlNodeCollection divs = doc.DocumentNode.SelectNodes("//div[@class='mw-content-ltr']") ??
doc.DocumentNode.SelectNodes("//div[@id='bodyContent']");
if (divs != null)
{
int paragraphCount = 0;
foreach (HtmlNode node in divs[0].ChildNodes)
{
if (isTocOrH2(node) || node.ChildNodes.Any(c => isTocOrH2(c)))
{
break;
}
if (node.InnerText.Trim() != "")
{
if (node.Name == "p")
{
text += extractText(node) + "\r\n\r\n";
paragraphCount++;
}
else if (paragraphCount > 0 && node.Name == "ul" || node.Name == "ol")
{
foreach (HtmlNode listItem in node.SelectNodes("li"))
{
text += "- " + extractText(listItem) + "\r\n";
}
text += "\r\n";
paragraphCount++;
}
}
}
}
txtText.Text = text;
MessageBox.Show("URL was read successfully.", "OK", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
//naver링크안에 있는 각각의 상품 url
public void geturl(String link, int code)
{
HtmlAgilityPack.HtmlWeb web2 = new HtmlAgilityPack.HtmlWeb();
HtmlAgilityPack.HtmlDocument document2 = web2.Load(link);
String tmp = document2.DocumentNode.InnerText;
switch (code)
{
case 1:
if (name == "G마켓")
{
int index = 0, index2 = 0;
index = tmp.IndexOf("targetUrl") + 13;
index2 = tmp.IndexOf("var refererLink") - 5;
tmp = tmp.Substring(index, index2 - index).Trim();
tmp = ("http://cr2.shopping.naver.com" + tmp);
geturl(tmp, 2);
}
else if (name == "11번가")
{
int index = 0, index2 = 0;
index = tmp.IndexOf("targetUrl") + 13;
index2 = tmp.IndexOf("var refererLink") - 5;
tmp = tmp.Substring(index, index2 - index).Trim();
tmp = ("http://cr2.shopping.naver.com" + tmp);
geturl(tmp, 3);
}
else if (name == "CJmall")
{
int index = 0, index2 = 0;
index = tmp.IndexOf("targetUrl") + 13;
index2 = tmp.IndexOf("var refererLink") - 5;
tmp = tmp.Substring(index, index2 - index).Trim();
tmp = ("http://cr2.shopping.naver.com" + tmp);
geturl(tmp, 4);
}
else if (name == "GSSHOP")
{
int index = 0, index2 = 0;
index = tmp.IndexOf("targetUrl") + 13;
index2 = tmp.IndexOf("var refererLink") - 5;
tmp = tmp.Substring(index, index2 - index).Trim();
tmp = ("http://cr2.shopping.naver.com" + tmp);
geturl(tmp, 5);
}
else if (name == "옥션")
{
int index = 0, index2 = 0;
index = tmp.IndexOf("targetUrl") + 13;
index2 = tmp.IndexOf("var refererLink") - 5;
tmp = tmp.Substring(index, index2 - index).Trim();
tmp = ("http://cr2.shopping.naver.com" + tmp);
geturl(tmp, 6);
}
else
{
realurl = link;
}
break;
//G마켓url일때
case 2:
int index3 = 0, index4 = 0;
index3 = tmp.IndexOf("replace") + 9;
index4 = tmp.IndexOf("}") - 6;
tmp = tmp.Substring(index3, index4 - index3);
realurl = tmp.Split('"').ElementAt(tmp.Split('"').Length - 1);
break;
//11번가url일때
case 3:
String tmp2 = document2.DocumentNode.Descendants().Where(x =>
x.Name == "script").ElementAt(2).InnerText;
realurl = tmp2.Split('"').ElementAt(1);
break;
//cjmall
case 4:
String kk = document2.DocumentNode.Descendants().Where(x =>
x.Name == "script").Last().InnerText;
realurl = kk.Split('"').ElementAt(1);
break;
//gsshop
case 5:
String kk2 = document2.DocumentNode.Descendants().Where(x =>
x.Name == "script").Last().InnerText;
String tmp3 = "http://with.gsshop.com";
realurl = tmp3 + kk2.Split('"').ElementAt(1);
break;
case 6:
String kk3 = document2.DocumentNode.Descendants().Where(x =>
x.Name == "script").ElementAt(3).InnerText;
itemNum = kk3.Split('"').ElementAt(kk3.Split('"').Length - 2);
realurl = "http://itempage3.auction.co.kr/DetailView.aspx?ItemNo=" + itemNum + "&frm3=V2";
break;
}
}
private async Task <IEnumerable <string> > GetPageLinks(string url, bool needMatch = true)
{
try
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
var htmlDocument = new HtmlAgilityPack.HtmlDocument();
CookieCollection cookieCollection = null;
switch (url)
{
case string c when c.Contains("rst"): //do it wothout hardcode, maybe test call, check encoding, set here and call again to scrap?
WebClient wc = new WebClient();
wc.Encoding = System.Text.Encoding.GetEncoding(1251);
wc.Headers.Add("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
string str = String.Empty;
try
{
str = await wc.DownloadStringTaskAsync(c);
}
catch (System.Exception)
{
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);
req.Method = "GET";
req.AllowAutoRedirect = true;
req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36";
HttpWebResponse webresponse = (HttpWebResponse)req.GetResponse();
Console.Write(webresponse.StatusCode); //check the statusCode
Stream receiveStream = webresponse.GetResponseStream();
using (StreamReader reader = new StreamReader(receiveStream))
{
str = reader.ReadToEnd();
}
}
htmlDocument.LoadHtml(str);
break;
// CookieCollection cookieCollection = null;
// HtmlWeb webC = new HtmlWeb
// {
// UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36",
// UseCookies = true,
// OverrideEncoding = System.Text.Encoding.GetEncoding(1251),
// PreRequest = request =>
// {
// if (cookieCollection != null && cookieCollection.Count > 0)
// request.CookieContainer.Add(cookieCollection);
// return true;
// },
// PostResponse = (request, response) =>
// {
// cookieCollection = response.Cookies;
// }
// };
// htmlDocument = await webC.LoadFromWebAsync(c);
// break;
case string a when !a.Contains("rst"):
// case string a when a.Contains("mobile"):
HtmlWeb web = new HtmlWeb
{
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36",
UseCookies = true,
PreRequest = request =>
{
if (cookieCollection != null && cookieCollection.Count > 0)
{
request.CookieContainer.Add(cookieCollection);
}
return(true);
},
PostResponse = (request, response) => { cookieCollection = response.Cookies; }
};
htmlDocument = await web.LoadFromWebAsync(a);
break;
default:
throw new Exception("Create a more multipurpose method, lazy piece of ..");
}
var linkList = htmlDocument.DocumentNode
.Descendants("a")
.Select(a => a.GetAttributeValue("href", null))
.Where(u => !string.IsNullOrEmpty(u))
.Distinct();
if (_regex != null)
{
linkList = linkList.Where(x => _regex.IsMatch(x));
}
if (url.Contains("rst"))
{
foreach (var item in linkList)
{
linkList = linkList.Select(x => x.Equals(item) ? $"https://rst.ua{x}" : x);
}
}
if (!url.Contains("rst") && linkList.Count() > 4)
{
var lastPage = Int32.Parse(Regex.Match(linkList.Last(), @"\d+$").Value);
var secondLastPage = linkList.Count() > 1 ? Int32.Parse(Regex.Match(linkList.SkipLast(1).Last(), @"\d+$").Value) : 0;
if (lastPage - secondLastPage >= 1)
{
var newUrl = linkList.SkipLast(1).Last();
for (int i = secondLastPage + 1; i < lastPage; i++)
{
var urlToAdd = newUrl;
urlToAdd = Regex.Replace(urlToAdd, @"\d+$", i.ToString());
linkList = linkList.Append(urlToAdd);
}
}
}
linkList = linkList.Prepend(url);
return(linkList);
}
catch (Exception exception)
{
return(Enumerable.Empty <string>());
}
}
static void Main(string[] args)
{
System.Net.WebClient wc = new System.Net.WebClient();
//string urlnew = "http://globaldocuments.morningstar.com/DocumentLibrary/Document/81d1472d3c8f11f430218f84ac24deab.msdoc/original/d284756d10q.htm";
//string urlold = "http://globaldocuments.morningstar.com/DocumentLibrary/Document/b4acd5a4e7f551a2532e6ada39c37ae8.msdoc/original/d229830d10q.htm";
string urlold = "https://www.sec.gov/Archives/edgar/data/789019/000156459017000654/msft-10q_20161231.htm";
string urlnew = "https://www.sec.gov/Archives/edgar/data/789019/000119312516742796/d245252d10q.htm";
string newFiling = wc.DownloadString(urlnew);
string oldFiling = wc.DownloadString(urlold);
newFiling = cleanup(newFiling);
oldFiling = cleanup(oldFiling);
HtmlAgilityPack.HtmlDocument htmlDocNew = new HtmlAgilityPack.HtmlDocument();
htmlDocNew.LoadHtml(newFiling);
var title = htmlDocNew.DocumentNode.SelectSingleNode("//head/title");
Console.WriteLine(title.InnerText);
HtmlAgilityPack.HtmlDocument htmlDocOld = new HtmlAgilityPack.HtmlDocument();
htmlDocOld.LoadHtml(oldFiling);
StreamWriter swTablesNew = new StreamWriter("tablesNew.html");
Hashtable htDataTablesNew = new Hashtable();
int tableSequenceNew = 0;
foreach (HtmlNode table in htmlDocNew.DocumentNode.SelectNodes("//table"))
{
++tableSequenceNew;
swTablesNew.WriteLine(table.OuterHtml + "<br/><br/><br/>");
int rows = 0;
bool dataTable = false;
if (table.SelectNodes("tr") != null)
{
foreach (HtmlNode row in table.SelectNodes("tr"))
{
if (++rows > 1)
{
dataTable = true;
break;
}
}
}
if (dataTable)
{
string table_Id = "Table_Sequence_Id_" + tableSequenceNew.ToString();
htDataTablesNew.Add(table_Id, table);
HtmlNode spaceHolderNode = HtmlNode.CreateNode("<Place_Holder tableId=\"" + table_Id + "\"></Place_Holder>");
table.ParentNode.ReplaceChild(spaceHolderNode, table);
}
}
swTablesNew.Flush();
StreamWriter swTablesOld = new StreamWriter("tablesOld.html");
Hashtable htDataTablesOld = new Hashtable();
int tableSequenceOld = 0;
foreach (HtmlNode table in htmlDocOld.DocumentNode.SelectNodes("//table"))
{
++tableSequenceOld;
swTablesOld.WriteLine(table.OuterHtml + "<br/><br/><br/>");
int rows = 0;
bool dataTable = false;
if (table.SelectNodes("tr") != null)
{
foreach (HtmlNode row in table.SelectNodes("tr"))
{
if (++rows > 1)
{
dataTable = true;
break;
}
}
}
if (dataTable)
{
string table_Id = "Table_Sequence_Id_" + tableSequenceOld.ToString();
htDataTablesOld.Add(table_Id, table);
HtmlNode spaceHolderNode = HtmlNode.CreateNode("<Place_Holder tableId=\"" + table_Id + "\"></Place_Holder>");
table.ParentNode.ReplaceChild(spaceHolderNode, table);
}
}
swTablesOld.Flush();
StreamWriter swold = new StreamWriter("c:/temp/old.html");
swold.Write(htmlDocOld.DocumentNode.OuterHtml);
swold.Flush();
StreamWriter swnew = new StreamWriter("c:/temp/new.html");
swnew.Write(htmlDocNew.DocumentNode.OuterHtml);
swnew.Flush();
global::HtmlDiff.HtmlDiff diff = new global::HtmlDiff.HtmlDiff(htmlDocOld.DocumentNode.OuterHtml, htmlDocNew.DocumentNode.OuterHtml);
diff.AddBlockExpression(new Regex(@"(Jan(uary)?|Feb(ruary)?|Ma(r(ch)?|y)|Apr(il)?|Ju((ly?)|(ne?))|Aug(ust)?|Oct(ober)?|(Sep(?=\b|t)t?|Nov|Dec)(ember)?)[\s]*[\d]{1,2}[\s]*[\d]{4}", RegexOptions.IgnoreCase));
diff.AddBlockExpression(new Regex(@"(Jan(uary)?|Feb(ruary)?|Ma(r(ch)?|y)|Apr(il)?|Ju((ly?)|(ne?))|Aug(ust)?|Oct(ober)?|(Sep(?=\b|t)t?|Nov|Dec)(ember)?)[\s]*[\d]{1,2}[\,\.]{1}[\s]*[\d]{4}", RegexOptions.IgnoreCase));
diff.AddBlockExpression(new Regex(@"(\$\d+.\d+)"));
diff.AddBlockExpression(new Regex(@"(\d+,\d+)"));
//diff.AddBlockExpression(new Regex(@"(\d+.\d+%)"));
diff.IgnoreWhitespaceDifferences = true;
diff.OrphanMatchThreshold = 0.1;
var delta = diff.Build();
HtmlDocument htmlDocDelta = new HtmlDocument();
htmlDocDelta.LoadHtml(delta);
foreach (HtmlNode table in htmlDocDelta.DocumentNode.SelectNodes("//place_holder"))
{
table.ParentNode.ReplaceChild((HtmlNode)htDataTablesNew[table.Attributes[0].Value], table);;
}
//HtmlTextNode Hnode = htmlDocDelta.DocumentNode.SelectSingleNode("//head/title") as HtmlTextNode;
//Hnode.Text = title.InnerText;
string head = "<head><link href=\"App_Themes/default/styles.css\" type=\"text/css\" rel=\"stylesheet\" /></head>";
StreamWriter sw = new StreamWriter("c:/temp/mytest.html");
sw.Write(head + htmlDocDelta.DocumentNode.OuterHtml);
sw.Flush();
//Console.Read();
}
private void GetList(IListSheet listSheet, string pageSourceDir, ExcelWriter categoryCW)
{
for (int i = 0; i < listSheet.RowCount; i++)
{
string pageUrl = listSheet.PageUrlList[i];
Dictionary <string, string> row = listSheet.GetRow(i);
string cookie = row["cookie"];
string district = row["detailPageName"];
string localFilePath = this.RunPage.GetFilePath(pageUrl, pageSourceDir);
TextReader tr = null;
try
{
tr = new StreamReader(localFilePath);
string webPageHtml = tr.ReadToEnd();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(webPageHtml);
HtmlNodeCollection allCategory1Nodes = htmlDoc.DocumentNode.SelectNodes("//div[@class=\"all-sort-list\"]/div[@class=\"item bo\"]");
if (allCategory1Nodes != null)
{
for (int j = 0; j < allCategory1Nodes.Count; j++)
{
HtmlNode category1Node = allCategory1Nodes[j];
HtmlNode c1Node = category1Node.SelectSingleNode("./h3/a");
string c1Name = CommonUtil.HtmlDecode(c1Node.InnerText.Trim());
string c1Url = c1Node.Attributes["href"].Value;
string[] c1UrlStrs = c1Url.Split(new string[] { "/", "." }, StringSplitOptions.RemoveEmptyEntries);
string c1Code = c1UrlStrs[c1UrlStrs.Length - 2];
//增加一级分类
{
Dictionary <string, string> f2vs = new Dictionary <string, string>();
string listPageUrl = c1Url;
f2vs.Add("detailPageUrl", listPageUrl);
f2vs.Add("detailPageName", c1Name);
f2vs.Add("cookie", cookie);
f2vs.Add("category1Code", c1Code);
f2vs.Add("category2Code", "");
f2vs.Add("category3Code", "");
f2vs.Add("category1Name", c1Name);
f2vs.Add("category2Name", "");
f2vs.Add("category3Name", "");
f2vs.Add("district", district);
categoryCW.AddRow(f2vs);
}
HtmlNodeCollection allCategory2ParentNodes = category1Node.SelectNodes("./div[@class=\"item-list clearfix\"]/div[@class=\"subitem\"]");
if (allCategory2ParentNodes != null)
{
//第一个大类不要
for (int k = 0; k < allCategory2ParentNodes.Count; k++)
{
HtmlNode category2ParentNode = allCategory2ParentNodes[k];
HtmlNodeCollection allCategory2Nodes = category2ParentNode.SelectNodes("./h2");
foreach (HtmlNode category2Node in allCategory2Nodes)
{
string childNodeText = category2Node.InnerText;
if (childNodeText != "所有商品")
{
string c2Name = CommonUtil.HtmlDecode(category2Node.InnerText.Trim());
HtmlNode c2Node = category2Node.SelectSingleNode("./a");
string c2Code = "";
string c2Url = "";
if (c2Node != null)
{
c2Url = c2Node.Attributes["href"].Value;
string[] c2UrlStrs = c2Url.Split(new string[] { "/", "." }, StringSplitOptions.RemoveEmptyEntries);
c2Code = c2UrlStrs[c2UrlStrs.Length - 2];
}
//增加二级分类
if (c2Url.Length != 0)
{
Dictionary <string, string> f2vs = new Dictionary <string, string>();
string listPageUrl = c2Url;
f2vs.Add("detailPageUrl", listPageUrl);
f2vs.Add("detailPageName", c1Name + "_" + c2Name);
f2vs.Add("cookie", cookie);
f2vs.Add("category1Code", c1Code);
f2vs.Add("category2Code", c2Code);
f2vs.Add("category3Code", "");
f2vs.Add("category1Name", c1Name);
f2vs.Add("category2Name", c2Name);
f2vs.Add("category3Name", "");
f2vs.Add("district", district);
categoryCW.AddRow(f2vs);
}
List <HtmlNode> allCategory3Nodes = new List <HtmlNode>();
HtmlNode nextNode = category2Node.NextSibling;
while (nextNode != null && nextNode.Name.ToUpper() != "H2")
{
if (nextNode.Name.ToUpper() == "P")
{
allCategory3Nodes.Add(nextNode);
}
nextNode = nextNode.NextSibling;
}
if (allCategory3Nodes.Count != 0)
{
//最末级为三级分类
foreach (HtmlNode category3Node in allCategory3Nodes)
{
HtmlNode c3Node = category3Node.SelectSingleNode("./a");
string c3Name = CommonUtil.HtmlDecode(c3Node.InnerText.Trim());
string c3Url = c3Node.Attributes["href"].Value;
string[] c3UrlStrs = c3Url.Split(new string[] { "/", "." }, StringSplitOptions.RemoveEmptyEntries);
string c3Code = c3UrlStrs[c3UrlStrs.Length - 2];
Dictionary <string, string> f2vs = new Dictionary <string, string>();
string listPageUrl = c3Url;
f2vs.Add("detailPageUrl", listPageUrl);
f2vs.Add("detailPageName", c1Name + "_" + c2Name + "_" + c3Name);
f2vs.Add("cookie", cookie);
f2vs.Add("category1Code", c1Code);
f2vs.Add("category2Code", c2Code);
f2vs.Add("category3Code", c3Code);
f2vs.Add("category1Name", c1Name);
f2vs.Add("category2Name", c2Name);
f2vs.Add("category3Name", c3Name);
f2vs.Add("district", district);
categoryCW.AddRow(f2vs);
}
}
}
}
}
}
}
}
}
catch (Exception ex)
{
this.RunPage.InvokeAppendLogText("读取出错. url = " + pageUrl + ". " + ex.Message, LogLevelType.Error, true);
throw ex;
}
}
}
void Search(string keywords, int KeyWordsId)
{
string key = string.Empty;
foreach (string k in keywords.Split(' '))
{
if (string.IsNullOrEmpty(key))
{
key += k;
}
else
{
key += "+" + k;
}
}
string url = "https://www.aliexpress.com/wholesale?catId=0&g=n&SearchText=" + key;
//string html = HttpHelper.get(url);
string html = GetHtml(url);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
HtmlNodeCollection items = doc.DocumentNode.SelectNodes("//*[@id=\"hs-list-items\"]");
if (items == null)
{
for (int i = 0; i < 10; i++)
{
System.Threading.Thread.Sleep(1000);
html = GetHtml(url);
doc.LoadHtml(html);
items = doc.DocumentNode.SelectNodes("//*[@id=\"hs-list-items\"]");
if (items != null)
{
break;
}
}
}
else
{
string topProduct = items[0].InnerHtml;
items = doc.DocumentNode.SelectNodes("//*[@id=\"hs-below-list-items\"]");
html = items[0].InnerHtml;
html = html.Replace("<script type=\"text/x-handlebars-template\" id=\"lazy-render\" class=\"lazy-render\">", "");
html = html.Replace("</script>", "");
html = html.Replace("<ul>", "");
html = "<ul>" + topProduct + html;
doc.LoadHtml(html);
items = doc.DocumentNode.SelectNodes("ul/li");
List <ProductListing> list = GetProduct(items, KeyWordsId);
//foreach(ProductInfo info in list)
//{
// GetHtml("https://"+info.ProudctLink);
//}
}
}
/// <summary>
/// Fill an object and go through it's properties and fill them too.
/// </summary>
/// <param name="targetType">Type of object to want to fill. It should have atleast one property that defined XPath.</param>
/// <param name="htmlDocument">If htmlDocument includes data , leave this parameter null. Else pass your specific htmldocument.</param>
/// <returns>Returns an object of type targetType including Encapsulated data.</returns>
public object GetEncapsulatedData(Type targetType, HtmlDocument htmlDocument = null)
{
#region SettingPrerequisite
if (targetType == null)
{
throw new ArgumentNullException("Parameter targetType is null");
}
HtmlDocument source = null;
if (htmlDocument == null)
{
source = OwnerDocument;
}
else
{
source = htmlDocument;
}
object targetObject;
if (targetType.IsInstantiable() == false) // if it can not create instanse of T because of lack of constructor in type T.
{
throw new MissingMethodException("Parameterless Constructor excpected for " + targetType.FullName);
}
else
{
targetObject = Activator.CreateInstance(targetType);
}
#endregion SettingPrerequisite
#region targetObject_Defined_XPath
if (targetType.IsDefinedAttribute(typeof(HasXPathAttribute)) == true) // Object has xpath attribute (Defined HasXPath)
{
// Store list of properties that defined xpath attribute
IEnumerable <PropertyInfo> validProperties = targetType.GetPropertiesDefinedXPath();
if (validProperties.CountOfIEnumerable() == 0) // if no XPath property exist in type T while T defined HasXpath attribute.
{
throw new MissingXPathException("Type " + targetType.FullName +
" defined HasXPath Attribute but it does not have any property with XPath Attribte.");
}
else
{
// Fill targetObject variable Properties ( T targetObject )
foreach (PropertyInfo propertyInfo in validProperties)
{
// Get xpath attribute from valid properties
// for .Net old versions:
XPathAttribute xPathAttribute = (propertyInfo.GetCustomAttributes(typeof(XPathAttribute), false) as IList)[0] as XPathAttribute;
#region Property_IsNOT_IEnumerable
if (propertyInfo.IsIEnumerable() == false) // Property is None-IEnumerable
{
HtmlNode htmlNode = null;
// try to fill htmlNode based on XPath given
try
{
htmlNode = source.DocumentNode.SelectSingleNode(xPathAttribute.XPath);
}
catch // if it can not select node based on given xpath
{
throw new NodeNotFoundException("Cannot find node with giving XPath to bind to " + propertyInfo.PropertyType.FullName + " " + propertyInfo.Name);
}
if (htmlNode == null)
{
throw new NodeNotFoundException("Cannot find node with givig XPath to bind to " +
propertyInfo.PropertyType.FullName + " " + propertyInfo.Name);
}
#region Property_Is_HasXPath_UserDefinedClass
// Property is None-IEnumerable HasXPath-user-defined class
if (propertyInfo.PropertyType.IsDefinedAttribute(typeof(HasXPathAttribute)) == true)
{
HtmlDocument innerHtmlDocument = new HtmlDocument();
innerHtmlDocument.LoadHtml(htmlNode.InnerHtml);
object o = GetEncapsulatedData(propertyInfo.PropertyType, innerHtmlDocument);
propertyInfo.SetValue(targetObject, o, null);
}
#endregion Property_Is_HasXPath_UserDefinedClass
#region Property_Is_SimpleType
// Property is None-IEnumerable value-type or .Net class or user-defined class.
// AND does not deifned xpath and shouldn't have property that defined xpath.
else
{
string result = string.Empty;
if (xPathAttribute.AttributeName == null) // It target None-IEnumerable value of HTMLTag
{
result = Tools.GetNodeValueBasedOnXPathReturnType <string>(htmlNode, xPathAttribute);
}
else // It target None-IEnumerable attribute of HTMLTag
{
result = htmlNode.GetAttributeValue(xPathAttribute.AttributeName, null);
}
if (result == null)
{
throw new NodeAttributeNotFoundException("Can not find " +
xPathAttribute.AttributeName + " Attribute in " + htmlNode.Name +
" related to " + propertyInfo.PropertyType.FullName + " " + propertyInfo.Name);
}
object resultCastedToTargetPropertyType;
try
{
resultCastedToTargetPropertyType = Convert.ChangeType(result, propertyInfo.PropertyType);
}
catch (FormatException)
{
throw new FormatException("Can not convert Invalid string to " + propertyInfo.PropertyType.FullName + " " + propertyInfo.Name);
}
catch (Exception ex)
{
throw new Exception("Unhandled Exception : " + ex.Message);
}
propertyInfo.SetValue(targetObject, resultCastedToTargetPropertyType, null);
}
#endregion Property_Is_SimpleType
}
#endregion Property_IsNOT_IEnumerable
#region Property_Is_IEnumerable
else // Property is IEnumerable<T>
{
IList <Type> T_Types = propertyInfo.GetGenericTypes() as IList <Type>; // Get T type
if (T_Types == null || T_Types.Count == 0)
{
throw new ArgumentException(propertyInfo.Name + " should have one generic argument.");
}
else if (T_Types.Count > 1)
{
throw new ArgumentException(propertyInfo.Name + " should have one generic argument.");
}
else if (T_Types.Count == 1) // It is NOT something like Dictionary<Tkey , Tvalue>
{
HtmlNodeCollection nodeCollection;
// try to fill nodeCollection based on given xpath.
try
{
nodeCollection = source.DocumentNode.SelectNodes(xPathAttribute.XPath);
}
catch
{
throw new NodeNotFoundException("Cannot find node with givig XPath to bind to " + propertyInfo.PropertyType.FullName + " " + propertyInfo.Name);
}
if (nodeCollection == null || nodeCollection.Count == 0)
{
throw new NodeNotFoundException("Cannot find node with givig XPath to bind to " + propertyInfo.PropertyType.FullName + " " + propertyInfo.Name);
}
IList result = T_Types[0].CreateIListOfType();
#region Property_Is_IEnumerable<HasXPath-UserDefinedClass>
if (T_Types[0].IsDefinedAttribute(typeof(HasXPathAttribute)) == true) // T is IEnumerable HasXPath-user-defined class (T type Defined XPath properties)
{
foreach (HtmlNode node in nodeCollection)
{
HtmlDocument innerHtmlDocument = new HtmlDocument();
innerHtmlDocument.LoadHtml(node.InnerHtml);
object o = GetEncapsulatedData(T_Types[0], innerHtmlDocument);
result.Add(o);
}
}
#endregion Property_Is_IEnumerable<HasXPath-UserDefinedClass>
#region Property_Is_IEnumerable<SimpleClass>
else // T is value-type or .Net class or user-defined class ( without xpath )
{
if (xPathAttribute.AttributeName == null) // It target value
{
try
{
result = Tools.GetNodesValuesBasedOnXPathReturnType(nodeCollection, xPathAttribute, T_Types[0]);
}
catch (FormatException)
{
throw new FormatException("Can not convert Invalid string in node collection to " + T_Types[0].FullName + " " + propertyInfo.Name);
}
catch (Exception ex)
{
throw new Exception("Unhandled Exception : " + ex.Message);
}
}
else // It target attribute
{
foreach (HtmlNode node in nodeCollection)
{
string nodeAttributeValue = node.GetAttributeValue(xPathAttribute.AttributeName, null);
if (nodeAttributeValue == null)
{
throw new NodeAttributeNotFoundException("Can not find " + xPathAttribute.AttributeName + " Attribute in " + node.Name + " related to " +
propertyInfo.PropertyType.FullName + " " + propertyInfo.Name);
}
object resultCastedToTargetPropertyType;
try
{
resultCastedToTargetPropertyType = Convert.ChangeType(nodeAttributeValue, T_Types[0]);
}
catch (FormatException) // if it can not cast result(string) to type of property.
{
throw new FormatException("Can not convert Invalid string to " + T_Types[0].FullName + " " + propertyInfo.Name);
}
catch (Exception ex)
{
throw new Exception("Unhandled Exception : " + ex.Message);
}
result.Add(resultCastedToTargetPropertyType);
}
}
}
#endregion Property_Is_IEnumerable<SimpleClass>
if (result == null || result.Count == 0)
{
throw new Exception("Cannot fill " + propertyInfo.PropertyType.FullName + " " + propertyInfo.Name + " because it is null.");
}
propertyInfo.SetValue(targetObject, result, null);
}
}
#endregion Property_IsNOT_IEnumerable
}
return(targetObject);
}
}
#endregion targetObject_Defined_XPath
#region targetObject_NOTDefined_XPath
else // Object doesen't have xpath attribute
{
throw new MissingXPathException("Type T must define HasXPath attribute and include properties with XPath attribute.");
}
#endregion targetObject_NOTDefined_XPath
}
public void Load()
{
Fields.Clear();
HttpClient httpclient = new HttpClient();
Task <string> task = httpclient.GetStringAsync("http://192.168.4.1");
task.Wait();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(task.Result);
if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0)
{
throw new Exception("Html parse error");
}
if (htmlDoc.DocumentNode == null)
{
throw new Exception("DocumentNode is null");
}
IEnumerable <HtmlNode> inputs = htmlDoc.DocumentNode.Descendants("input");
if (inputs == null)
{
throw new Exception("Unsupported device");
}
foreach (HtmlNode input in inputs)
{
FormField field = new FormField();
field.name = input.Attributes["name"] == null ? "" : input.Attributes["name"].Value;
field.value = input.Attributes["value"] == null ? "" : input.Attributes["value"].Value;
Fields.Add(field);
}
IEnumerable <HtmlNode> selects = htmlDoc.DocumentNode.Descendants("select");
foreach (HtmlNode select in selects)
{
FormField field = new FormField();
field.name = select.Attributes["name"] == null ? "" : select.Attributes["name"].Value;
IEnumerable <HtmlNode> options = select.Descendants("option").Where(n => n.Attributes["selected"] != null);
if (options != null && options.Count() == 1)
{
HtmlNode option = options.First();
field.value = option.Attributes["value"] == null ? "" : option.Attributes["value"].Value;
}
Fields.Add(field);
}
var upd = Fields.Where(n => n.name == "upd");
if (upd != null && upd.Count() != 1)
{
throw new Exception("No update capabilities");
}
var sid = Fields.Where(n => n.name == "sid");
var wpw = Fields.Where(n => n.name == "wpw");
var svr = Fields.Where(n => n.name == "svr");
string pattern = @"<h1>(.*)<\/h1><span>LAST STATE: (.*)<br>Firmware: (.*)<br>GUID: (.*)<br>MAC: (.*)<\/span>";
Match match = Regex.Match(task.Result, pattern);
if (match.Groups.Count != 6 || sid.Count() != 1 || wpw.Count() != 1 || svr.Count() != 1)
{
throw new Exception("Unsupported device");
}
Name = match.Groups[1].Value.Trim();
LastDeviceState = match.Groups[2].Value.Trim();
Firmware = match.Groups[3].Value.Trim();
GUID = match.Groups[4].Value.Trim();
MAC = match.Groups[5].Value.Trim();
}
private bool GetAllListPageUrl(IListSheet listSheet)
{
string exportDir = this.RunPage.GetExportDir();
string pageSourceDir = this.RunPage.GetDetailSourceFileDir();
Dictionary <string, int> resultColumnDic = new Dictionary <string, int>();
resultColumnDic.Add("detailPageUrl", 0);
resultColumnDic.Add("detailPageName", 1);
resultColumnDic.Add("cookie", 2);
resultColumnDic.Add("grabStatus", 3);
resultColumnDic.Add("giveUpGrab", 4);
resultColumnDic.Add("species", 5);
resultColumnDic.Add("year", 6);
string resultFilePath = Path.Combine(exportDir, "oalib获取列表页.xlsx");
ExcelWriter resultEW = new ExcelWriter(resultFilePath, "List", resultColumnDic, null);
string detailPageUrlColumnName = SysConfig.DetailPageUrlFieldName;
for (int i = 0; i < listSheet.RowCount; i++)
{
Dictionary <string, string> row = listSheet.GetRow(i);
bool giveUp = "Y".Equals(row[SysConfig.GiveUpGrabFieldName]);
if (!giveUp)
{
string url = row[detailPageUrlColumnName];
string cookie = row["cookie"];
string species = row["species"].Trim();
string year = row["year"].Trim();
string localFilePath = this.RunPage.GetFilePath(url, pageSourceDir);
TextReader tr = null;
try
{
tr = new StreamReader(localFilePath);
string webPageHtml = tr.ReadToEnd();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(webPageHtml);
HtmlNodeCollection allPageNodes = htmlDoc.DocumentNode.SelectNodes("//ul[@id=\"pages\"]/li");
if (allPageNodes != null)
{
HtmlNode lastPageNode = allPageNodes[allPageNodes.Count - 1];
string pageCountStr = lastPageNode.InnerText.Trim();
int pageCount = int.Parse(pageCountStr);
string searchKeyStr = CommonUtil.UrlEncode(species);
for (int j = 0; j < pageCount; j++)
{
string pageIndexStr = (j + 1).ToString();
string detailPageName = species + "_" + year + "_" + pageIndexStr;
string detailPageUrl = "http://www.oalib.com/search?type=0&oldType=0&kw=%22" + searchKeyStr + "%22&searchField=All&__multiselect_searchField=&fromYear=" + year + "&toYear=&pageNo=" + pageIndexStr;
Dictionary <string, string> f2vs = new Dictionary <string, string>();
f2vs.Add("detailPageUrl", detailPageUrl);
f2vs.Add("detailPageName", detailPageName);
f2vs.Add("cookie", cookie);
f2vs.Add("species", species);
f2vs.Add("year", year);
resultEW.AddRow(f2vs);
}
}
}
catch (Exception ex)
{
if (tr != null)
{
tr.Dispose();
tr = null;
}
this.RunPage.InvokeAppendLogText("读取出错. " + ex.Message + " LocalPath = " + localFilePath, LogLevelType.Error, true);
throw ex;
}
}
}
resultEW.SaveToDisk();
return(true);
}
public DataTable ConsultaTCSunat(DateTime Fecha)
{
try
{
año = Fecha.ToString("yyyy");
mes = Fecha.ToString("MM");
WebBrowser objWebBrowser = new WebBrowser();
objWebBrowser.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(WebBrowserDocumentCompleted);
objWebBrowser.Navigate("http://www.sunat.gob.pe/cl-at-ittipcam/tcS01Alias");
//Se queda en espera hasta que obtenga respuesta de la consulta
while (rpta == false)
{
Application.DoEvents();
}
//Obtenemos Informacion
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(cadhtml);
HtmlAgilityPack.HtmlNodeCollection NodesTr = document.DocumentNode.SelectNodes("//table[@class='class=\"form-table\"']//tr");
if (NodesTr != null)
{
DataTable dt = new DataTable();
dt.Columns.Add("Día", typeof(String));
dt.Columns.Add("Compra", typeof(String));
dt.Columns.Add("Venta", typeof(String));
int iNumFila = 0;
foreach (HtmlNode Node in NodesTr)
{
if (iNumFila > 0)
{
int iNumColumna = 0;
DataRow dr = dt.NewRow();
foreach (HtmlNode subNode in Node.Elements("td"))
{
if (iNumColumna == 0)
{
dr = dt.NewRow();
}
string sValue = subNode.InnerHtml.ToString().Trim();
sValue = System.Text.RegularExpressions.Regex.Replace(sValue, "<.*?>", " ");
dr[iNumColumna] = sValue;
iNumColumna++;
if (iNumColumna == 3)
{
dt.Rows.Add(dr);
iNumColumna = 0;
}
}
}
iNumFila++;
}
dt.AcceptChanges();
return(dt);
//dgvTipo.DataSource = dt;
//dgvTipo.ReadOnly = true;
//dgvTipo.AllowUserToAddRows = false;
//dgvTipo.AllowUserToDeleteRows = false;
//dgvTipo.AllowUserToOrderColumns = false;
//dgvTipo.RowHeadersVisible = false;
}
consulta = false;
rpta = false;
mes = "";
año = "";
cadhtml = "";
return(null);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "Tipo de Cambio", MessageBoxButtons.OK, MessageBoxIcon.Error);
return(null);
}
}
static System.Collections.Generic.List <BetDTO> ConvertData(string data, System.Collections.Generic.Dictionary <string, System.DateTime> oldBetList, string accountName)
{
System.Collections.Generic.List <BetDTO> list = null;
if (data != string.Empty)
{
if (!data.Contains("No information is available"))
{
list = new List <BetDTO>();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(data);
foreach (HtmlNode table in doc.DocumentNode.SelectNodes("//table/tr/td/table"))
{
foreach (HtmlNode row in table.SelectNodes("tr"))
{
//iBet.Utilities.WriteLog.Write("------------Bet-------------");
HtmlNodeCollection cells = row.SelectNodes("td");
string refID = cells[1].InnerText.Substring(8, 10);
if (row.InnerText.Contains("Soccer"))
{
//string stringEntry = cells[2].FirstChild.FirstChild.FirstChild.InnerText;
string stringEntry = cells[2].FirstChild.FirstChild.NextSibling.InnerText;
if (!stringEntry.Contains("Mix Parlay"))
{
BetDTO betDTO = new BetDTO();
betDTO.Account = accountName;
betDTO.RefID = refID;
betDTO.DateTime = DateTime.Parse(cells[1].FirstChild.NextSibling.InnerText);
string string1 = string.Empty, string2 = string.Empty, string3 = string.Empty, string4 = string.Empty;
string[] array1 = null, array2 = null;
try
{
if (stringEntry != string.Empty && !stringEntry.Contains("FT.") && !stringEntry.Contains("HT."))
{
betDTO.Dumb = false;
//betDTO.Choice = cells[2].FirstChild.FirstChild.FirstChild.FirstChild.InnerText;
betDTO.Choice = cells[2].FirstChild.FirstChild.NextSibling.FirstChild.InnerText.Trim();
betDTO.Odd = cells[2].FirstChild.FirstChild.NextSibling.FirstChild.NextSibling.InnerText.Trim();
//if (cells[2].FirstChild.FirstChild.FirstChild.FirstChild.NextSibling.LastChild.InnerHtml.Contains("["))
//string1 = cells[2].FirstChild.FirstChild.FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.InnerText.Trim().Replace("[", "").Replace("]", "");
if (stringEntry.Contains("["))
{
array1 = stringEntry.Split(new string[] { "[" }, System.StringSplitOptions.None);
array2 = array1[1].Split(new string[] { "]" }, System.StringSplitOptions.None);
string1 = array2[0];
}
//string1 = cells[2].FirstChild.FirstChild.FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.InnerText.Trim().Replace("[", "").Replace("]", "");
string2 = cells[2].FirstChild.FirstChild.NextSibling.InnerText;
string3 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.InnerText;
string4 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.InnerText;
betDTO.League = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.NextSibling.InnerText;
}
else
{
string stringEntry2 = cells[2].FirstChild.FirstChild.FirstChild.NextSibling.FirstChild.InnerText;
bool _htChoice = stringEntry2.Contains("HT.");
bool _ftChoice = stringEntry2.Contains("FT.");
if (_htChoice || _ftChoice)
{
betDTO.Dumb = false;
betDTO.Choice = stringEntry2.Replace(" ", "");
betDTO.Odd = "12";
try
{
string1 = cells[2].FirstChild.FirstChild.FirstChild.NextSibling.FirstChild.NextSibling.InnerText.Replace("[", "").Replace("]", "");
}
catch
{
string1 = string.Empty;
}
string2 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.FirstChild.InnerText;
string3 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.InnerText;
string4 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.InnerText;
betDTO.League = cells[2].FirstChild.FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.NextSibling.FirstChild.NextSibling.InnerText;
}
else
{
betDTO.Dumb = false;
betDTO.Choice = stringEntry2;
betDTO.Odd = cells[2].FirstChild.FirstChild.NextSibling.FirstChild.NextSibling.InnerText;
if (cells[2].FirstChild.FirstChild.NextSibling.LastChild.InnerHtml.Contains("["))
{
string1 = cells[2].FirstChild.FirstChild.FirstChild.NextSibling.FirstChild.NextSibling.NextSibling.NextSibling.InnerText.Trim().Replace("[", "").Replace("]", "");
}
string2 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.FirstChild.InnerText;
string3 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.FirstChild.InnerText;
string4 = cells[2].FirstChild.FirstChild.NextSibling.NextSibling.InnerText;
betDTO.League = cells[2].FirstChild.FirstChild.FirstChild.NextSibling.NextSibling.NextSibling.NextSibling.FirstChild.NextSibling.InnerText.Replace(" ", "");
}
}
//
if (betDTO.Dumb != true)
{
if (string1 != string.Empty)
{
betDTO.Score = string1;
array1 = string1.Split(new string[] { "-" }, System.StringSplitOptions.None);
betDTO.HomeScore = array1[0];
betDTO.AwayScore = array1[1];
betDTO.Live = true;
}
else
{
betDTO.Score = "?-?";
betDTO.HomeScore = "0";
betDTO.AwayScore = "0";
betDTO.Live = false;
}
if (string3.Contains("nbsp"))
{
array2 = string3.Split(new string[] { " - vs - " }, System.StringSplitOptions.None);
betDTO.HomeTeamName = array2[0];
betDTO.AwayTeamName = array2[1].TrimEnd(' ');
}
else
{
//iBet.Utilities.WriteLog.Write("Can not parse name of match: " + string3);
}
if (string4 != string.Empty)
{
if (string4.Contains("1st Handicap"))
{
betDTO.Type = eOddType.FirstHalfHandicap;
}
else if (string4.Contains("1st Over"))
{
betDTO.Type = eOddType.FirstHalfOverUnder;
}
else if (string4.Contains("Handicap"))
{
betDTO.Type = eOddType.FulltimeHandicap;
}
else if (string4.Contains("Over"))
{
betDTO.Type = eOddType.FulltimeOverUnder;
}
else if (string4.Contains("FT.1X2"))
{
betDTO.Type = eOddType.FT;
}
else if (string4.Contains("1st 1X2"))
{
betDTO.Type = eOddType.HT;
}
else
{
betDTO.Type = eOddType.Unknown;
}
}
betDTO.OddValue = cells[3].FirstChild.InnerText;
betDTO.Stake = int.Parse(cells[4].InnerText.Replace(",", ""));
//if (cells[5].FirstChild.InnerText.StartsWith("Run"))
// betDTO.Status = true;
//else
// betDTO.Status = false;
//betDTO.IP = cells[5].FirstChild.NextSibling.InnerText;
}
//
}
catch (Exception ex)
{
betDTO.Dumb = true;
//iBet.Utilities.WriteLog.Write("----bet convert Dumb----");
//iBet.Utilities.WriteLog.Write("Meassge: " + ex);
//iBet.Utilities.WriteLog.Write(row.InnerHtml);
//iBet.Utilities.WriteLog.Write("----end----");
}
if (betDTO.Dumb == false)
{
list.Add(betDTO);
}
}
}
}
}
}
}
return(list);
}
//데이터를 저장하는 함수
public void collectdata(HtmlAgilityPack.HtmlDocument document)
{
//list source
IEnumerable <HtmlAgilityPack.HtmlNode> values =
document.DocumentNode.SelectSingleNode(".//div[@class='price_diff_lst']").Descendants().Where(
x =>
x.Name == "table" && x.Attributes.Contains("border") &&
x.Attributes["border"].Value.Split().Contains("1"));
//데이터를 빼오기위해 필요한 변수들
String totalhtml;
int index;
int index2;
String urlhtml;
values.First().Remove();
foreach (HtmlAgilityPack.HtmlNode value in values)
{
if (value.NodeType != HtmlAgilityPack.HtmlNodeType.Text)
{
try
{
//사이트이름 뽑아낸다
try { name = value.SelectSingleNode(".//img").GetAttributeValue("alt", ""); }
catch (NullReferenceException e1) { name = value.SelectSingleNode(".//span[@class='mall_type']").InnerText; }
//가격 뽑아낸다
price = value.Descendants().Where(x => x.Name == "td").ElementAt(2).SelectSingleNode(".//span").InnerText.Trim();
//배송비 뽑아낸다
delivery = value.SelectSingleNode(".//p[@class='condition']").InnerText.Trim();
//카드정보 뽑아낸다
profit = value.SelectSingleNode(".//em").InnerText;
String[] tmp = profit.Split('\n');
profit = "";
foreach (var VARIABLE in tmp)
{
profit = VARIABLE.Trim();
}
//url뽑아낸다
geturl(value.SelectSingleNode(".//a").GetAttributeValue("href", ""), 1);
}
catch (UriFormatException e)
{
Console.WriteLine("네이버 UriFormatException");
}
catch (NullReferenceException e)
{
try { name = value.SelectSingleNode(".//span[@class='mall_type']").InnerText; }
catch (NullReferenceException e1) { }
}
catch (ArgumentException e)
{
Console.WriteLine("네이버 ArgumentException");
}
catch (IndexOutOfRangeException e)
{
Console.WriteLine("네이버 IndexOutOfRangeException");
}
catch (InvalidOperationException e)
{
Console.WriteLine("네이버 InvalidOperationException");
}
finally
{
if (name != "")
{
try
{
int tmp = 0;
int tmp2 = 0;
if (name != "옥션")
{
itemNum = "";
}
if (name.Trim() == "11번가")
{
tmp = realurl.IndexOf(@"prdNo=") + 6;
tmp2 = realurl.IndexOf(@"&Na");
itemNum = realurl.Substring(tmp, tmp2 - tmp);
}
else if (name == "G마켓")
{
tmp = realurl.IndexOf("=") + 1;
tmp2 = realurl.IndexOf("&");
itemNum = realurl.Substring(tmp, tmp2 - tmp);
}
else if (name == "GSSHOP")
{
tmp = realurl.IndexOf("=") + 1;
tmp2 = realurl.IndexOf("&");
itemNum = realurl.Substring(tmp, tmp2 - tmp);
}
else if (name == "CJmall")
{
tmp = realurl.IndexOf("=") + 1;
tmp2 = realurl.IndexOf("&");
itemNum = realurl.Substring(tmp, tmp2 - tmp);
}
}
catch (ArgumentException) { Console.WriteLine("아이템넘버 exception"); }
datas = new string[6];
datas[0] = realurl.Trim();
datas[1] = name.Trim();
datas[2] = itemNum.Trim();
datas[3] = price.Trim();
datas[4] = profit.Trim();
datas[5] = delivery.Trim();
naverdatalist.Add(datas);
}
}
}
}
}
internal HtmlDocumentLoadCompleted(HtmlDocument doc)
{
Document = doc;
}
private void CheckForNews(string siteUrl)
{
try
{
string originalPage = string.Empty;
while (Thread.CurrentThread.IsAlive)
{
Thread.Sleep(sleep);
string newHTML = GetCode(siteUrl);
originalPage = string.IsNullOrEmpty(originalPage) ? newHTML: originalPage;
if (!string.IsNullOrEmpty(newHTML) && !originalPage.Equals(newHTML))
{
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(newHTML);
var linkedPages = doc.DocumentNode.Descendants("a")
.Select(a => a.GetAttributeValue("href", null))
.Where(u => !string.IsNullOrEmpty(u));
originalPage = newHTML;
WriteCodeOnFile(originalPage);
string subject = "WEBSITE UPDATE ALERT";
string message = string.Format("New update available on {0}", siteUrl);
// Find all the links in the page, remove socials, adds the domain to the absolute one
foreach (string link in linkedPages)
{
if (link.ToLower().Contains("facebook") ||
link.ToLower().Contains("twitter") ||
link.ToLower().Contains("vimeo"))
{
continue;
}
string url;
if (link.StartsWith("/"))
{
string domain = siteUrl.Substring(0, siteUrl.IndexOf("/", siteUrl.IndexOf("/") + 2));
url = string.Concat(domain, link);
}
else
{
url = link;
}
message = string.Concat(message, Environment.NewLine, url);
}
if (DateTime.Now.DayOfWeek == DayOfWeek.Sunday || DateTime.Now.DayOfWeek == DayOfWeek.Saturday ||
DateTime.Now.Hour < 08 || DateTime.Now.Hour > 17)
{
Tuple <bool, string> answer = mailSender.sendMail(subject, message);
if (!answer.Item1)
{
WriteCodeOnFile(string.Concat(message, Environment.NewLine, string.Format("Unable to send mail:\r\n{0}", answer.Item2)));
}
}
else
{
this.Invoke(new Action(() =>
{
latestUpdate.Text = string.Format(LATEST_UPDATE, DateTime.Now.ToString("dd/MM/yyyy HH:mm"));
}));
this.Invoke(new Action(() =>
{
MessageBox.Show(
this,
message,
subject,
MessageBoxButtons.OK,
MessageBoxIcon.Exclamation);
}));
Process.Start("chrome.exe", siteUrl);
}
}
counter++;
this.Invoke(new Action(() =>
{
Counter.Text = string.Format(CYCLE_COUNTER, counter.ToString());
}));
}
} catch (Exception ex)
{
}
}
// этот метод парсит обхявление
public async void ParseArticles(Request[] refs)
{
foreach (var item in refs)
{
int time1 = x.Next(2000, 6000);
Thread.Sleep(time1);
try
{
//NetworkCredential credentials = new NetworkCredential();
Task <HtmlAgilityPack.HtmlDocument> _task = new Task <HtmlAgilityPack.HtmlDocument>(() => { return(new HtmlWeb().Load(item.Url, "GET")); }); _task.Start();
HtmlAgilityPack.HtmlDocument doc = await _task.ConfigureAwait(false);
// 3 это формирует запрос, который уходит на сервер и обрабатывается там и выдает ответ.
//HttpWebRequest request = WebRequest.Create(item.Url) as HttpWebRequest;
////// Obtain the 'Proxy' of the Default browser.
////IWebProxy proxy = request.Proxy;
////WebProxy myProxy = new WebProxy();
////string proxyAddress;
////try
////{
//// proxyAddress = "http://201.55.46.6:80";
//// if (proxyAddress.Length > 0)
//// {
//// Uri newUri = new Uri(proxyAddress);
//// // Associate the newUri object to 'myProxy' object so that new myProxy settings can be set.
//// myProxy.Address = newUri;
//// // Create a NetworkCredential object and associate it with the
//// // Proxy property of request object.
//// request.Proxy = myProxy;
//// }
////}
////catch (Exception ex)
////{
//// ex.ToString();
////}
////request.Proxy = new WebProxy("87.98.147.195", 3128);
//request.UserAgent = item.UserAgent;
//request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*;q=0.8";
//request.KeepAlive = true;
//request.AllowAutoRedirect = true;
//request.Timeout = 60000;
//request.Method = "POST";
//request.Referer = item.UrlReferer;
//request.Headers["Accept-Language"] = "ru-RU";
//// 4 получаем ответ
//HttpWebResponse response = request.GetResponse() as HttpWebResponse;
//// 5 поток данных получаемых с сервера
//StreamReader sr = new StreamReader(response.GetResponseStream());
//sr.ReadLine();
//string html = sr.ReadToEnd();
//HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
//doc.LoadHtml(html);
int time2 = x.Next(2000, 6000);
Thread.Sleep(time2);
//парсим номер телефона
var _phone = await _parsePhone(AvitoUrl + doc.DocumentNode.SelectSingleNode("//a[@title='Телефон продавца']").Attributes["href"].Value + "?async", item.Url, item.UserAgent);
//Articles _article = new Articles();
//_article.Url = item.Url;
//_article.Phone = _phone;
//Console.WriteLine(_article.Url + " " + _article.Phone);
int time3 = time1 + time2;
stringBuilder.Append(item.Url + "; " + _phone + "; " + time3).AppendLine();
counter++;
SaveAs saveAs = new SaveAs();
saveAs.SaveAsCSV(stringBuilder, counter.ToString());
stringBuilder.Clear();
//if (counter > 100)
//{
if (counter % 10 == 0)
{
Thread.Sleep(x.Next(counter / 2, counter));
//SaveAs saveAs = new SaveAs();
//saveAs.SaveAsCSV(stringBuilder, counter);
//MessageBox.Show(stringBuilder.ToString());
}
//}
}
catch (Exception ex)
{
counter++;
StringBuilder sb = new StringBuilder();
sb.Append(item.Url + "/r/n" + ex.ToString());
SaveAs saveAs = new SaveAs();
saveAs.SaveAsCSV(sb, counter.ToString() + " error");
sb.Clear();
Thread.Sleep(x.Next(counter * 5, counter * 20));
error.Add(true);
if (error.Count > 2)
{
Thread.Sleep(x.Next(counter * 10, counter * 20));
//error.Clear();
}
}
}
}
