public void R(string url) { HtmlWeb hw = new HtmlWeb(); HtmlDocument doc = hw.Load(url); List <string> hrefTags = new List <string>(); List <string> text = new List <string>(); foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//text()")) { if (node.InnerText.Length > 10) { text.Add(node.InnerText); } } foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]")) { HtmlAttribute att = link.Attributes["href"]; hrefTags.Add(att.Value); } List <string> uniqueUsers = new List <string>(); List <string> userTags = new List <string>(); foreach (var tag in hrefTags) { if (Regex.IsMatch(tag, @"users/\d+")) { var user = tag.Split(("/").ToCharArray()).Last(); userTags.Add(user); if (!uniqueUsers.Contains(user) && !MONITOR.listOfUniqueUsers.Contains(user)) { uniqueUsers.Add(user); MONITOR.listOfUniqueUsers.Add(user); using (StreamWriter sw = File.AppendText(@"D:/cache/cache-so-usernames.txt")) { sw.WriteLine(user); //MONITOR.Log(user); } } } } foreach (var tag in hrefTags) { if (Regex.IsMatch(tag, @"http.*?questions/\d+")) { if (!tag.Contains("promotion")) { if (!MONITOR.visitedpages.Contains(tag)) { MONITOR.Log(tag); System.Threading.Thread.Sleep(60000); MONITOR.visitedpages.Add(tag); using (N n = new N()) { n.R(tag); } } } } } }
public static void LogError(string data) { Console.ForegroundColor = ConsoleColor.Red; MONITOR.Log(data); Console.ResetColor(); }
public void Dispose() { MONITOR.Log("object has been disposed."); }
public static void LogDiagnostic(string data) { Console.ForegroundColor = ConsoleColor.Cyan; MONITOR.Log(data); Console.ResetColor(); }