public void Read() { ConnectionStreamResponse r = new ConnectionStreamResponse() { NextStart = 0 }; while (!r.EndOfConnections) { r = RequestConnectionStream(r.NextStart); if (r == null || !(String.IsNullOrEmpty(r.Error))) { //return on error return; } else { Connections.AddRange(r.ConnectionEntries); } if (!r.EndOfConnections) { Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests)); } } }
public ConnectionStreamResponse RequestConnectionStream(int nextStart) { ConnectionStreamResponse fullResponse = new ConnectionStreamResponse(); try { HttpWebRequest request = BuildRequest(Direction, nextStart); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Encoding responseEncoding = Encoding.GetEncoding(response.CharacterSet); String result = ""; using (StreamReader sr = new StreamReader(response.GetResponseStream(), responseEncoding)) { result = sr.ReadToEnd(); JObject model = JObject.Parse(result); fullResponse.NextStart = (int)model["nextStart"]; String htmlDocument = (String)model["html"]; var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.OptionWriteEmptyNodes = true; doc.LoadHtml(htmlDocument); var connectionNodes = doc.DocumentNode.SelectNodes(String.Format("//ul[@id='profileGrid']//li")); foreach (var connectionNode in connectionNodes) { ConnectionEntry entry = new ConnectionEntry(); entry.Direction = Direction; entry.UserURL = connectionNode.SelectSingleNode("div//a")?.Attributes["href"]?.Value; if (!String.IsNullOrEmpty(entry.UserURL) && entry.UserURL.StartsWith(@"/")) { entry.UserName = entry.UserURL.Replace(@"/", ""); entry.UserURL = string.Format(@"https://myspace.com{0}", entry.UserURL); } entry.ThumbnailURL = connectionNode.SelectSingleNode("div//a//img")?.Attributes["src"]?.Value; entry.PersonalName = connectionNode.SelectSingleNode("div//a//div//h6")?.InnerHtml; entry.ProfileID = connectionNode.SelectSingleNode(String.Format("//div[@data-title='{0}']", entry.PersonalName))?.Attributes["data-id"]?.Value; entry.ArtistID = connectionNode.SelectSingleNode(String.Format("//div[@data-title='{0}']", entry.PersonalName))?.Attributes["data-artist-id"]?.Value; if (!String.IsNullOrEmpty(entry.UserURL)) { fullResponse.ConnectionEntries.Add(entry); } } } } catch (Exception e) { fullResponse.Error = e.Message; } return(fullResponse); }