public void Read()
{
ConnectionStreamResponse r = new ConnectionStreamResponse()
{
NextStart = 0
};
while (!r.EndOfConnections)
{
r = RequestConnectionStream(r.NextStart);
if (r == null || !(String.IsNullOrEmpty(r.Error)))
{
//return on error
return;
}
else
{
Connections.AddRange(r.ConnectionEntries);
}
if (!r.EndOfConnections)
{
Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests));
}
}
}
public ConnectionStreamResponse RequestConnectionStream(int nextStart)
{
ConnectionStreamResponse fullResponse = new ConnectionStreamResponse();
try
{
HttpWebRequest request = BuildRequest(Direction, nextStart);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Encoding responseEncoding = Encoding.GetEncoding(response.CharacterSet);
String result = "";
using (StreamReader sr = new StreamReader(response.GetResponseStream(), responseEncoding))
{
result = sr.ReadToEnd();
JObject model = JObject.Parse(result);
fullResponse.NextStart = (int)model["nextStart"];
String htmlDocument = (String)model["html"];
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.OptionWriteEmptyNodes = true;
doc.LoadHtml(htmlDocument);
var connectionNodes = doc.DocumentNode.SelectNodes(String.Format("//ul[@id='profileGrid']//li"));
foreach (var connectionNode in connectionNodes)
{
ConnectionEntry entry = new ConnectionEntry();
entry.Direction = Direction;
entry.UserURL = connectionNode.SelectSingleNode("div//a")?.Attributes["href"]?.Value;
if (!String.IsNullOrEmpty(entry.UserURL) && entry.UserURL.StartsWith(@"/"))
{
entry.UserName = entry.UserURL.Replace(@"/", "");
entry.UserURL = string.Format(@"https://myspace.com{0}", entry.UserURL);
}
entry.ThumbnailURL = connectionNode.SelectSingleNode("div//a//img")?.Attributes["src"]?.Value;
entry.PersonalName = connectionNode.SelectSingleNode("div//a//div//h6")?.InnerHtml;
entry.ProfileID = connectionNode.SelectSingleNode(String.Format("//div[@data-title='{0}']", entry.PersonalName))?.Attributes["data-id"]?.Value;
entry.ArtistID = connectionNode.SelectSingleNode(String.Format("//div[@data-title='{0}']", entry.PersonalName))?.Attributes["data-artist-id"]?.Value;
if (!String.IsNullOrEmpty(entry.UserURL))
{
fullResponse.ConnectionEntries.Add(entry);
}
}
}
}
catch (Exception e)
{
fullResponse.Error = e.Message;
}
return(fullResponse);
}
