public void Read()
{
PhotoStreamResponse r = new PhotoStreamResponse()
{
EndOfPhotos = false,
LastPhotoID = null
};
while (!r.EndOfPhotos)
{
r = RequestPhotoStream(r.LastPhotoID);
if (r == null || !(String.IsNullOrEmpty(r.Error)))
{
//return on error
return;
}
else
{
Photos.AddRange(r.PhotosEntries);
}
if (!r.EndOfPhotos)
{
Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests));
}
}
}
public PhotoStreamResponse RequestPhotoStream(String startingImageID)
{
PhotoStreamResponse fullResponse = new PhotoStreamResponse();
try
{
HttpWebRequest request = BuildRequest(startingImageID);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Encoding responseEncoding = Encoding.GetEncoding(response.CharacterSet);
String result = "";
using (StreamReader sr = new StreamReader(response.GetResponseStream(), responseEncoding))
{
result = sr.ReadToEnd();
JObject model = JObject.Parse(result);
fullResponse.EndOfPhotos = (bool)model["endOfPhotos"];
String htmlDocument = (String)model["view"];
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.OptionWriteEmptyNodes = true;
doc.LoadHtml(htmlDocument);
var photoNodes = doc.DocumentNode.SelectNodes(String.Format("//ul[@id='photosContainer']//li"));
if (photoNodes != null)
{
foreach (var photoNode in photoNodes)
{
PhotoEntry entry = new PhotoEntry();
entry.Caption = photoNode.SelectSingleNode("div//div//span[@class='photoCaption postText']")?.InnerText;
entry.ThumbnailImageURL = photoNode.SelectSingleNode("a//img")?.Attributes["src"]?.Value;
entry.FullImageURL = !String.IsNullOrEmpty(entry.ThumbnailImageURL) ? CrawlUtil.ModifyUriFileName(entry.ThumbnailImageURL, x => "full") : null;
entry.PhotoID = photoNode.Attributes["data-photoId"]?.Value;
entry.AlbumName = photoNode.SelectSingleNode("span[@itemprop='name']")?.InnerText;
entry.DetailPageURL = photoNode.SelectSingleNode("a")?.Attributes["content"]?.Value;
if (!String.IsNullOrEmpty(entry.DetailPageURL))
{
entry.DetailPageURL = String.Format(@"https://myspace.com{0}", entry.DetailPageURL);
ParseDetailPage(entry, entry.DetailPageURL);
}
if (!String.IsNullOrEmpty(entry.PhotoID))
{
fullResponse.PhotosEntries.Add(entry);
}
}
}
}
}
catch (Exception e)
{
fullResponse.Error = e.Message;
}
if (fullResponse.PhotosEntries != null & fullResponse.PhotosEntries.Count > 0)
{
fullResponse.LastPhotoID = fullResponse.PhotosEntries[fullResponse.PhotosEntries.Count - 1]?.PhotoID;
}
return(fullResponse);
}
