public PhotoStreamResponse RequestPhotoStream(String startingImageID)
{
PhotoStreamResponse fullResponse = new PhotoStreamResponse();
try
{
HttpWebRequest request = BuildRequest(startingImageID);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Encoding responseEncoding = Encoding.GetEncoding(response.CharacterSet);
String result = "";
using (StreamReader sr = new StreamReader(response.GetResponseStream(), responseEncoding))
{
result = sr.ReadToEnd();
JObject model = JObject.Parse(result);
fullResponse.EndOfPhotos = (bool)model["endOfPhotos"];
String htmlDocument = (String)model["view"];
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.OptionWriteEmptyNodes = true;
doc.LoadHtml(htmlDocument);
var photoNodes = doc.DocumentNode.SelectNodes(String.Format("//ul[@id='photosContainer']//li"));
if (photoNodes != null)
{
foreach (var photoNode in photoNodes)
{
PhotoEntry entry = new PhotoEntry();
entry.Caption = photoNode.SelectSingleNode("div//div//span[@class='photoCaption postText']")?.InnerText;
entry.ThumbnailImageURL = photoNode.SelectSingleNode("a//img")?.Attributes["src"]?.Value;
entry.FullImageURL = !String.IsNullOrEmpty(entry.ThumbnailImageURL) ? CrawlUtil.ModifyUriFileName(entry.ThumbnailImageURL, x => "full") : null;
entry.PhotoID = photoNode.Attributes["data-photoId"]?.Value;
entry.AlbumName = photoNode.SelectSingleNode("span[@itemprop='name']")?.InnerText;
entry.DetailPageURL = photoNode.SelectSingleNode("a")?.Attributes["content"]?.Value;
if (!String.IsNullOrEmpty(entry.DetailPageURL))
{
entry.DetailPageURL = String.Format(@"https://myspace.com{0}", entry.DetailPageURL);
ParseDetailPage(entry, entry.DetailPageURL);
}
if (!String.IsNullOrEmpty(entry.PhotoID))
{
fullResponse.PhotosEntries.Add(entry);
}
}
}
}
}
catch (Exception e)
{
fullResponse.Error = e.Message;
}
if (fullResponse.PhotosEntries != null & fullResponse.PhotosEntries.Count > 0)
{
fullResponse.LastPhotoID = fullResponse.PhotosEntries[fullResponse.PhotosEntries.Count - 1]?.PhotoID;
}
return(fullResponse);
}
public void Read()
{
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.OptionWriteEmptyNodes = true;
try
{
var webRequest = HttpWebRequest.Create(String.Format(@"https://myspace.com/{0}/music/songs", UserName));
((HttpWebRequest)webRequest).UserAgent = CrawlUtil.GetUserAgent();
Stream stream = webRequest.GetResponse().GetResponseStream();
doc.Load(stream);
stream.Close();
var songsNode = doc.DocumentNode.SelectNodes(@"//button[@class='playBtn play_25 song']");
if (songsNode != null)
{
foreach (var songNode in songsNode)
{
SongEntry entry = new SongEntry();
#region Parse summary page
entry.SongID = songNode?.Attributes["data-song-id"]?.Value;
entry.SongTitle = songNode?.Attributes["data-title"]?.Value;
entry.SongURL = songNode?.Attributes["data-url"]?.Value;
if (!String.IsNullOrEmpty(entry.SongURL))
{
entry.SongURL = String.Format(@"https://myspace.com{0}", entry.SongURL);
}
entry.AlbumID = songNode?.Attributes["data-album-id"]?.Value;
entry.AlbumTitle = songNode?.Attributes["data-album-title"]?.Value;
entry.AlbumURL = songNode?.Attributes["data-album-url"]?.Value;
if (!String.IsNullOrEmpty(entry.AlbumURL))
{
entry.AlbumURL = String.Format(@"https://myspace.com{0}", entry.AlbumURL);
}
entry.ArtistID = songNode?.Attributes["data-artist-id"]?.Value;
entry.ArtistTitle = songNode?.Attributes["data-artist-name"]?.Value;
entry.ArtistURL = songNode?.Attributes["data-artist-url"]?.Value;
if (!String.IsNullOrEmpty(entry.ArtistURL))
{
entry.ArtistURL = String.Format(@"https://myspace.com{0}", entry.ArtistURL);
}
entry.DurationInSeconds = songNode?.Attributes["data-duration"]?.Value;
entry.VideoID = songNode?.Attributes["data-video-id"]?.Value;
entry.YoutubeID = songNode?.Attributes["data-youtube-id"]?.Value;
if (!String.IsNullOrEmpty(entry.YoutubeID))
{
entry.YoutubeURL = String.Format(@"https://www.youtube.com/watch?v={0}", entry.YoutubeID);
}
entry.ImageThumbnailURL = songNode?.Attributes["data-image-url"]?.Value;
entry.ImageURL = !String.IsNullOrEmpty(entry.ImageThumbnailURL) ? CrawlUtil.ModifyUriFileName(entry.ImageThumbnailURL, x => "full") : null;
entry.GenreID = songNode?.Attributes["data-genre-id"]?.Value;
entry.GenreName = songNode?.Attributes["data-genre-name"]?.Value;
entry.MediaID = songNode?.Attributes["data-media-id"]?.Value;
entry.MediaType = songNode?.Attributes["data-media-type"]?.Value;
entry.UID = songNode?.Attributes["data-uid"]?.Value;
String isPremiumFlag = songNode?.Attributes["data-is-premium"]?.Value;
entry.IsPremium = isPremiumFlag != null && isPremiumFlag.ToLower().Equals("true");
String isExplicitFlag = songNode?.Attributes["data-is-explicit"]?.Value;
entry.IsExplicit = isExplicitFlag != null && isExplicitFlag.ToLower().Equals("true");
String isFullLength = songNode?.Attributes["data-is-full-length"]?.Value;
entry.IsFullLength = isFullLength != null && isFullLength.ToLower().Equals("true");
String isAdsProhibited = songNode?.Attributes["data-ads-prohibited"]?.Value;
entry.IsAdsProhibited = isAdsProhibited != null && isAdsProhibited.ToLower().Equals("true");
#endregion
#region Parse detial page
if (!String.IsNullOrEmpty(entry.SongURL))
{
Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests));
try
{
var detailDoc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
detailDoc.OptionWriteEmptyNodes = true;
var webDetailRequest = HttpWebRequest.Create(entry.SongURL);
((HttpWebRequest)webDetailRequest).UserAgent = CrawlUtil.GetUserAgent();
Stream detailStream = webDetailRequest.GetResponse().GetResponseStream();
detailDoc.Load(detailStream);
detailStream.Close();
var playsNodes = detailDoc.DocumentNode.SelectNodes(@"//div[@class='plays']");
if (playsNodes != null &&
playsNodes.Count >= 2 &&
String.Equals(playsNodes[0]?.InnerText, "PLAYS", StringComparison.OrdinalIgnoreCase))
{
entry.PlayCount = playsNodes[1]?.InnerText;
}
var asideNodes = detailDoc.DocumentNode.SelectNodes(@"//aside[@class='dotted top']");
if (asideNodes != null &&
asideNodes.Count >= 1 &&
asideNodes[0]?.InnerText != null &&
asideNodes[0].InnerText.Contains("Length"))
{
var songDetailItemNodesDt = asideNodes[0].SelectNodes("//dt");
var songDetailItemNodesDd = asideNodes[0].SelectNodes("//dd");
if (songDetailItemNodesDt != null &&
songDetailItemNodesDt.Count > 0 &&
songDetailItemNodesDd != null &&
songDetailItemNodesDd.Count > 0 &&
songDetailItemNodesDt.Count == songDetailItemNodesDd.Count)
{
Dictionary <String, int> tableIndex = new Dictionary <string, int>();
int songDetailItemDtCount = -1;
foreach (var item in songDetailItemNodesDt)
{
songDetailItemDtCount++;
tableIndex.Add(item.InnerText, songDetailItemDtCount);
}
if (tableIndex.ContainsKey("Label") &&
songDetailItemNodesDd.Count >= tableIndex["Label"])
{
int index = tableIndex["Label"];
entry.Label = songDetailItemNodesDd[index]?.InnerText;
}
if (tableIndex.ContainsKey("Release") &&
songDetailItemNodesDd.Count >= tableIndex["Release"])
{
int index = tableIndex["Release"];
entry.ReleaseDate = songDetailItemNodesDd[index]?.InnerText;
}
}
}
}
catch (Exception e2)
{
}
}
#endregion
if (!String.IsNullOrEmpty(entry.SongID))
{
Songs.Add(entry);
}
}
}
}
catch (Exception e)
{
}
}
/// <summary>
/// Parse root profile page.
/// </summary>
/// <param name="userName">Profile page to parse.</param>
private void ParseProfilePage(String userName)
{
String profileURL = String.Format(@"https://myspace.com/{0}", userName);
var doc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty;
doc.OptionWriteEmptyNodes = true;
try
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
var webRequest = HttpWebRequest.Create(profileURL);
((HttpWebRequest)webRequest).UserAgent = CrawlUtil.GetUserAgent();
Stream stream = webRequest.GetResponse().GetResponseStream();
doc.Load(stream);
stream.Close();
Profile.URL = String.Format(@"https://myspace.com/{0}", userName);
Profile.UserName = userName;
Profile.ProfileThumbnailImageURL = doc.DocumentNode.SelectSingleNode(@"//a[@id='profileImage']//img")?.Attributes["src"]?.Value;
Profile.ProfileImageURL = !String.IsNullOrEmpty(Profile.ProfileThumbnailImageURL) ? CrawlUtil.ModifyUriFileName(Profile.ProfileThumbnailImageURL, x => "600x600") : null;
Profile.ProfileID = doc.DocumentNode.SelectSingleNode(@"//div[@class='connectButton notReversed tooltips']")?.Attributes["data-id"]?.Value;
String privateFlag = doc.DocumentNode.SelectSingleNode(@"//div[@class='connectButton notReversed tooltips']")?.Attributes["data-is-private"]?.Value;
Profile.IsPrivate = privateFlag != null && privateFlag.ToLower().Equals("true");
Profile.PersonalName = doc.DocumentNode.SelectSingleNode(@"//div[@class='connectButton notReversed tooltips']")?.Attributes["data-title"]?.Value;
Profile.LocationDescription = doc.DocumentNode.SelectSingleNode(@"//div[@id='profileDetails']//div[@id='locAndWeb']//div[@class='location_white location ']")?.Attributes["data-display-text"]?.Value;
Profile.Website = doc.DocumentNode.SelectSingleNode(@"//div[@id='profileDetails']//div[@id='locAndWeb']//div[@class='ribbon_white website ']//a")?.InnerText;
Profile.OutConnectionTotal = doc.DocumentNode.SelectSingleNode(String.Format(@"//div[@id='profileDetails']//div[@id='connectionsCount']//a[@href='/{0}/connections/out']//span", Profile.UserName))?.InnerText;
Profile.InConnectionTotal = doc.DocumentNode.SelectSingleNode(String.Format(@"//div[@id='profileDetails']//div[@id='connectionsCount']//a[@href='/{0}/connections/in']//span", Profile.UserName))?.InnerText;
if (!Profile.IsPrivate)
{
var top8FriendsNode = doc.DocumentNode.SelectNodes(@"//div[@class='friendsWrapper']//ul//li//a");
if (top8FriendsNode != null)
{
foreach (var friendNode in top8FriendsNode)
{
Top8FriendEntry friendEntry = new Top8FriendEntry();
friendEntry.UserURL = friendNode?.Attributes["href"]?.Value;
if (!String.IsNullOrEmpty(friendEntry.UserURL) && friendEntry.UserURL.StartsWith("/"))
{
friendEntry.UserURL = string.Format(@"https://myspace.com{0}", friendEntry.UserURL);
}
friendEntry.ProfileID = friendNode?.Attributes["data-profileid"]?.Value;
friendEntry.ThumbnailURL = friendNode?.Attributes["data-image-url"]?.Value;
friendEntry.UserName = friendNode?.Attributes["data-title"]?.Value;
Profile.Top8Friends.Add(friendEntry);
}
}
}
}
catch (Exception e)
{
}
}