public PhotoStreamResponse RequestPhotoStream(String startingImageID) { PhotoStreamResponse fullResponse = new PhotoStreamResponse(); try { HttpWebRequest request = BuildRequest(startingImageID); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); Encoding responseEncoding = Encoding.GetEncoding(response.CharacterSet); String result = ""; using (StreamReader sr = new StreamReader(response.GetResponseStream(), responseEncoding)) { result = sr.ReadToEnd(); JObject model = JObject.Parse(result); fullResponse.EndOfPhotos = (bool)model["endOfPhotos"]; String htmlDocument = (String)model["view"]; var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.OptionWriteEmptyNodes = true; doc.LoadHtml(htmlDocument); var photoNodes = doc.DocumentNode.SelectNodes(String.Format("//ul[@id='photosContainer']//li")); if (photoNodes != null) { foreach (var photoNode in photoNodes) { PhotoEntry entry = new PhotoEntry(); entry.Caption = photoNode.SelectSingleNode("div//div//span[@class='photoCaption postText']")?.InnerText; entry.ThumbnailImageURL = photoNode.SelectSingleNode("a//img")?.Attributes["src"]?.Value; entry.FullImageURL = !String.IsNullOrEmpty(entry.ThumbnailImageURL) ? CrawlUtil.ModifyUriFileName(entry.ThumbnailImageURL, x => "full") : null; entry.PhotoID = photoNode.Attributes["data-photoId"]?.Value; entry.AlbumName = photoNode.SelectSingleNode("span[@itemprop='name']")?.InnerText; entry.DetailPageURL = photoNode.SelectSingleNode("a")?.Attributes["content"]?.Value; if (!String.IsNullOrEmpty(entry.DetailPageURL)) { entry.DetailPageURL = String.Format(@"https://myspace.com{0}", entry.DetailPageURL); ParseDetailPage(entry, entry.DetailPageURL); } if (!String.IsNullOrEmpty(entry.PhotoID)) { fullResponse.PhotosEntries.Add(entry); } } } } } catch (Exception e) { fullResponse.Error = e.Message; } if (fullResponse.PhotosEntries != null & fullResponse.PhotosEntries.Count > 0) { fullResponse.LastPhotoID = fullResponse.PhotosEntries[fullResponse.PhotosEntries.Count - 1]?.PhotoID; } return(fullResponse); }
public void Read() { var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.OptionWriteEmptyNodes = true; try { var webRequest = HttpWebRequest.Create(String.Format(@"https://myspace.com/{0}/music/songs", UserName)); ((HttpWebRequest)webRequest).UserAgent = CrawlUtil.GetUserAgent(); Stream stream = webRequest.GetResponse().GetResponseStream(); doc.Load(stream); stream.Close(); var songsNode = doc.DocumentNode.SelectNodes(@"//button[@class='playBtn play_25 song']"); if (songsNode != null) { foreach (var songNode in songsNode) { SongEntry entry = new SongEntry(); #region Parse summary page entry.SongID = songNode?.Attributes["data-song-id"]?.Value; entry.SongTitle = songNode?.Attributes["data-title"]?.Value; entry.SongURL = songNode?.Attributes["data-url"]?.Value; if (!String.IsNullOrEmpty(entry.SongURL)) { entry.SongURL = String.Format(@"https://myspace.com{0}", entry.SongURL); } entry.AlbumID = songNode?.Attributes["data-album-id"]?.Value; entry.AlbumTitle = songNode?.Attributes["data-album-title"]?.Value; entry.AlbumURL = songNode?.Attributes["data-album-url"]?.Value; if (!String.IsNullOrEmpty(entry.AlbumURL)) { entry.AlbumURL = String.Format(@"https://myspace.com{0}", entry.AlbumURL); } entry.ArtistID = songNode?.Attributes["data-artist-id"]?.Value; entry.ArtistTitle = songNode?.Attributes["data-artist-name"]?.Value; entry.ArtistURL = songNode?.Attributes["data-artist-url"]?.Value; if (!String.IsNullOrEmpty(entry.ArtistURL)) { entry.ArtistURL = String.Format(@"https://myspace.com{0}", entry.ArtistURL); } entry.DurationInSeconds = songNode?.Attributes["data-duration"]?.Value; entry.VideoID = songNode?.Attributes["data-video-id"]?.Value; entry.YoutubeID = songNode?.Attributes["data-youtube-id"]?.Value; if (!String.IsNullOrEmpty(entry.YoutubeID)) { entry.YoutubeURL = String.Format(@"https://www.youtube.com/watch?v={0}", entry.YoutubeID); } entry.ImageThumbnailURL = songNode?.Attributes["data-image-url"]?.Value; entry.ImageURL = !String.IsNullOrEmpty(entry.ImageThumbnailURL) ? CrawlUtil.ModifyUriFileName(entry.ImageThumbnailURL, x => "full") : null; entry.GenreID = songNode?.Attributes["data-genre-id"]?.Value; entry.GenreName = songNode?.Attributes["data-genre-name"]?.Value; entry.MediaID = songNode?.Attributes["data-media-id"]?.Value; entry.MediaType = songNode?.Attributes["data-media-type"]?.Value; entry.UID = songNode?.Attributes["data-uid"]?.Value; String isPremiumFlag = songNode?.Attributes["data-is-premium"]?.Value; entry.IsPremium = isPremiumFlag != null && isPremiumFlag.ToLower().Equals("true"); String isExplicitFlag = songNode?.Attributes["data-is-explicit"]?.Value; entry.IsExplicit = isExplicitFlag != null && isExplicitFlag.ToLower().Equals("true"); String isFullLength = songNode?.Attributes["data-is-full-length"]?.Value; entry.IsFullLength = isFullLength != null && isFullLength.ToLower().Equals("true"); String isAdsProhibited = songNode?.Attributes["data-ads-prohibited"]?.Value; entry.IsAdsProhibited = isAdsProhibited != null && isAdsProhibited.ToLower().Equals("true"); #endregion #region Parse detial page if (!String.IsNullOrEmpty(entry.SongURL)) { Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests)); try { var detailDoc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; detailDoc.OptionWriteEmptyNodes = true; var webDetailRequest = HttpWebRequest.Create(entry.SongURL); ((HttpWebRequest)webDetailRequest).UserAgent = CrawlUtil.GetUserAgent(); Stream detailStream = webDetailRequest.GetResponse().GetResponseStream(); detailDoc.Load(detailStream); detailStream.Close(); var playsNodes = detailDoc.DocumentNode.SelectNodes(@"//div[@class='plays']"); if (playsNodes != null && playsNodes.Count >= 2 && String.Equals(playsNodes[0]?.InnerText, "PLAYS", StringComparison.OrdinalIgnoreCase)) { entry.PlayCount = playsNodes[1]?.InnerText; } var asideNodes = detailDoc.DocumentNode.SelectNodes(@"//aside[@class='dotted top']"); if (asideNodes != null && asideNodes.Count >= 1 && asideNodes[0]?.InnerText != null && asideNodes[0].InnerText.Contains("Length")) { var songDetailItemNodesDt = asideNodes[0].SelectNodes("//dt"); var songDetailItemNodesDd = asideNodes[0].SelectNodes("//dd"); if (songDetailItemNodesDt != null && songDetailItemNodesDt.Count > 0 && songDetailItemNodesDd != null && songDetailItemNodesDd.Count > 0 && songDetailItemNodesDt.Count == songDetailItemNodesDd.Count) { Dictionary <String, int> tableIndex = new Dictionary <string, int>(); int songDetailItemDtCount = -1; foreach (var item in songDetailItemNodesDt) { songDetailItemDtCount++; tableIndex.Add(item.InnerText, songDetailItemDtCount); } if (tableIndex.ContainsKey("Label") && songDetailItemNodesDd.Count >= tableIndex["Label"]) { int index = tableIndex["Label"]; entry.Label = songDetailItemNodesDd[index]?.InnerText; } if (tableIndex.ContainsKey("Release") && songDetailItemNodesDd.Count >= tableIndex["Release"]) { int index = tableIndex["Release"]; entry.ReleaseDate = songDetailItemNodesDd[index]?.InnerText; } } } } catch (Exception e2) { } } #endregion if (!String.IsNullOrEmpty(entry.SongID)) { Songs.Add(entry); } } } } catch (Exception e) { } }
/// <summary> /// Parse root profile page. /// </summary> /// <param name="userName">Profile page to parse.</param> private void ParseProfilePage(String userName) { String profileURL = String.Format(@"https://myspace.com/{0}", userName); var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.OptionWriteEmptyNodes = true; try { ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; var webRequest = HttpWebRequest.Create(profileURL); ((HttpWebRequest)webRequest).UserAgent = CrawlUtil.GetUserAgent(); Stream stream = webRequest.GetResponse().GetResponseStream(); doc.Load(stream); stream.Close(); Profile.URL = String.Format(@"https://myspace.com/{0}", userName); Profile.UserName = userName; Profile.ProfileThumbnailImageURL = doc.DocumentNode.SelectSingleNode(@"//a[@id='profileImage']//img")?.Attributes["src"]?.Value; Profile.ProfileImageURL = !String.IsNullOrEmpty(Profile.ProfileThumbnailImageURL) ? CrawlUtil.ModifyUriFileName(Profile.ProfileThumbnailImageURL, x => "600x600") : null; Profile.ProfileID = doc.DocumentNode.SelectSingleNode(@"//div[@class='connectButton notReversed tooltips']")?.Attributes["data-id"]?.Value; String privateFlag = doc.DocumentNode.SelectSingleNode(@"//div[@class='connectButton notReversed tooltips']")?.Attributes["data-is-private"]?.Value; Profile.IsPrivate = privateFlag != null && privateFlag.ToLower().Equals("true"); Profile.PersonalName = doc.DocumentNode.SelectSingleNode(@"//div[@class='connectButton notReversed tooltips']")?.Attributes["data-title"]?.Value; Profile.LocationDescription = doc.DocumentNode.SelectSingleNode(@"//div[@id='profileDetails']//div[@id='locAndWeb']//div[@class='location_white location ']")?.Attributes["data-display-text"]?.Value; Profile.Website = doc.DocumentNode.SelectSingleNode(@"//div[@id='profileDetails']//div[@id='locAndWeb']//div[@class='ribbon_white website ']//a")?.InnerText; Profile.OutConnectionTotal = doc.DocumentNode.SelectSingleNode(String.Format(@"//div[@id='profileDetails']//div[@id='connectionsCount']//a[@href='/{0}/connections/out']//span", Profile.UserName))?.InnerText; Profile.InConnectionTotal = doc.DocumentNode.SelectSingleNode(String.Format(@"//div[@id='profileDetails']//div[@id='connectionsCount']//a[@href='/{0}/connections/in']//span", Profile.UserName))?.InnerText; if (!Profile.IsPrivate) { var top8FriendsNode = doc.DocumentNode.SelectNodes(@"//div[@class='friendsWrapper']//ul//li//a"); if (top8FriendsNode != null) { foreach (var friendNode in top8FriendsNode) { Top8FriendEntry friendEntry = new Top8FriendEntry(); friendEntry.UserURL = friendNode?.Attributes["href"]?.Value; if (!String.IsNullOrEmpty(friendEntry.UserURL) && friendEntry.UserURL.StartsWith("/")) { friendEntry.UserURL = string.Format(@"https://myspace.com{0}", friendEntry.UserURL); } friendEntry.ProfileID = friendNode?.Attributes["data-profileid"]?.Value; friendEntry.ThumbnailURL = friendNode?.Attributes["data-image-url"]?.Value; friendEntry.UserName = friendNode?.Attributes["data-title"]?.Value; Profile.Top8Friends.Add(friendEntry); } } } } catch (Exception e) { } }