public void Read() { PhotoStreamResponse r = new PhotoStreamResponse() { EndOfPhotos = false, LastPhotoID = null }; while (!r.EndOfPhotos) { r = RequestPhotoStream(r.LastPhotoID); if (r == null || !(String.IsNullOrEmpty(r.Error))) { //return on error return; } else { Photos.AddRange(r.PhotosEntries); } if (!r.EndOfPhotos) { Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests)); } } }
public void Read() { ConnectionStreamResponse r = new ConnectionStreamResponse() { NextStart = 0 }; while (!r.EndOfConnections) { r = RequestConnectionStream(r.NextStart); if (r == null || !(String.IsNullOrEmpty(r.Error))) { //return on error return; } else { Connections.AddRange(r.ConnectionEntries); } if (!r.EndOfConnections) { Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests)); } } }
/// <summary> /// Parse the given profile page. /// </summary> /// <param name="logger">Logging manager.</param> /// <param name="hashKey">Hashkey to use for API calls.</param> /// <param name="userName">Profile name to parse.</param> /// <param name="delayBetweenPages">Delay between page requests. This is a base value which will be slightly randomized.</param> /// <param name="delayBetweenAPIRequests">Delay between API calls. This is a base value which will be slightly randomized.</param> /// <param name="captureConnections">Should connections be parsed?</param> /// <returns>Parsed profile.</returns> public Profile Parse( ILog logger, String hashKey, String userName, int delayBetweenPages, int delayBetweenAPIRequests, bool captureConnections) { HashKey = hashKey; DelayBetweenPages = delayBetweenPages; DelayBetweenAPIRequests = delayBetweenAPIRequests; CaptureConnections = captureConnections; Profile = new Profile(); Profile.Captured = DateTime.Now; Profile.CapturedConnections = captureConnections; ParseProfilePage(userName); Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenPages)); if (!Profile.IsPrivate) { ParseBio(); Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenPages)); logger.Log(String.Format("Parsing Photos: UserName={0}", userName)); ParsePhotos(); Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenPages)); if (CaptureConnections) { logger.Log(String.Format("Parsing Connections Out: UserName={0}", userName)); ParseConnections(ConnectionDirection.Out); Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenPages)); logger.Log(String.Format("Parsing Connections In: UserName={0}", userName)); ParseConnections(ConnectionDirection.In); Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenPages)); } logger.Log(String.Format("Parsing Songs: UserName={0}", userName)); ParseSongs(); Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenPages)); logger.Log(String.Format("Parsing Videos: Videos={0}", userName)); ParseVideos(); Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenPages)); } else { logger.Log(String.Format("Private profile: UserName={0}", userName)); } return(Profile); }
/// <summary> /// Crawl an individual profile, then its connections. /// </summary> /// <param name="userName">User name of profile to parse</param> /// <param name="delayBetweenPages">Delay between pages</param> /// <param name="depth">Depth to parse from seed</param> public void CrawlNode(String userName, int delayBetweenPages, int depth) { if (depth + 1 > MaxDepth) { return; } if (File.Exists(UserNameCrawlLock(userName))) { Log(String.Format(@"Skip Locked Profile: UserName={0}", userName)); return; } #region Check if all connections have been parsed if (File.Exists(UserNameCompletePath(userName))) { try { CompleteFile checkFile = JsonConvert.DeserializeObject <CompleteFile>(File.ReadAllText(UserNameCompletePath(userName))); if (checkFile != null && checkFile.AllConnectionsParsed) { Log(String.Format(@"Skip Completed Profile: UserName={0}", userName)); return; } } catch (Exception chk) { Log(String.Format(@"Error Checking Lock File: UserName={0}, {1}", userName, chk?.Message)); } } #endregion #region Create Directory if not exists try { String path = Path.Combine(StoreDirectory, userName); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } } catch (Exception e) { Log(String.Format("Error Creating Directory: UserName={0}", userName)); } #endregion #region Create Lock file try { File.Create(UserNameCrawlLock(userName)).Dispose(); } catch (Exception e) { Log(String.Format(@"Error Creating Lock File: UserName={0}, {1}", userName, e?.Message)); } #endregion Profile profile = null; if (!UserNameHasBeenParsed(userName)) { bool captureConnections = CanCaptureConnections(userName); Log(String.Format(@"Parse Profile: UserName={0}, Depth={1}/{2}, CaptureConnections={3}", userName, depth.ToString(), MaxDepth.ToString(), captureConnections.ToString())); profile = new ProfileParser().Parse( this, HashKey, userName, DelayBetweenPages, DelayBetweenAPIRequests, captureConnections); if (profile != null) { ProfileFinished(profile); #region Download additional files if passes check if (CapturePhotos) { #region Profile Photos Log(String.Format(@"Download Profile Photos: UserName={0}", profile.UserName)); if (!String.IsNullOrEmpty(profile.ProfileImageURL) && !File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile.jpg")) && !File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile.jpg.error"))) { bool success = CrawlUtil.GetFile(profile.ProfileImageURL, Path.Combine(StoreDirectory, profile.UserName, "profile.jpg"), 30000); if (!success) { File.Create(Path.Combine(StoreDirectory, profile.UserName, "profile.jpg.error")).Dispose(); Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", profile.UserName)); } } if (!String.IsNullOrEmpty(profile.ProfileThumbnailImageURL) && !File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg")) && !File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg.error"))) { bool success = CrawlUtil.GetFile(profile.ProfileThumbnailImageURL, Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg"), 30000); if (!success) { File.Create(Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg.error")).Dispose(); Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", profile.UserName)); } } #endregion #region Photos Albums if (profile.Photos != null && profile.Photos.Count > 0) { Log(String.Format(@"Download Photos: UserName={0}", profile.UserName)); #region Ensure Photos directory exists String photoAlbumsPath = Path.Combine(UserNameDirectoryPath(profile.UserName), "Photos"); if (!Directory.Exists(photoAlbumsPath)) { Directory.CreateDirectory(photoAlbumsPath); } #endregion foreach (PhotoEntry entry in profile.Photos) { try { if (!String.IsNullOrEmpty(entry.PhotoID)) { String picturePath = photoAlbumsPath; if (!String.IsNullOrEmpty(entry.AlbumName)) { #region Ensure Photo album directory exists picturePath = Path.Combine(picturePath, entry.AlbumName); if (!Directory.Exists(picturePath)) { Directory.CreateDirectory(picturePath); } #endregion #region Download Thumbnail if (!String.IsNullOrEmpty(entry.ThumbnailImageURL) && !File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID))) && !File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID)))) { Log(String.Format(@"Download Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName)); bool success = CrawlUtil.GetFile(entry.ThumbnailImageURL, Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID)), 30000); if (!success) { File.Create(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID))).Dispose(); Log(String.Format(@"Error Downloading Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName)); } } #endregion #region Download Full Photo if (!String.IsNullOrEmpty(entry.FullImageURL) && !File.Exists(Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID))) && !File.Exists(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID)))) { Log(String.Format(@"Download Photo: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName)); bool success = CrawlUtil.GetFile(entry.FullImageURL, Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID)), 30000); if (!success) { File.Create(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID))).Dispose(); Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName)); } } #endregion } //Wait between each photo. Thread.Sleep(CrawlUtil.GetVariableDelay(200)); } } catch (Exception e) { Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}", profile.UserName, entry?.PhotoID)); } } } #endregion #region Song Artwork if (profile.Songs != null && profile.Songs.Count > 0) { Log(String.Format(@"Download Song Artwork: UserName={0}", profile.UserName)); #region Ensure Photos directory exists String songArtworkPath = Path.Combine(UserNameDirectoryPath(profile.UserName), "Song_Artwork"); if (!Directory.Exists(songArtworkPath)) { Directory.CreateDirectory(songArtworkPath); } #endregion foreach (SongEntry entry in profile.Songs) { #region Download Thumbnail String thumbnailFileName = entry.ImageThumbnailURL?.Replace(@"/", "___")?.Replace(":", "---"); if (!String.IsNullOrEmpty(thumbnailFileName) && !File.Exists(Path.Combine(songArtworkPath, thumbnailFileName)) && !File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName)))) { Log(String.Format(@"Download Song Artwork Thumbnail: UserName={0}, Name={1}", profile.UserName, thumbnailFileName)); bool success = CrawlUtil.GetFile(entry.ImageThumbnailURL, Path.Combine(songArtworkPath, thumbnailFileName), 30000); if (!success) { File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName))).Dispose(); Log(String.Format(@"Error Downloading Song Artwork Thumbnail: UserName={0}, Name={1}", profile.UserName, thumbnailFileName)); } } #endregion #region Download Full Image String imageFileName = entry.ImageURL?.Replace(@"/", "___")?.Replace(":", "---"); if (!String.IsNullOrEmpty(imageFileName) && !File.Exists(Path.Combine(songArtworkPath, imageFileName)) && !File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName)))) { Log(String.Format(@"Download Song Artwork: UserName={0}, Name={1}", profile.UserName, imageFileName)); bool success = CrawlUtil.GetFile(entry.ImageURL, Path.Combine(songArtworkPath, imageFileName), 60000); if (!success) { File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName))).Dispose(); Log(String.Format(@"Error Downloading Song Artwork: UserName={0}, Name={1}", profile.UserName, imageFileName)); } } #endregion //Wait between each photo. Thread.Sleep(CrawlUtil.GetVariableDelay(200)); } } #endregion } #endregion } } else { Log(String.Format(@"Load Profile: UserName={0}", userName)); try { profile = JsonConvert.DeserializeObject <Profile>(File.ReadAllText(UserNameProfilePath(userName))); } catch (Exception e) { Log(String.Format(@"Error Loading Profile: UserName={0}, {1}", userName, e?.Message)); } } if (profile != null) { CompleteFile completeFile = null; #region Ensure complete file has been created and populated if (!File.Exists(UserNameCompletePath(userName))) { try { completeFile = new CompleteFile(userName); if (profile.Connections != null) { foreach (ConnectionEntry c in profile.Connections) { completeFile.AllConnectionsParsed = false; completeFile.ConnectionsParsed.Add( new ConnectionParsedEntry() { UserName = c.UserName, Parsed = false, DateParsed = null } ); } } File.WriteAllText( UserNameCompletePath(userName), JsonConvert.SerializeObject(completeFile, Formatting.Indented)); } catch (Exception e) { Log(String.Format(@"Error Creating Complete File: UserName={0}", e?.Message)); } } else { completeFile = JsonConvert.DeserializeObject <CompleteFile>(File.ReadAllText(UserNameCompletePath(userName))); } #endregion foreach (ConnectionEntry c in profile.Connections) { ConnectionParsedEntry parsedEntry = completeFile.ConnectionsParsed.Where(x => String.Equals(x.UserName, c.UserName, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); if (parsedEntry != null) { if (!parsedEntry.Parsed) { CrawlNode(c.UserName, DelayBetweenPages, depth + 1); #region Update complete file try { parsedEntry.Parsed = true; parsedEntry.DateParsed = DateTime.Now; File.WriteAllText( UserNameCompletePath(userName), JsonConvert.SerializeObject(completeFile, Formatting.Indented)); } catch (Exception p) { Log(String.Format(@"Error Updating Complete Entry: UserName={0}, ConnectionUserName={1}, {2]", userName, c.UserName, p?.Message)); } #endregion } } else { Log(String.Format(@"Error Missing Connection Complete Entry: UserName={0}, ConnectionUserName={1}", userName, c.UserName)); } } #region Update complete flag try { Log(String.Format(@"All Connections Parsed: UserName={0}", userName)); completeFile.AllConnectionsParsed = true; File.WriteAllText( UserNameCompletePath(userName), JsonConvert.SerializeObject(completeFile, Formatting.Indented)); } catch (Exception p) { Log(String.Format(@"Error Updating Complete File: UserName={0}, {1}", userName, p?.Message)); } #endregion } }
public void Read() { var doc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; doc.OptionWriteEmptyNodes = true; try { var webRequest = HttpWebRequest.Create(String.Format(@"https://myspace.com/{0}/music/songs", UserName)); ((HttpWebRequest)webRequest).UserAgent = CrawlUtil.GetUserAgent(); Stream stream = webRequest.GetResponse().GetResponseStream(); doc.Load(stream); stream.Close(); var songsNode = doc.DocumentNode.SelectNodes(@"//button[@class='playBtn play_25 song']"); if (songsNode != null) { foreach (var songNode in songsNode) { SongEntry entry = new SongEntry(); #region Parse summary page entry.SongID = songNode?.Attributes["data-song-id"]?.Value; entry.SongTitle = songNode?.Attributes["data-title"]?.Value; entry.SongURL = songNode?.Attributes["data-url"]?.Value; if (!String.IsNullOrEmpty(entry.SongURL)) { entry.SongURL = String.Format(@"https://myspace.com{0}", entry.SongURL); } entry.AlbumID = songNode?.Attributes["data-album-id"]?.Value; entry.AlbumTitle = songNode?.Attributes["data-album-title"]?.Value; entry.AlbumURL = songNode?.Attributes["data-album-url"]?.Value; if (!String.IsNullOrEmpty(entry.AlbumURL)) { entry.AlbumURL = String.Format(@"https://myspace.com{0}", entry.AlbumURL); } entry.ArtistID = songNode?.Attributes["data-artist-id"]?.Value; entry.ArtistTitle = songNode?.Attributes["data-artist-name"]?.Value; entry.ArtistURL = songNode?.Attributes["data-artist-url"]?.Value; if (!String.IsNullOrEmpty(entry.ArtistURL)) { entry.ArtistURL = String.Format(@"https://myspace.com{0}", entry.ArtistURL); } entry.DurationInSeconds = songNode?.Attributes["data-duration"]?.Value; entry.VideoID = songNode?.Attributes["data-video-id"]?.Value; entry.YoutubeID = songNode?.Attributes["data-youtube-id"]?.Value; if (!String.IsNullOrEmpty(entry.YoutubeID)) { entry.YoutubeURL = String.Format(@"https://www.youtube.com/watch?v={0}", entry.YoutubeID); } entry.ImageThumbnailURL = songNode?.Attributes["data-image-url"]?.Value; entry.ImageURL = !String.IsNullOrEmpty(entry.ImageThumbnailURL) ? CrawlUtil.ModifyUriFileName(entry.ImageThumbnailURL, x => "full") : null; entry.GenreID = songNode?.Attributes["data-genre-id"]?.Value; entry.GenreName = songNode?.Attributes["data-genre-name"]?.Value; entry.MediaID = songNode?.Attributes["data-media-id"]?.Value; entry.MediaType = songNode?.Attributes["data-media-type"]?.Value; entry.UID = songNode?.Attributes["data-uid"]?.Value; String isPremiumFlag = songNode?.Attributes["data-is-premium"]?.Value; entry.IsPremium = isPremiumFlag != null && isPremiumFlag.ToLower().Equals("true"); String isExplicitFlag = songNode?.Attributes["data-is-explicit"]?.Value; entry.IsExplicit = isExplicitFlag != null && isExplicitFlag.ToLower().Equals("true"); String isFullLength = songNode?.Attributes["data-is-full-length"]?.Value; entry.IsFullLength = isFullLength != null && isFullLength.ToLower().Equals("true"); String isAdsProhibited = songNode?.Attributes["data-ads-prohibited"]?.Value; entry.IsAdsProhibited = isAdsProhibited != null && isAdsProhibited.ToLower().Equals("true"); #endregion #region Parse detial page if (!String.IsNullOrEmpty(entry.SongURL)) { Thread.Sleep(CrawlUtil.GetVariableDelay(DelayBetweenAPIRequests)); try { var detailDoc = new HtmlAgilityPack.HtmlDocument(); HtmlAgilityPack.HtmlNode.ElementsFlags["br"] = HtmlAgilityPack.HtmlElementFlag.Empty; detailDoc.OptionWriteEmptyNodes = true; var webDetailRequest = HttpWebRequest.Create(entry.SongURL); ((HttpWebRequest)webDetailRequest).UserAgent = CrawlUtil.GetUserAgent(); Stream detailStream = webDetailRequest.GetResponse().GetResponseStream(); detailDoc.Load(detailStream); detailStream.Close(); var playsNodes = detailDoc.DocumentNode.SelectNodes(@"//div[@class='plays']"); if (playsNodes != null && playsNodes.Count >= 2 && String.Equals(playsNodes[0]?.InnerText, "PLAYS", StringComparison.OrdinalIgnoreCase)) { entry.PlayCount = playsNodes[1]?.InnerText; } var asideNodes = detailDoc.DocumentNode.SelectNodes(@"//aside[@class='dotted top']"); if (asideNodes != null && asideNodes.Count >= 1 && asideNodes[0]?.InnerText != null && asideNodes[0].InnerText.Contains("Length")) { var songDetailItemNodesDt = asideNodes[0].SelectNodes("//dt"); var songDetailItemNodesDd = asideNodes[0].SelectNodes("//dd"); if (songDetailItemNodesDt != null && songDetailItemNodesDt.Count > 0 && songDetailItemNodesDd != null && songDetailItemNodesDd.Count > 0 && songDetailItemNodesDt.Count == songDetailItemNodesDd.Count) { Dictionary <String, int> tableIndex = new Dictionary <string, int>(); int songDetailItemDtCount = -1; foreach (var item in songDetailItemNodesDt) { songDetailItemDtCount++; tableIndex.Add(item.InnerText, songDetailItemDtCount); } if (tableIndex.ContainsKey("Label") && songDetailItemNodesDd.Count >= tableIndex["Label"]) { int index = tableIndex["Label"]; entry.Label = songDetailItemNodesDd[index]?.InnerText; } if (tableIndex.ContainsKey("Release") && songDetailItemNodesDd.Count >= tableIndex["Release"]) { int index = tableIndex["Release"]; entry.ReleaseDate = songDetailItemNodesDd[index]?.InnerText; } } } } catch (Exception e2) { } } #endregion if (!String.IsNullOrEmpty(entry.SongID)) { Songs.Add(entry); } } } } catch (Exception e) { } }
/// <summary> /// Startdownload process. /// </summary> public void Download() { #region Ensure path exists String path = UserNameDirectoryPath(UserName); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } #endregion Logger = new Logger(UserNameDirectoryPath(UserName), String.Format("download_{0}", UserName)); Logger.Log(String.Format(@"Start Download Process: UserName={0}", UserName)); if (!File.Exists(UserNameProfilePath(UserName))) { #region Download Profile Logger.Log(String.Format(@"Download Profile: UserName={0}", UserName)); ProfileParser profileParse = new ProfileParser(); Profile = profileParse.Parse(Logger, HashKey, UserName, 250, 200, CaptureConnections); if (Profile == null) { Logger.Log(String.Format(@"Empty Profile: UserName={0}", UserName)); return; } Logger.Log(String.Format(@"Downloaded Profile: UserName={0}", UserName)); try { File.WriteAllText( UserNameProfilePath(UserName), JsonConvert.SerializeObject(Profile, Formatting.Indented)); } catch (Exception e) { Logger.Log(String.Format(@"Error Saving Profile: UserName={0}, {1}", UserName, e?.Message)); return; } Logger.Log(String.Format(@"Save Profile JSON: UserName={0}", UserName)); #endregion } else { #region Load Profile JSON Logger.Log(String.Format(@"Load Profile JSON: UserName={0}", UserName)); try { Profile = JsonConvert.DeserializeObject <Profile>(File.ReadAllText(UserNameProfilePath(UserName))); if (Profile == null) { Logger.Log(String.Format(@"Empty Profile: UserName={0}", UserName)); return; } } catch (Exception e) { Logger.Log(String.Format(@"Error loading profile: UserName={0}, {1}", UserName, e?.Message)); return; } Logger.Log(String.Format(@"Loaded Profile JSON: UserName={0}", UserName)); #endregion } //Download additional files if passes check if (CapturePhotos) { if (DownloadPhotosCheck == null || (Profile != null && DownloadPhotosCheck.Invoke(Profile))) { #region Profile Photos Logger.Log(String.Format(@"Download Profile Photos: UserName={0}", UserName)); if (!String.IsNullOrEmpty(Profile.ProfileImageURL) && !File.Exists(Path.Combine(StoreDirectory, UserName, "profile.jpg")) && !File.Exists(Path.Combine(StoreDirectory, UserName, "profile.jpg.error"))) { bool success = CrawlUtil.GetFile(Profile.ProfileImageURL, Path.Combine(StoreDirectory, UserName, "profile.jpg"), 30000); if (!success) { File.Create(Path.Combine(StoreDirectory, UserName, "profile.jpg.error")).Dispose(); Logger.Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", UserName)); } } if (!String.IsNullOrEmpty(Profile.ProfileThumbnailImageURL) && !File.Exists(Path.Combine(StoreDirectory, UserName, "profile_sm.jpg")) && !File.Exists(Path.Combine(StoreDirectory, UserName, "profile_sm.jpg.error"))) { bool success = CrawlUtil.GetFile(Profile.ProfileThumbnailImageURL, Path.Combine(StoreDirectory, UserName, "profile_sm.jpg"), 30000); if (!success) { File.Create(Path.Combine(StoreDirectory, UserName, "profile_sm.jpg.error")).Dispose(); Logger.Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", UserName)); } } #endregion #region Photos Albums if (Profile.Photos != null && Profile.Photos.Count > 0) { Logger.Log(String.Format(@"Download Photos: UserName={0}", UserName)); #region Ensure Photos directory exists String photoAlbumsPath = Path.Combine(UserNameDirectoryPath(UserName), "Photos"); if (!Directory.Exists(photoAlbumsPath)) { Directory.CreateDirectory(photoAlbumsPath); } #endregion foreach (PhotoEntry entry in Profile.Photos) { try { if (!String.IsNullOrEmpty(entry.PhotoID)) { String picturePath = photoAlbumsPath; if (!String.IsNullOrEmpty(entry.AlbumName)) { #region Ensure Photo album directory exists picturePath = Path.Combine(picturePath, entry.AlbumName); if (!Directory.Exists(picturePath)) { Directory.CreateDirectory(picturePath); } #endregion #region Download Thumbnail if (!String.IsNullOrEmpty(entry.ThumbnailImageURL) && !File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID))) && !File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID)))) { Logger.Log(String.Format(@"Download Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName)); bool success = CrawlUtil.GetFile(entry.ThumbnailImageURL, Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID)), 30000); if (!success) { File.Create(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID))).Dispose(); Logger.Log(String.Format(@"Error Downloading Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName)); } } #endregion #region Download Full Photo if (!String.IsNullOrEmpty(entry.FullImageURL) && !File.Exists(Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID))) && !File.Exists(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID)))) { Logger.Log(String.Format(@"Download Photo: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName)); bool success = CrawlUtil.GetFile(entry.FullImageURL, Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID)), 30000); if (!success) { File.Create(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID))).Dispose(); Logger.Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName)); } } #endregion } //Wait between each photo. Thread.Sleep(CrawlUtil.GetVariableDelay(200)); } } catch (Exception e) { Logger.Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}", UserName, entry?.PhotoID)); } } } #endregion #region Song Artwork if (Profile.Songs != null && Profile.Songs.Count > 0) { Logger.Log(String.Format(@"Download Song Artwork: UserName={0}", UserName)); #region Ensure Photos directory exists String songArtworkPath = Path.Combine(UserNameDirectoryPath(UserName), "Song_Artwork"); if (!Directory.Exists(songArtworkPath)) { Directory.CreateDirectory(songArtworkPath); } #endregion foreach (SongEntry entry in Profile.Songs) { #region Download Thumbnail String thumbnailFileName = entry.ImageThumbnailURL?.Replace(@"/", "___")?.Replace(":", "---"); if (!String.IsNullOrEmpty(thumbnailFileName) && !File.Exists(Path.Combine(songArtworkPath, thumbnailFileName)) && !File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName)))) { Logger.Log(String.Format(@"Download Song Artwork Thumbnail: UserName={0}, Name={1}", UserName, thumbnailFileName)); bool success = CrawlUtil.GetFile(entry.ImageThumbnailURL, Path.Combine(songArtworkPath, thumbnailFileName), 30000); if (!success) { File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName))).Dispose(); Logger.Log(String.Format(@"Error Downloading Song Artwork Thumbnail: UserName={0}, Name={1}", UserName, thumbnailFileName)); } } #endregion #region Download Full Image String imageFileName = entry.ImageURL?.Replace(@"/", "___")?.Replace(":", "---"); if (!String.IsNullOrEmpty(imageFileName) && !File.Exists(Path.Combine(songArtworkPath, imageFileName)) && !File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName)))) { Logger.Log(String.Format(@"Download Song Artwork: UserName={0}, Name={1}", UserName, imageFileName)); bool success = CrawlUtil.GetFile(entry.ImageURL, Path.Combine(songArtworkPath, imageFileName), 60000); if (!success) { File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName))).Dispose(); Logger.Log(String.Format(@"Error Downloading Song Artwork: UserName={0}, Name={1}", UserName, imageFileName)); } } #endregion //Wait between each photo. Thread.Sleep(CrawlUtil.GetVariableDelay(200)); } } #endregion } } Logger.Log(String.Format(@"Done: UserName={0}", UserName)); }