/// <summary>
/// Crawl an individual profile, then its connections.
/// </summary>
/// <param name="userName">User name of profile to parse</param>
/// <param name="delayBetweenPages">Delay between pages</param>
/// <param name="depth">Depth to parse from seed</param>
public void CrawlNode(String userName, int delayBetweenPages, int depth)
{
if (depth + 1 > MaxDepth)
{
return;
}
if (File.Exists(UserNameCrawlLock(userName)))
{
Log(String.Format(@"Skip Locked Profile: UserName={0}", userName));
return;
}
#region Check if all connections have been parsed
if (File.Exists(UserNameCompletePath(userName)))
{
try
{
CompleteFile checkFile = JsonConvert.DeserializeObject <CompleteFile>(File.ReadAllText(UserNameCompletePath(userName)));
if (checkFile != null && checkFile.AllConnectionsParsed)
{
Log(String.Format(@"Skip Completed Profile: UserName={0}", userName));
return;
}
}
catch (Exception chk)
{
Log(String.Format(@"Error Checking Lock File: UserName={0}, {1}", userName, chk?.Message));
}
}
#endregion
#region Create Directory if not exists
try
{
String path = Path.Combine(StoreDirectory, userName);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
}
catch (Exception e)
{
Log(String.Format("Error Creating Directory: UserName={0}", userName));
}
#endregion
#region Create Lock file
try
{
File.Create(UserNameCrawlLock(userName)).Dispose();
}
catch (Exception e)
{
Log(String.Format(@"Error Creating Lock File: UserName={0}, {1}", userName, e?.Message));
}
#endregion
Profile profile = null;
if (!UserNameHasBeenParsed(userName))
{
bool captureConnections = CanCaptureConnections(userName);
Log(String.Format(@"Parse Profile: UserName={0}, Depth={1}/{2}, CaptureConnections={3}",
userName, depth.ToString(), MaxDepth.ToString(), captureConnections.ToString()));
profile = new ProfileParser().Parse(
this,
HashKey,
userName,
DelayBetweenPages,
DelayBetweenAPIRequests,
captureConnections);
if (profile != null)
{
ProfileFinished(profile);
#region Download additional files if passes check
if (CapturePhotos)
{
#region Profile Photos
Log(String.Format(@"Download Profile Photos: UserName={0}", profile.UserName));
if (!String.IsNullOrEmpty(profile.ProfileImageURL) &&
!File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile.jpg")) &&
!File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile.jpg.error")))
{
bool success = CrawlUtil.GetFile(profile.ProfileImageURL, Path.Combine(StoreDirectory, profile.UserName, "profile.jpg"), 30000);
if (!success)
{
File.Create(Path.Combine(StoreDirectory, profile.UserName, "profile.jpg.error")).Dispose();
Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", profile.UserName));
}
}
if (!String.IsNullOrEmpty(profile.ProfileThumbnailImageURL) &&
!File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg")) &&
!File.Exists(Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg.error")))
{
bool success = CrawlUtil.GetFile(profile.ProfileThumbnailImageURL, Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg"), 30000);
if (!success)
{
File.Create(Path.Combine(StoreDirectory, profile.UserName, "profile_sm.jpg.error")).Dispose();
Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", profile.UserName));
}
}
#endregion
#region Photos Albums
if (profile.Photos != null && profile.Photos.Count > 0)
{
Log(String.Format(@"Download Photos: UserName={0}", profile.UserName));
#region Ensure Photos directory exists
String photoAlbumsPath = Path.Combine(UserNameDirectoryPath(profile.UserName), "Photos");
if (!Directory.Exists(photoAlbumsPath))
{
Directory.CreateDirectory(photoAlbumsPath);
}
#endregion
foreach (PhotoEntry entry in profile.Photos)
{
try
{
if (!String.IsNullOrEmpty(entry.PhotoID))
{
String picturePath = photoAlbumsPath;
if (!String.IsNullOrEmpty(entry.AlbumName))
{
#region Ensure Photo album directory exists
picturePath = Path.Combine(picturePath, entry.AlbumName);
if (!Directory.Exists(picturePath))
{
Directory.CreateDirectory(picturePath);
}
#endregion
#region Download Thumbnail
if (!String.IsNullOrEmpty(entry.ThumbnailImageURL) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID))) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID))))
{
Log(String.Format(@"Download Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName));
bool success = CrawlUtil.GetFile(entry.ThumbnailImageURL,
Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID)),
30000);
if (!success)
{
File.Create(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID))).Dispose();
Log(String.Format(@"Error Downloading Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName));
}
}
#endregion
#region Download Full Photo
if (!String.IsNullOrEmpty(entry.FullImageURL) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID))) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID))))
{
Log(String.Format(@"Download Photo: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName));
bool success = CrawlUtil.GetFile(entry.FullImageURL,
Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID)),
30000);
if (!success)
{
File.Create(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID))).Dispose();
Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}, Album={2}", profile.UserName, entry.PhotoID, entry.AlbumName));
}
}
#endregion
}
//Wait between each photo.
Thread.Sleep(CrawlUtil.GetVariableDelay(200));
}
}
catch (Exception e)
{
Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}", profile.UserName, entry?.PhotoID));
}
}
}
#endregion
#region Song Artwork
if (profile.Songs != null && profile.Songs.Count > 0)
{
Log(String.Format(@"Download Song Artwork: UserName={0}", profile.UserName));
#region Ensure Photos directory exists
String songArtworkPath = Path.Combine(UserNameDirectoryPath(profile.UserName), "Song_Artwork");
if (!Directory.Exists(songArtworkPath))
{
Directory.CreateDirectory(songArtworkPath);
}
#endregion
foreach (SongEntry entry in profile.Songs)
{
#region Download Thumbnail
String thumbnailFileName = entry.ImageThumbnailURL?.Replace(@"/", "___")?.Replace(":", "---");
if (!String.IsNullOrEmpty(thumbnailFileName) &&
!File.Exists(Path.Combine(songArtworkPath, thumbnailFileName)) &&
!File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName))))
{
Log(String.Format(@"Download Song Artwork Thumbnail: UserName={0}, Name={1}", profile.UserName, thumbnailFileName));
bool success = CrawlUtil.GetFile(entry.ImageThumbnailURL,
Path.Combine(songArtworkPath, thumbnailFileName),
30000);
if (!success)
{
File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName))).Dispose();
Log(String.Format(@"Error Downloading Song Artwork Thumbnail: UserName={0}, Name={1}", profile.UserName, thumbnailFileName));
}
}
#endregion
#region Download Full Image
String imageFileName = entry.ImageURL?.Replace(@"/", "___")?.Replace(":", "---");
if (!String.IsNullOrEmpty(imageFileName) &&
!File.Exists(Path.Combine(songArtworkPath, imageFileName)) &&
!File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName))))
{
Log(String.Format(@"Download Song Artwork: UserName={0}, Name={1}", profile.UserName, imageFileName));
bool success = CrawlUtil.GetFile(entry.ImageURL,
Path.Combine(songArtworkPath, imageFileName),
60000);
if (!success)
{
File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName))).Dispose();
Log(String.Format(@"Error Downloading Song Artwork: UserName={0}, Name={1}", profile.UserName, imageFileName));
}
}
#endregion
//Wait between each photo.
Thread.Sleep(CrawlUtil.GetVariableDelay(200));
}
}
#endregion
}
#endregion
}
}
else
{
Log(String.Format(@"Load Profile: UserName={0}", userName));
try
{
profile = JsonConvert.DeserializeObject <Profile>(File.ReadAllText(UserNameProfilePath(userName)));
}
catch (Exception e)
{
Log(String.Format(@"Error Loading Profile: UserName={0}, {1}", userName, e?.Message));
}
}
if (profile != null)
{
CompleteFile completeFile = null;
#region Ensure complete file has been created and populated
if (!File.Exists(UserNameCompletePath(userName)))
{
try
{
completeFile = new CompleteFile(userName);
if (profile.Connections != null)
{
foreach (ConnectionEntry c in profile.Connections)
{
completeFile.AllConnectionsParsed = false;
completeFile.ConnectionsParsed.Add(
new ConnectionParsedEntry()
{
UserName = c.UserName,
Parsed = false,
DateParsed = null
}
);
}
}
File.WriteAllText(
UserNameCompletePath(userName),
JsonConvert.SerializeObject(completeFile, Formatting.Indented));
}
catch (Exception e)
{
Log(String.Format(@"Error Creating Complete File: UserName={0}", e?.Message));
}
}
else
{
completeFile = JsonConvert.DeserializeObject <CompleteFile>(File.ReadAllText(UserNameCompletePath(userName)));
}
#endregion
foreach (ConnectionEntry c in profile.Connections)
{
ConnectionParsedEntry parsedEntry = completeFile.ConnectionsParsed.Where(x => String.Equals(x.UserName, c.UserName, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (parsedEntry != null)
{
if (!parsedEntry.Parsed)
{
CrawlNode(c.UserName, DelayBetweenPages, depth + 1);
#region Update complete file
try
{
parsedEntry.Parsed = true;
parsedEntry.DateParsed = DateTime.Now;
File.WriteAllText(
UserNameCompletePath(userName),
JsonConvert.SerializeObject(completeFile, Formatting.Indented));
}
catch (Exception p)
{
Log(String.Format(@"Error Updating Complete Entry: UserName={0}, ConnectionUserName={1}, {2]", userName, c.UserName, p?.Message));
}
#endregion
}
}
else
{
Log(String.Format(@"Error Missing Connection Complete Entry: UserName={0}, ConnectionUserName={1}", userName, c.UserName));
}
}
#region Update complete flag
try
{
Log(String.Format(@"All Connections Parsed: UserName={0}", userName));
completeFile.AllConnectionsParsed = true;
File.WriteAllText(
UserNameCompletePath(userName),
JsonConvert.SerializeObject(completeFile, Formatting.Indented));
}
catch (Exception p)
{
Log(String.Format(@"Error Updating Complete File: UserName={0}, {1}", userName, p?.Message));
}
#endregion
}
}
/// <summary>
/// Startdownload process.
/// </summary>
public void Download()
{
#region Ensure path exists
String path = UserNameDirectoryPath(UserName);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
#endregion
Logger = new Logger(UserNameDirectoryPath(UserName), String.Format("download_{0}", UserName));
Logger.Log(String.Format(@"Start Download Process: UserName={0}", UserName));
if (!File.Exists(UserNameProfilePath(UserName)))
{
#region Download Profile
Logger.Log(String.Format(@"Download Profile: UserName={0}", UserName));
ProfileParser profileParse = new ProfileParser();
Profile = profileParse.Parse(Logger, HashKey, UserName, 250, 200, CaptureConnections);
if (Profile == null)
{
Logger.Log(String.Format(@"Empty Profile: UserName={0}", UserName));
return;
}
Logger.Log(String.Format(@"Downloaded Profile: UserName={0}", UserName));
try
{
File.WriteAllText(
UserNameProfilePath(UserName),
JsonConvert.SerializeObject(Profile, Formatting.Indented));
}
catch (Exception e)
{
Logger.Log(String.Format(@"Error Saving Profile: UserName={0}, {1}", UserName, e?.Message));
return;
}
Logger.Log(String.Format(@"Save Profile JSON: UserName={0}", UserName));
#endregion
}
else
{
#region Load Profile JSON
Logger.Log(String.Format(@"Load Profile JSON: UserName={0}", UserName));
try
{
Profile = JsonConvert.DeserializeObject <Profile>(File.ReadAllText(UserNameProfilePath(UserName)));
if (Profile == null)
{
Logger.Log(String.Format(@"Empty Profile: UserName={0}", UserName));
return;
}
}
catch (Exception e)
{
Logger.Log(String.Format(@"Error loading profile: UserName={0}, {1}", UserName, e?.Message));
return;
}
Logger.Log(String.Format(@"Loaded Profile JSON: UserName={0}", UserName));
#endregion
}
//Download additional files if passes check
if (CapturePhotos)
{
if (DownloadPhotosCheck == null || (Profile != null && DownloadPhotosCheck.Invoke(Profile)))
{
#region Profile Photos
Logger.Log(String.Format(@"Download Profile Photos: UserName={0}", UserName));
if (!String.IsNullOrEmpty(Profile.ProfileImageURL) &&
!File.Exists(Path.Combine(StoreDirectory, UserName, "profile.jpg")) &&
!File.Exists(Path.Combine(StoreDirectory, UserName, "profile.jpg.error")))
{
bool success = CrawlUtil.GetFile(Profile.ProfileImageURL, Path.Combine(StoreDirectory, UserName, "profile.jpg"), 30000);
if (!success)
{
File.Create(Path.Combine(StoreDirectory, UserName, "profile.jpg.error")).Dispose();
Logger.Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", UserName));
}
}
if (!String.IsNullOrEmpty(Profile.ProfileThumbnailImageURL) &&
!File.Exists(Path.Combine(StoreDirectory, UserName, "profile_sm.jpg")) &&
!File.Exists(Path.Combine(StoreDirectory, UserName, "profile_sm.jpg.error")))
{
bool success = CrawlUtil.GetFile(Profile.ProfileThumbnailImageURL, Path.Combine(StoreDirectory, UserName, "profile_sm.jpg"), 30000);
if (!success)
{
File.Create(Path.Combine(StoreDirectory, UserName, "profile_sm.jpg.error")).Dispose();
Logger.Log(String.Format(@"Error Profile Photo Thumbnail: UserName={0}", UserName));
}
}
#endregion
#region Photos Albums
if (Profile.Photos != null && Profile.Photos.Count > 0)
{
Logger.Log(String.Format(@"Download Photos: UserName={0}", UserName));
#region Ensure Photos directory exists
String photoAlbumsPath = Path.Combine(UserNameDirectoryPath(UserName), "Photos");
if (!Directory.Exists(photoAlbumsPath))
{
Directory.CreateDirectory(photoAlbumsPath);
}
#endregion
foreach (PhotoEntry entry in Profile.Photos)
{
try
{
if (!String.IsNullOrEmpty(entry.PhotoID))
{
String picturePath = photoAlbumsPath;
if (!String.IsNullOrEmpty(entry.AlbumName))
{
#region Ensure Photo album directory exists
picturePath = Path.Combine(picturePath, entry.AlbumName);
if (!Directory.Exists(picturePath))
{
Directory.CreateDirectory(picturePath);
}
#endregion
#region Download Thumbnail
if (!String.IsNullOrEmpty(entry.ThumbnailImageURL) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID))) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID))))
{
Logger.Log(String.Format(@"Download Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName));
bool success = CrawlUtil.GetFile(entry.ThumbnailImageURL,
Path.Combine(picturePath, String.Format("{0}_sm.jpg", entry.PhotoID)),
30000);
if (!success)
{
File.Create(Path.Combine(picturePath, String.Format("{0}_sm.error", entry.PhotoID))).Dispose();
Logger.Log(String.Format(@"Error Downloading Photo Thumbnail: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName));
}
}
#endregion
#region Download Full Photo
if (!String.IsNullOrEmpty(entry.FullImageURL) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID))) &&
!File.Exists(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID))))
{
Logger.Log(String.Format(@"Download Photo: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName));
bool success = CrawlUtil.GetFile(entry.FullImageURL,
Path.Combine(picturePath, String.Format("{0}.jpg", entry.PhotoID)),
30000);
if (!success)
{
File.Create(Path.Combine(picturePath, String.Format("{0}.error", entry.PhotoID))).Dispose();
Logger.Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}, Album={2}", UserName, entry.PhotoID, entry.AlbumName));
}
}
#endregion
}
//Wait between each photo.
Thread.Sleep(CrawlUtil.GetVariableDelay(200));
}
}
catch (Exception e)
{
Logger.Log(String.Format(@"Error Downloading Photo: UserName={0}, PhotoID={1}", UserName, entry?.PhotoID));
}
}
}
#endregion
#region Song Artwork
if (Profile.Songs != null && Profile.Songs.Count > 0)
{
Logger.Log(String.Format(@"Download Song Artwork: UserName={0}", UserName));
#region Ensure Photos directory exists
String songArtworkPath = Path.Combine(UserNameDirectoryPath(UserName), "Song_Artwork");
if (!Directory.Exists(songArtworkPath))
{
Directory.CreateDirectory(songArtworkPath);
}
#endregion
foreach (SongEntry entry in Profile.Songs)
{
#region Download Thumbnail
String thumbnailFileName = entry.ImageThumbnailURL?.Replace(@"/", "___")?.Replace(":", "---");
if (!String.IsNullOrEmpty(thumbnailFileName) &&
!File.Exists(Path.Combine(songArtworkPath, thumbnailFileName)) &&
!File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName))))
{
Logger.Log(String.Format(@"Download Song Artwork Thumbnail: UserName={0}, Name={1}", UserName, thumbnailFileName));
bool success = CrawlUtil.GetFile(entry.ImageThumbnailURL,
Path.Combine(songArtworkPath, thumbnailFileName),
30000);
if (!success)
{
File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", thumbnailFileName))).Dispose();
Logger.Log(String.Format(@"Error Downloading Song Artwork Thumbnail: UserName={0}, Name={1}", UserName, thumbnailFileName));
}
}
#endregion
#region Download Full Image
String imageFileName = entry.ImageURL?.Replace(@"/", "___")?.Replace(":", "---");
if (!String.IsNullOrEmpty(imageFileName) &&
!File.Exists(Path.Combine(songArtworkPath, imageFileName)) &&
!File.Exists(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName))))
{
Logger.Log(String.Format(@"Download Song Artwork: UserName={0}, Name={1}", UserName, imageFileName));
bool success = CrawlUtil.GetFile(entry.ImageURL,
Path.Combine(songArtworkPath, imageFileName),
60000);
if (!success)
{
File.Create(Path.Combine(songArtworkPath, String.Format("{0}.error", imageFileName))).Dispose();
Logger.Log(String.Format(@"Error Downloading Song Artwork: UserName={0}, Name={1}", UserName, imageFileName));
}
}
#endregion
//Wait between each photo.
Thread.Sleep(CrawlUtil.GetVariableDelay(200));
}
}
#endregion
}
}
Logger.Log(String.Format(@"Done: UserName={0}", UserName));
}