/// <summary>
/// Extracts the given archive.
/// </summary>
/// <param name="archive">the archive to extract</param>
/// <param name="overwrite">true if existing files should be overwritten</param>
/// <param name="append">true if only new files should be extracted if the user is already registered</param>
private static void Extract(FileSystemInfo archive, bool overwrite, bool append)
{
string username = Path.GetFileNameWithoutExtension(archive.Name);
bool userExists = FileStore.FileExists(Resources.UserDirectory, username + ".json");
if (userExists && !append)
{
return;
}
using (ZipArchive zipArchive = ZipFile.OpenRead(archive.FullName))
{
foreach (ZipArchiveEntry entry in zipArchive.Entries)
{
string destination = FileStore.GetAbsolutePath(entry.FullName);
if (File.Exists(destination) && overwrite)
{
entry.ExtractToFile(destination, true);
}
else if (!File.Exists(destination))
{
FileStore.CreateDirectory(Path.GetDirectoryName(entry.FullName));
entry.ExtractToFile(destination, false);
}
}
}
}
/// <summary>
/// Scrapes users until the target has been reached.
/// </summary>
/// <param name="pageNumber">the number of the page to start scraping at</param>
public void ScrapeUsers(int pageNumber)
{
if (pageNumber < 0)
{
throw new ArgumentOutOfRangeException("Page number must be at least 0.");
}
int userCount = FileStore.GetFiles(Resources.UserDirectory).Length;
Logger.Log("Scraping list of recent projects.");
// Keep downloading projects until the target has been reached
while (userCount < _targetUserCount && pageNumber * PageSize < 10000)
{
Logger.Log(string.Format("Downloading page {0}. ({1} / {2} users registered)",
pageNumber, userCount, _targetUserCount));
// Loop over projects
JArray projects = GetRecentProjects(pageNumber, PageSize);
foreach (JToken project in projects)
{
string fileName = project["author"]["username"].ToString();
// Skip project if user is already known
if (FileStore.FileExists(Resources.UserDirectory, fileName + ".json"))
{
continue;
}
// Add user
FileStore.WriteFile(Resources.UserDirectory, fileName + ".json", "");
userCount++;
if (userCount >= _targetUserCount)
{
break;
}
}
pageNumber++;
}
if (userCount < _targetUserCount)
{
Logger.Log("Kragle was unable to scrape more users because the Scratch API records only the 10,000"
+ " latest activities.");
}
Logger.Log(string.Format("Successfully registered {0} users.\n", userCount));
}
/// <summary>
/// Gets the size of the image.
/// </summary>
/// <param name="path">The path.</param>
/// <returns></returns>
private Size GetImageSize(string path)
{
var size = Size.Empty;
if (!string.IsNullOrEmpty(path) && _fileStore.FileExists(path))
{
using (var stream = _fileStore.OpenFile(path))
using (var img = Image.FromStream(stream))
{
size = img.Size;
}
}
return(size);
}
/// <summary>
/// Creates an archive for the specified user containing all that user's data.
/// </summary>
/// <param name="username">a username</param>
private static void Archive(string username)
{
FileStore.CreateDirectory(Resources.ArchiveDirectory);
if (FileStore.FileExists(Resources.ArchiveDirectory, username + ".zip"))
{
return;
}
string archivePath = FileStore.GetAbsolutePath(Resources.ArchiveDirectory, username + ".zip");
using (var fileStream = new FileStream(archivePath, FileMode.CreateNew))
using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
{
ArchiveUserData(archive, username);
ArchiveCurrentProjectList(archive, username);
ArchiveOldProjectLists(archive, username);
ArchiveProjectCode(archive, username);
}
}
/// <summary>
/// Downloads project code for all registered projects.
/// </summary>
public void DownloadProjects()
{
DateTime currentDate = DateTime.Now.Date;
FileInfo[] users = FileStore.GetFiles(Resources.ProjectDirectory);
int userTotal = users.Length;
int userCurrent = 0;
Logger.Log(string.Format("Downloading code for {0} users.", userTotal));
// Iterate over users
foreach (FileInfo user in users)
{
string username = user.Name.Remove(user.Name.Length - 5);
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Downloading code for user " + LoggerHelper.ForceLength(username, 10), userCurrent, userTotal));
JArray projects;
try
{
projects = JArray.Parse(FileStore.ReadFile(Resources.ProjectDirectory, username + ".json"));
}
catch (JsonReaderException e)
{
Logger.Log("Could not parse list of projects of user `" + username + "`", e);
return;
}
// Iterate over user projects
foreach (JToken project in projects)
{
DateTime modifyDate = DateTime.Parse(project["history"]["modified"].ToString()).Date;
int projectId = Convert.ToInt32(project["id"].ToString());
string codeDir = Resources.CodeDirectory + "/" + projectId;
string yesterdayFileName = currentDate.AddDays(-1).ToString("yyyy-MM-dd") + ".json";
string todayFileName = currentDate.ToString("yyyy-MM-dd") + ".json";
if (FileStore.FileExists(codeDir, todayFileName))
{
// Code already downloaded today
continue;
}
if (currentDate.Subtract(modifyDate).Days > 0 && FileStore.FileExists(codeDir, yesterdayFileName))
{
// No code modifications in last day, copy old file
FileStore.CopyFile(codeDir, yesterdayFileName, codeDir, todayFileName);
continue;
}
string projectCode = GetProjectCode(projectId);
if (projectCode == null)
{
// Code not downloaded for whatever reason
continue;
}
if (!Downloader.IsValidJson(projectCode))
{
// Invalid JSON, no need to save it
continue;
}
FileStore.WriteFile(codeDir, todayFileName, projectCode);
}
}
Logger.Log(string.Format("Successfully downloaded code for {0} users.\n", userCurrent));
}
