/// <summary>
/// Validates all project lists of each user.
/// </summary>
public void ValidateProjectLists()
{
DirectoryInfo[] users = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = users.Length;
int userCurrent = 0;
Logger.Log(string.Format("Validating project lists of {0} users.", userTotal));
foreach (DirectoryInfo user in users)
{
string username = user.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Validating project lists of user " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
FileInfo[] projectLists = user.GetFiles();
foreach (FileInfo projectList in projectLists)
{
if (!IsValidJson(File.ReadAllText(projectList.FullName)))
{
InvalidProjectLists++;
Logger.Log("Project list of user " + username + " at date " + projectList.Name + " is invalid");
}
}
}
}
/// <summary>
/// Validates all code files of each project.
/// </summary>
public void ValidateCode()
{
DirectoryInfo[] projects = FileStore.GetDirectories(Resources.CodeDirectory);
int projectTotal = projects.Length;
int projectCurrent = 0;
Logger.Log(string.Format("Validating code of {0} projects.", projectTotal));
foreach (DirectoryInfo project in projects)
{
string projectId = project.Name;
projectCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Validating code of project with id " + projectId, projectCurrent, projectTotal));
FileInfo[] codeFiles = project.GetFiles();
foreach (FileInfo codeFile in codeFiles)
{
if (!IsValidJson(File.ReadAllText(codeFile.FullName)))
{
InvalidCodeFiles++;
Logger.Log("Code of project " + projectId + " at date " + codeFile.Name + " is invalid");
}
}
}
}
private static void RemoveUnchangedProjectCode()
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.CodeDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Removing unchanged projects of " + userDirs.Length + " users.");
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Removing unchanged projects of user " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
FileInfo[] projectLists = userDir.GetFiles().OrderBy(projectList => projectList.Name).ToArray();
if (projectLists.Length <= 1)
{
userDir.Delete(true);
}
}
}
/// <summary>
/// Generates files for all projects of all registered users, but does not download project code yet.
/// </summary>
public void UpdateProjectList()
{
DateTime currentDate = DateTime.Now.Date;
FileInfo[] users = FileStore.GetFiles(Resources.UserDirectory);
int userTotal = users.Length;
int userCurrent = 0;
Logger.Log(string.Format("Downloading project lists for {0} users.", userTotal));
// Iterate over users
foreach (FileInfo user in users)
{
string username = user.Name.Remove(user.Name.Length - 5);
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Downloading project list for user " + LoggerHelper.ForceLength(username, 10), userCurrent,
userTotal));
// Get list of user projects
string projects = GetUserProjects(username);
if (projects == null)
{
continue;
}
// Save list of projects
FileStore.WriteFile(Resources.ProjectDirectory, username + ".json", projects);
FileStore.WriteFile(Resources.ProjectDirectory + "/" + username,
currentDate.ToString("yyyy-MM-dd") + ".json", projects);
}
Logger.Log(string.Format("Successfully downloaded project lists for {0} users.\n", userCurrent));
}
/// <summary>
/// Downloads metadata for all users. If the <code>cache</code> attribute is not set, existing metadata is
/// updated.
/// </summary>
public void DownloadMetadata()
{
FileInfo[] users = FileStore.GetFiles(Resources.UserDirectory);
int userTotal = users.Length;
int userCurrent = 0;
Logger.Log(string.Format("Downloading metadata for {0} users.", userTotal));
foreach (FileInfo user in users)
{
string username = user.Name.Remove(user.Name.Length - 5);
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Downloading metadata for user " + LoggerHelper.ForceLength(username, 10), userCurrent,
userTotal));
if (user.Length > 0)
{
// Metadata already downloaded
continue;
}
string metadata = GetMetadata(username);
FileStore.WriteFile(Resources.UserDirectory, username + ".json", metadata);
}
Logger.Log(string.Format("Successfully downloaded metadata for {0} users.\n", userCurrent));
}
private static void RemoveUsersWithoutProjects()
{
FileInfo[] userDirs = FileStore.GetFiles(Resources.UserDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Checking " + userDirs.Length + " users if they have projects.");
foreach (FileInfo userDir in userDirs)
{
string username = Path.GetFileNameWithoutExtension(userDir.Name);
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Checking if user " + LoggerHelper.ForceLength(username, 10) + " has projects",
userCurrent, userTotal));
string userProjects = Resources.ProjectDirectory + "/" + username;
if (!FileStore.DirectoryExists(userProjects) || FileStore.GetFiles(userProjects).Length == 0)
{
FileStore.RemoveFile(Resources.UserDirectory, username + ".json");
FileStore.RemoveFile(Resources.ProjectDirectory, username + ".json");
FileStore.RemoveDirectory(Resources.ProjectDirectory + "/" + username);
}
}
}
/// <summary>
/// Writes all code and their relations to projects to CSV files.
/// </summary>
/// <param name="skip">the number of projects to skip before parsing</param>
/// <param name="limit">the number of projects to parse code for</param>
public void WriteCode(int skip, int limit)
{
DirectoryInfo[] projects = FileStore.GetDirectories(Resources.CodeDirectory);
int projectTotal = projects.Length;
int projectCurrent = 0;
Logger.Log("Parsing code of " + projectTotal + " projects to CSV.");
using (CsvWriter commandWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.CommandsCsv), new[]
{
"scriptId", "projectId", "date", "depth", "scopeType", "scopeName", "command", "param1", "param2",
"param3", "param4", "param5", "param6", "param7", "param8", "param9", "param10", "param11", "param12",
"param13", "param14", "param15", "param16", "param17", "param18", "param19", "param20"
}, int.Parse(Resources.CommandsCsvLinesPerFile)))
using (CsvWriter scriptWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ScriptsCsv),
new[] { "scriptId", "projectId", "date", "scopeType", "scopeName", "lineCount" },
int.Parse(Resources.ScriptsCsvLinesPerFile)))
using (CsvWriter procedureWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ProceduresCsv),
new[] { "projectId", "date", "scopeType", "scopeName", "name", "argumentCount" },
int.Parse(Resources.ProceduresCsvLinesPerFile)))
{
foreach (DirectoryInfo project in projects)
{
projectCurrent++;
if (projectCurrent <= skip)
{
continue;
}
if (limit > 0 && projectCurrent > skip + limit)
{
break;
}
int projectId = int.Parse(project.Name);
Logger.Log(LoggerHelper.FormatProgress("Parsing code of project " + projectId,
projectCurrent, projectTotal));
foreach (FileInfo codeFile in project.GetFiles())
{
string code = File.ReadAllText(codeFile.FullName);
string codeDate = codeFile.Name.Substring(0, codeFile.Name.Length - 5);
ParsedCode parsedCode = ParseCode(projectId, DateTime.Parse(codeDate), code);
WriteAllToCsv(commandWriter, parsedCode.Commands);
WriteAllToCsv(scriptWriter, parsedCode.Scripts);
WriteAllToCsv(procedureWriter, parsedCode.Procedures);
}
}
}
}
/// <summary>
/// Writes all user data to CSV files.
/// </summary>
public void WriteUsers()
{
using (CsvWriter writer = new CsvWriter(FileStore.GetAbsolutePath(Resources.UsersCsv),
new[] { "id", "username", "joinDate", "country" }, int.Parse(Resources.UsersCsvLinesPerFile)))
{
FileInfo[] userFiles = FileStore.GetFiles(Resources.UserDirectory);
int userTotal = userFiles.Length;
int userCurrent = 0;
Logger.Log("Parsing " + userTotal + " users to CSV.");
if (userFiles.Length > 0 && File.ReadAllText(userFiles[0].FullName).Length == 0)
{
Logger.Log("Missing metadata for users.");
return;
}
foreach (FileInfo userFile in userFiles)
{
string username = userFile.Name.Remove(userFile.Name.Length - 5);
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Parsing user " + LoggerHelper.ForceLength(username, 10), userCurrent, userTotal));
string contents = File.ReadAllText(userFile.FullName);
if (contents.Length == 0)
{
Logger.Log("Missing metadata for user " + userFile.Name);
return;
}
JObject user;
try
{
user = JObject.Parse(File.ReadAllText(userFile.FullName));
}
catch (JsonReaderException e)
{
Logger.Log("The metadata for user `" + userFile.Name + "` could not be parsed.", e);
return;
}
writer
.Write(int.Parse(user["id"].ToString()))
.Write(user["username"].ToString())
.Write(((DateTime)user["history"]["joined"]).ToString("yyyy-MM-ddTHH:mm:ss"))
.Write(user["profile"]["country"].ToString())
.Newline();
}
}
}
/// <summary>
/// Extracts all archives into the data folder.
/// </summary>
/// <param name="overwrite">true if existing files should be overwritten</param>
/// <param name="append">true if only new files should be extracted if the user is already registered</param>
public void Extract(bool overwrite = false, bool append = false)
{
FileInfo[] archives = FileStore.GetFiles(Resources.ArchiveDirectory);
int archiveTotal = archives.Length;
int archiveCurrent = 0;
foreach (FileInfo archive in archives)
{
archiveCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Extracting archive " + LoggerHelper.ForceLength(archive.Name, 10), archiveCurrent, archiveTotal));
Extract(archive, overwrite, append);
}
}
/// <summary>
/// Creates a separate archive for each user containing all that user's data.
/// </summary>
/// <param name="overwrite">true if existing archives should be overwritten</param>
public void Archive(bool overwrite = false)
{
FileInfo[] users = FileStore.GetFiles(Resources.UserDirectory);
int userTotal = users.Length;
int userCurrent = 0;
foreach (FileInfo user in users)
{
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Archiving user " + LoggerHelper.ForceLength(user.Name, 10), userCurrent, userTotal));
string username = user.Name.Remove(user.Name.Length - 5);
Archive(username);
}
}
/// <summary>
/// Validates metadata for all users.
/// </summary>
public void ValidateUsers()
{
FileInfo[] users = FileStore.GetFiles(Resources.UserDirectory);
int userTotal = users.Length;
int userCurrent = 0;
Logger.Log(string.Format("Validating {0} users.", userTotal));
foreach (FileInfo user in users)
{
string username = user.Name.Remove(user.Name.Length - 5);
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Validating user " + LoggerHelper.ForceLength(username, 10), userCurrent, userTotal));
if (!IsValidJson(File.ReadAllText(user.FullName)))
{
InvalidUsers++;
Logger.Log("Metadata of user " + username + " is invalid");
}
}
}
/// <summary>
/// Downloads project code for all registered projects.
/// </summary>
public void DownloadProjects()
{
DateTime currentDate = DateTime.Now.Date;
FileInfo[] users = FileStore.GetFiles(Resources.ProjectDirectory);
int userTotal = users.Length;
int userCurrent = 0;
Logger.Log(string.Format("Downloading code for {0} users.", userTotal));
// Iterate over users
foreach (FileInfo user in users)
{
string username = user.Name.Remove(user.Name.Length - 5);
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Downloading code for user " + LoggerHelper.ForceLength(username, 10), userCurrent, userTotal));
JArray projects;
try
{
projects = JArray.Parse(FileStore.ReadFile(Resources.ProjectDirectory, username + ".json"));
}
catch (JsonReaderException e)
{
Logger.Log("Could not parse list of projects of user `" + username + "`", e);
return;
}
// Iterate over user projects
foreach (JToken project in projects)
{
DateTime modifyDate = DateTime.Parse(project["history"]["modified"].ToString()).Date;
int projectId = Convert.ToInt32(project["id"].ToString());
string codeDir = Resources.CodeDirectory + "/" + projectId;
string yesterdayFileName = currentDate.AddDays(-1).ToString("yyyy-MM-dd") + ".json";
string todayFileName = currentDate.ToString("yyyy-MM-dd") + ".json";
if (FileStore.FileExists(codeDir, todayFileName))
{
// Code already downloaded today
continue;
}
if (currentDate.Subtract(modifyDate).Days > 0 && FileStore.FileExists(codeDir, yesterdayFileName))
{
// No code modifications in last day, copy old file
FileStore.CopyFile(codeDir, yesterdayFileName, codeDir, todayFileName);
continue;
}
string projectCode = GetProjectCode(projectId);
if (projectCode == null)
{
// Code not downloaded for whatever reason
continue;
}
if (!Downloader.IsValidJson(projectCode))
{
// Invalid JSON, no need to save it
continue;
}
FileStore.WriteFile(codeDir, todayFileName, projectCode);
}
}
Logger.Log(string.Format("Successfully downloaded code for {0} users.\n", userCurrent));
}
public void PreparseCodeDuplicates()
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Removing code duplicates of " + userDirs.Length + " users.");
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Removing code duplicates of " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
Dictionary <int, DateTime> projectDates = new Dictionary <int, DateTime>();
FileInfo[] projectLists = userDir.GetFiles().OrderBy(projectList => projectList.Name).ToArray();
foreach (FileInfo projectList in projectLists)
{
JArray projects;
try
{
projects = JArray.Parse(File.ReadAllText(projectList.FullName));
}
catch (JsonReaderException e)
{
Logger.Log("The project metadata list of user `" + userDir.Name + "` could not be parsed.", e);
return;
}
foreach (JToken project in projects)
{
if (!(project is JObject))
{
Logger.Log("The metadata of a project of user `" + userDir.Name + "` could not be parsed.");
return;
}
JObject metadata = (JObject)project;
int projectId = int.Parse(metadata["id"].ToString());
DateTime modifyDate = DateTime.Parse(metadata["history"]["modified"].ToString());
if (projectDates.ContainsKey(projectId) && projectDates[projectId].Equals(modifyDate))
{
Logger.Log("Deleted duplicate code; " + projectId + "/" + projectList.Name);
string codePath = FileStore.GetAbsolutePath(Resources.CodeDirectory,
projectId + "/" + projectList.Name);
if (File.Exists(codePath))
{
File.Delete(codePath);
}
}
projectDates[projectId] = modifyDate;
}
}
}
}
private static void RemoveUnchangedProjectsFromLists()
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Updating project lists of " + userDirs.Length + " users.");
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Updating project lists of " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
FileInfo[] projectLists = userDir.GetFiles().OrderBy(projectList => projectList.Name).ToArray();
foreach (FileInfo projectList in projectLists)
{
JArray projects;
try
{
projects = JArray.Parse(File.ReadAllText(projectList.FullName));
}
catch (JsonReaderException e)
{
Logger.Log("The project metadata list of user `" + userDir.Name + "` could not be parsed.", e);
return;
}
JArray filteredProjects = new JArray();
foreach (JToken project in projects)
{
if (!(project is JObject))
{
Logger.Log("The metadata of a project of user `" + userDir.Name + "` could not be parsed.");
return;
}
JObject metadata = (JObject)project;
int projectId = int.Parse(metadata["id"].ToString());
if (FileStore.DirectoryExists(Resources.CodeDirectory + "/" + projectId))
{
filteredProjects.Add(project);
}
}
if (filteredProjects.Count == 0)
{
File.Delete(projectList.FullName);
}
else
{
File.WriteAllText(projectList.FullName, filteredProjects.ToString(Formatting.None));
}
}
}
}
/// <summary>
/// Writes all projects and their relations to authors to CSV files.
/// </summary>
public void WriteProjects()
{
using (CsvWriter projectRemixWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ProjectRemixCsv),
new[] { "childId", "parentId" }, int.Parse(Resources.ProjectRemixCsvLinesPerFile)))
using (CsvWriter projectWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ProjectsCsv), new[]
{
"authorId", "date", "projectId", "title", "modifyDate", "createDate", "shareDate", "viewCount",
"loveCount", "favoriteCount", "commentCount"
}, int.Parse(Resources.ProjectsCsvLinesPerFile)))
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Parsing metadata for " + userDirs.Length + " users to CSV.");
ISet <int> projectHistory = new HashSet <int>();
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Parsing project lists of user " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
foreach (FileInfo projectListFile in userDir.GetFiles())
{
JArray projectList;
try
{
projectList = JArray.Parse(File.ReadAllText(projectListFile.FullName));
}
catch (JsonReaderException e)
{
Logger.Log("The project list for user `" + username + "` could not be parsed.", e);
return;
}
foreach (JToken projectFile in projectList)
{
if (!(projectFile is JObject))
{
Logger.Log("A project of user `" + username + "` could not be parsed.");
return;
}
JObject project = (JObject)projectFile;
int authorId = int.Parse(project["author"]["id"].ToString());
int projectId = int.Parse(project["id"].ToString());
string remixParentId = project["remix"]["parent"].ToString();
string dataDate = projectListFile.Name.Substring(0, projectListFile.Name.Length - 5);
projectWriter
.Write(authorId)
.Write(dataDate)
.Write(projectId)
.Write(project["title"].ToString())
.Write(((DateTime)project["history"]["modified"]).ToString("yyyy-MM-ddTHH:mm:ss"))
.Write(((DateTime)project["history"]["created"]).ToString("yyyy-MM-ddTHH:mm:ss"))
.Write(((DateTime)project["history"]["shared"]).ToString("yyyy-MM-ddTHH:mm:ss"))
.Write(int.Parse(project["stats"]["views"].ToString()))
.Write(int.Parse(project["stats"]["loves"].ToString()))
.Write(int.Parse(project["stats"]["favorites"].ToString()))
.Write(int.Parse(project["stats"]["comments"].ToString()))
.Newline();
if (remixParentId != "" && !projectHistory.Contains(projectId))
{
projectRemixWriter
.Write(projectId)
.Write(int.Parse(remixParentId))
.Newline();
}
projectHistory.Add(projectId);
}
}
}
}
}