/// <summary>
/// Validates all code files of each project.
/// </summary>
public void ValidateCode()
{
DirectoryInfo[] projects = FileStore.GetDirectories(Resources.CodeDirectory);
int projectTotal = projects.Length;
int projectCurrent = 0;
Logger.Log(string.Format("Validating code of {0} projects.", projectTotal));
foreach (DirectoryInfo project in projects)
{
string projectId = project.Name;
projectCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Validating code of project with id " + projectId, projectCurrent, projectTotal));
FileInfo[] codeFiles = project.GetFiles();
foreach (FileInfo codeFile in codeFiles)
{
if (!IsValidJson(File.ReadAllText(codeFile.FullName)))
{
InvalidCodeFiles++;
Logger.Log("Code of project " + projectId + " at date " + codeFile.Name + " is invalid");
}
}
}
}
private static void RemoveUnchangedProjectCode()
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.CodeDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Removing unchanged projects of " + userDirs.Length + " users.");
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Removing unchanged projects of user " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
FileInfo[] projectLists = userDir.GetFiles().OrderBy(projectList => projectList.Name).ToArray();
if (projectLists.Length <= 1)
{
userDir.Delete(true);
}
}
}
/// <summary>
/// Validates all project lists of each user.
/// </summary>
public void ValidateProjectLists()
{
DirectoryInfo[] users = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = users.Length;
int userCurrent = 0;
Logger.Log(string.Format("Validating project lists of {0} users.", userTotal));
foreach (DirectoryInfo user in users)
{
string username = user.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Validating project lists of user " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
FileInfo[] projectLists = user.GetFiles();
foreach (FileInfo projectList in projectLists)
{
if (!IsValidJson(File.ReadAllText(projectList.FullName)))
{
InvalidProjectLists++;
Logger.Log("Project list of user " + username + " at date " + projectList.Name + " is invalid");
}
}
}
}
/// <summary>
/// Writes all code and their relations to projects to CSV files.
/// </summary>
/// <param name="skip">the number of projects to skip before parsing</param>
/// <param name="limit">the number of projects to parse code for</param>
public void WriteCode(int skip, int limit)
{
DirectoryInfo[] projects = FileStore.GetDirectories(Resources.CodeDirectory);
int projectTotal = projects.Length;
int projectCurrent = 0;
Logger.Log("Parsing code of " + projectTotal + " projects to CSV.");
using (CsvWriter commandWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.CommandsCsv), new[]
{
"scriptId", "projectId", "date", "depth", "scopeType", "scopeName", "command", "param1", "param2",
"param3", "param4", "param5", "param6", "param7", "param8", "param9", "param10", "param11", "param12",
"param13", "param14", "param15", "param16", "param17", "param18", "param19", "param20"
}, int.Parse(Resources.CommandsCsvLinesPerFile)))
using (CsvWriter scriptWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ScriptsCsv),
new[] { "scriptId", "projectId", "date", "scopeType", "scopeName", "lineCount" },
int.Parse(Resources.ScriptsCsvLinesPerFile)))
using (CsvWriter procedureWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ProceduresCsv),
new[] { "projectId", "date", "scopeType", "scopeName", "name", "argumentCount" },
int.Parse(Resources.ProceduresCsvLinesPerFile)))
{
foreach (DirectoryInfo project in projects)
{
projectCurrent++;
if (projectCurrent <= skip)
{
continue;
}
if (limit > 0 && projectCurrent > skip + limit)
{
break;
}
int projectId = int.Parse(project.Name);
Logger.Log(LoggerHelper.FormatProgress("Parsing code of project " + projectId,
projectCurrent, projectTotal));
foreach (FileInfo codeFile in project.GetFiles())
{
string code = File.ReadAllText(codeFile.FullName);
string codeDate = codeFile.Name.Substring(0, codeFile.Name.Length - 5);
ParsedCode parsedCode = ParseCode(projectId, DateTime.Parse(codeDate), code);
WriteAllToCsv(commandWriter, parsedCode.Commands);
WriteAllToCsv(scriptWriter, parsedCode.Scripts);
WriteAllToCsv(procedureWriter, parsedCode.Procedures);
}
}
}
}
/// <summary>
/// Loads the content recursively.
/// </summary>
/// <param name="dirs">The dirs.</param>
/// <param name="data">The data.</param>
/// <returns></returns>
private IDictionary <string, IContent> LoadContentRecursive(IEnumerable <string> dirs, IDictionary <string, IContent> data)
{
foreach (var dir in dirs)
{
var content = GetByPath(dir);
if (content != null)
{
data.Add(content.RelativeUrl, content);
}
LoadContentRecursive(_fileStore.GetDirectories(dir), data);
}
return(data);
}
public void PreparseCodeDuplicates()
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Removing code duplicates of " + userDirs.Length + " users.");
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Removing code duplicates of " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
Dictionary <int, DateTime> projectDates = new Dictionary <int, DateTime>();
FileInfo[] projectLists = userDir.GetFiles().OrderBy(projectList => projectList.Name).ToArray();
foreach (FileInfo projectList in projectLists)
{
JArray projects;
try
{
projects = JArray.Parse(File.ReadAllText(projectList.FullName));
}
catch (JsonReaderException e)
{
Logger.Log("The project metadata list of user `" + userDir.Name + "` could not be parsed.", e);
return;
}
foreach (JToken project in projects)
{
if (!(project is JObject))
{
Logger.Log("The metadata of a project of user `" + userDir.Name + "` could not be parsed.");
return;
}
JObject metadata = (JObject)project;
int projectId = int.Parse(metadata["id"].ToString());
DateTime modifyDate = DateTime.Parse(metadata["history"]["modified"].ToString());
if (projectDates.ContainsKey(projectId) && projectDates[projectId].Equals(modifyDate))
{
Logger.Log("Deleted duplicate code; " + projectId + "/" + projectList.Name);
string codePath = FileStore.GetAbsolutePath(Resources.CodeDirectory,
projectId + "/" + projectList.Name);
if (File.Exists(codePath))
{
File.Delete(codePath);
}
}
projectDates[projectId] = modifyDate;
}
}
}
}
private static void RemoveUnchangedProjectsFromLists()
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Updating project lists of " + userDirs.Length + " users.");
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Updating project lists of " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
FileInfo[] projectLists = userDir.GetFiles().OrderBy(projectList => projectList.Name).ToArray();
foreach (FileInfo projectList in projectLists)
{
JArray projects;
try
{
projects = JArray.Parse(File.ReadAllText(projectList.FullName));
}
catch (JsonReaderException e)
{
Logger.Log("The project metadata list of user `" + userDir.Name + "` could not be parsed.", e);
return;
}
JArray filteredProjects = new JArray();
foreach (JToken project in projects)
{
if (!(project is JObject))
{
Logger.Log("The metadata of a project of user `" + userDir.Name + "` could not be parsed.");
return;
}
JObject metadata = (JObject)project;
int projectId = int.Parse(metadata["id"].ToString());
if (FileStore.DirectoryExists(Resources.CodeDirectory + "/" + projectId))
{
filteredProjects.Add(project);
}
}
if (filteredProjects.Count == 0)
{
File.Delete(projectList.FullName);
}
else
{
File.WriteAllText(projectList.FullName, filteredProjects.ToString(Formatting.None));
}
}
}
}
/// <summary>
/// Writes all projects and their relations to authors to CSV files.
/// </summary>
public void WriteProjects()
{
using (CsvWriter projectRemixWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ProjectRemixCsv),
new[] { "childId", "parentId" }, int.Parse(Resources.ProjectRemixCsvLinesPerFile)))
using (CsvWriter projectWriter = new CsvWriter(FileStore.GetAbsolutePath(Resources.ProjectsCsv), new[]
{
"authorId", "date", "projectId", "title", "modifyDate", "createDate", "shareDate", "viewCount",
"loveCount", "favoriteCount", "commentCount"
}, int.Parse(Resources.ProjectsCsvLinesPerFile)))
{
DirectoryInfo[] userDirs = FileStore.GetDirectories(Resources.ProjectDirectory);
int userTotal = userDirs.Length;
int userCurrent = 0;
Logger.Log("Parsing metadata for " + userDirs.Length + " users to CSV.");
ISet <int> projectHistory = new HashSet <int>();
foreach (DirectoryInfo userDir in userDirs)
{
string username = userDir.Name;
userCurrent++;
Logger.Log(LoggerHelper.FormatProgress(
"Parsing project lists of user " + LoggerHelper.ForceLength(username, 10),
userCurrent, userTotal));
foreach (FileInfo projectListFile in userDir.GetFiles())
{
JArray projectList;
try
{
projectList = JArray.Parse(File.ReadAllText(projectListFile.FullName));
}
catch (JsonReaderException e)
{
Logger.Log("The project list for user `" + username + "` could not be parsed.", e);
return;
}
foreach (JToken projectFile in projectList)
{
if (!(projectFile is JObject))
{
Logger.Log("A project of user `" + username + "` could not be parsed.");
return;
}
JObject project = (JObject)projectFile;
int authorId = int.Parse(project["author"]["id"].ToString());
int projectId = int.Parse(project["id"].ToString());
string remixParentId = project["remix"]["parent"].ToString();
string dataDate = projectListFile.Name.Substring(0, projectListFile.Name.Length - 5);
projectWriter
.Write(authorId)
.Write(dataDate)
.Write(projectId)
.Write(project["title"].ToString())
.Write(((DateTime)project["history"]["modified"]).ToString("yyyy-MM-ddTHH:mm:ss"))
.Write(((DateTime)project["history"]["created"]).ToString("yyyy-MM-ddTHH:mm:ss"))
.Write(((DateTime)project["history"]["shared"]).ToString("yyyy-MM-ddTHH:mm:ss"))
.Write(int.Parse(project["stats"]["views"].ToString()))
.Write(int.Parse(project["stats"]["loves"].ToString()))
.Write(int.Parse(project["stats"]["favorites"].ToString()))
.Write(int.Parse(project["stats"]["comments"].ToString()))
.Newline();
if (remixParentId != "" && !projectHistory.Contains(projectId))
{
projectRemixWriter
.Write(projectId)
.Write(int.Parse(remixParentId))
.Newline();
}
projectHistory.Add(projectId);
}
}
}
}
}