public static void HandleUnregisteredProject(string projectId, string projectDirectoryPath) { ZemiIO.TryAppendFile(projectId.ToString(), Path.Combine(projectDirectoryPath, "UnregisteredProjects.txt")); }
private static void HandleParsedProject(string projectId, string projectDirectoryPath) { ZemiIO.TryAppendFile(projectId.ToString(), Path.Combine(projectDirectoryPath, "HandledProjects.txt")); }
public static void ProcessJSON2(string projectDirectoryPath, int threads = 1) { ZemiIO.EnableConcurrentWriting(); Say("Preparing database-dependent data..."); Say("Checking SpriteType rows.. ", false); SeedSpriteTypeTable(); Say("OK"); Say("Mapping sb2 opcodes to sb3 opcodes.. ", false); MapSb2ToSb3.Map(false); Say("OK"); Say("Loading sb2 and sb3 opcode caches.. ", false); Dictionary <string, OpCode> sb2Opcodes = GetOpcodes(true, false); Dictionary <string, OpCode> sb3OpCodes = GetOpcodes(false, true); Say("OK"); Say($"Checking log output files, set to {projectDirectoryPath}.. ", false); ZemiIO.CreateFileIfNotexists(projectDirectoryPath, "HandledProjects.txt"); ZemiIO.CreateFileIfNotexists(projectDirectoryPath, "EmptyProjects.txt"); ZemiIO.CreateFileIfNotexists(projectDirectoryPath, "UnregisteredProjects.txt"); ZemiIO.CreateFileIfNotexists(projectDirectoryPath, "MalformedBlocks.txt"); ZemiIO.CreateDirectoryIfNotExists(projectDirectoryPath, @"remixes\"); string remixPath = Path.Combine(projectDirectoryPath, @"remixes\"); ZemiIO.CreateDirectoryIfNotExists(projectDirectoryPath, @"sb1\"); string sb1Path = Path.Combine(projectDirectoryPath, @"sb1\"); Say("OK"); Say($"Enumerating files in {projectDirectoryPath}. This may take a while.. "); string[] allFilesInDirectory = Directory.GetFiles(projectDirectoryPath); ConcurrentStack <string> allFilesToHandle = new ConcurrentStack <string>(allFilesInDirectory); int count_allFilesInDirectory = allFilesInDirectory.Count(); allFilesInDirectory = null; GC.Collect(); Say($"Enumeration completed. Found {allFilesToHandle.Count()} files."); int totalFilesHandled = 0; System.Timers.Timer titleBarUpdateTimer = new System.Timers.Timer(); titleBarUpdateTimer.Elapsed += (sender, args) => Console.Title = $"{totalFilesHandled}/{count_allFilesInDirectory}"; titleBarUpdateTimer.Interval = 1000; titleBarUpdateTimer.Start(); Say("Starting parsers... NOW!"); for (int i = 0; i < threads; i++) { Say($"Starting parser {i+1} of {threads}."); new Thread(() => { using (ApplicationDatabase ctxt = new ApplicationDatabase()) { Sb2Parser sb2Parser = new Sb2Parser(sb2Opcodes); Sb3Parser sb3Parser = new Sb3Parser(sb3OpCodes); while (true) { string handling = null; while (handling == null) { allFilesToHandle.TryPop(out handling); } Interlocked.Increment(ref totalFilesHandled); string[] splitProjectFileName = handling.Split('.'); if (splitProjectFileName.Length != 3) { continue; //This guarantees we only handle projectid.sb3.json files, and not the handledprojects.txt, for example. } string projectId = Path.GetFileName(splitProjectFileName[0]); string projectSbVersion = splitProjectFileName[1]; int projectIdAsInt = Int32.Parse(projectId); if (projectSbVersion == "sb") { try { File.Move(handling, Path.Combine(sb1Path, Path.GetFileName(handling))); } catch (Exception ex) { Console.WriteLine(ex.Message, ex.StackTrace); } continue; } if (ctxt.Scripts.Any(o => o.ProjectId == projectIdAsInt)) { //Say($"Already handled {projectIdAsInt}"); continue; } Project possibleExistingProject = ctxt.Projects.AsNoTracking().Where(o => o.Id == projectIdAsInt).FirstOrDefault(); if (possibleExistingProject == null) { //Say($"Unregistered: {projectId}"); HandleUnregisteredProject(projectId.ToString(), projectDirectoryPath); continue; } else if (possibleExistingProject.IsRemix) { //Say($"Remix: {projectId}"); File.Move(handling, Path.Combine(remixPath, Path.GetFileName(handling))); continue; } string projectJson = File.ReadAllText(handling); if (string.IsNullOrEmpty(projectJson) || string.IsNullOrWhiteSpace(projectJson)) { Say($"Empty: {projectId}"); HandleEmptyProject(projectId.ToString(), projectDirectoryPath); continue; } else if (projectJson.StartsWith("ScratchV") || projectJson.StartsWith("PK")) { try { Say($"Uncaught sb1: {Path.GetFileName(handling)}"); File.Move(handling, Path.Combine(sb1Path, Path.GetFileName(handling))); } catch (Exception ex) { Console.WriteLine(ex.Message, ex.StackTrace); } continue; } try { switch (projectSbVersion) { case "sb2": sb2Parser.ParseProject(projectJson, projectId); break; case "sb3": sb3Parser.ParseProject(projectJson, Int32.Parse(projectId)); break; default: Say("Can't parse sb1 formats yet."); break; } HandleParsedProject(projectId, projectDirectoryPath); } catch (Exception ex) { Say(ex.Message); } } } }, 16000000).Start(); } Say("Look at 'em go!"); }