/// <summary> /// Dump top level of nodes for processing in another application or debugging /// NOTE: Since this only dumps the top level, only the first token of each node will be printed /// eg function arguments will have text as their function name, but the arguments won't be present in the output /// TODO: print all info in output, alternatively dump allLines structure as JSON or something /// </summary> /// <param name="lines"></param> /// <param name="subroutineDatabase"></param> /// <param name="savePath"></param> static bool DumpTopLevelNodes(string[] lines, SubroutineDatabase subroutineDatabase, string savePath) { RenpyScriptBuilder scriptBuilder = new RenpyScriptBuilder(); bool error = ParseSection(lines, subroutineDatabase, isProgramBlock: true, out List <List <Node> > allLines); if (lines.Length != allLines.Count()) { throw new Exception("lines weren't decoded correctly"); } using (StreamWriter writer = File.CreateText(savePath)) { foreach (var line in allLines) { foreach (Node n in line) { writer.Write($"{n.GetLexeme().type}\0{n.GetLexeme().text}\0"); } writer.WriteLine(); } } return(error); }
public LexerTest(string line, SubroutineDatabase subroutineDatabase) { this.line = line; this.subroutineDatabase = subroutineDatabase; this.lexemes = new List <Lexeme>(); //Console.WriteLine($"Full Line [{line}]"); }
static int Main(string[] args) { if (args.Length < 1) { Console.WriteLine("Error: First argument of program must be the script to be parsed"); Console.WriteLine("\nI suggest using this script for testing: https://github.com/07th-mod/umineko-question/raw/master/InDevelopment/ManualUpdates/0.utf"); Console.WriteLine("\nVisual Studio users can follow these instructions to set program arguments: https://stackoverflow.com/questions/298708/debugging-with-command-line-parameters-in-visual-studio"); Console.ReadKey(); return(-1); } // Read input script string inputFilePath = args[0]; Console.WriteLine($"Processing input file [{inputFilePath}]"); string[] lines = File.ReadAllLines(inputFilePath); // First pass of script is to build the function database SubroutineDatabase database = UserFunctionScanner.buildInitialUserList(lines, new SubroutineDatabase()); // Add predefined functions. If a function already exists, it will be added with an '_' prefix PredefinedFunctionInfoLoader.load("function_list.txt", database); // Add some extra pre-defined functions database["langen"] = new SubroutineInformation(false); database["langjp"] = new SubroutineInformation(false); database["showlangen"] = new SubroutineInformation(false); database["showlangjp"] = new SubroutineInformation(false); database["langall"] = new SubroutineInformation(false); //TODO: not sure if this is a real function or a typo. Only occurs once in umineko question script. database["endroll"] = new SubroutineInformation(true); database["steamsetachieve"] = new SubroutineInformation(true); database["getreadlang"] = new SubroutineInformation(true); database["say"] = new SubroutineInformation(true); database["tachistate"] = new SubroutineInformation(true); foreach (KeyValuePair <string, SubroutineInformation> kvp in database.GetRawDict()) { Console.WriteLine($"{kvp.Key}: {kvp.Value.hasArguments}"); } bool dumpNodes = true; bool error = true; if (dumpNodes) { string savePath = @"ponscripter_script_nodes.txt"; error = DumpTopLevelNodes(lines, database, savePath); } else { error = CompileScript(lines, database); } return(error ? -2 : 0); }
static bool ParseSection(string[] lines, SubroutineDatabase subroutineDatabase, bool isProgramBlock, out List <List <Node> > lines_nodes) { lines_nodes = new List <List <Node> >(); bool error = false; using (StreamWriter writer = File.CreateText(errorFileSavePath)) { for (int i = 0; i < lines.Length; i++) { string line = lines[i]; try { LexerTest test = new LexerTest(line, subroutineDatabase); test.LexSection(isProgramBlock); Parser p = new Parser(test.lexemes, subroutineDatabase); List <Node> nodes = p.Parse(); lines_nodes.Add(nodes); } catch (Exception e) { List <Node> dummyList = new List <Node>(); dummyList.Add(new CommentNode(new Lexeme(LexemeType.COMMENT, "<<<<<< PARSING ERROR >>>>>>"))); lines_nodes.Add(dummyList); // If lexing or parsing a line fails, record this line as having an error, then move on writer.WriteLine(); writer.WriteLine($"Error on line [{i+1}]: {line}"); writer.WriteLine($"Details: {e}"); error = true; } } } // If any errors were found during lexing/parsing, notify the user if (error) { Console.Error.WriteLine($"ERROR: One or more errors were found! See file {errorFileSavePath} for details"); } return(error); }
public static void load(string filePath, SubroutineDatabase database) { string[] lines = File.ReadAllLines(filePath); foreach (string line in lines) { string[] segments = line.Split(new char[] { '\0' }); string functionName = segments[0]; if (segments.Length > 2) { Console.WriteLine($"Function '{functionName}' has more than one set of arguments - will just use first function definition"); } string firstArgsList = segments[1]; //If the function is already defined by user, assume it has been overwritten, and prefix an underscore string databaseKeyName = database.ContainsKey(functionName) ? $"_{functionName}" : functionName; database[databaseKeyName] = decodeArgumentsFromList(functionName, firstArgsList); } }
static bool CompileScript(string[] lines, SubroutineDatabase subroutineDatabase) { #if true RenpyScriptBuilder scriptBuilder = new RenpyScriptBuilder(); TreeWalker walker = new TreeWalker(scriptBuilder); //CodeBlocks cbs = ReadSegments(lines); StringBuilder simpleWriter = new StringBuilder(); StringBuilder debugBuilder = new StringBuilder(); HashSet <string> modified_lines = new HashSet <string>(); bool error = ParseSection(lines, subroutineDatabase, isProgramBlock: true, out List <List <Node> > allLines); if (error) { return(error); } if (lines.Length != allLines.Count()) { throw new Exception("lines weren't decoded correctly"); } string debugPath = @"C:\drojf\large_projects\ponscripter_parser\renpy\ponscripty\game\debug.txt"; string savePath = @"C:\drojf\large_projects\ponscripter_parser\renpy\ponscripty\game\script.rpy"; scriptBuilder.SaveFile("prelude.rpy", savePath); using (StreamWriter writer = File.CreateText(savePath)) { writer.Write(simpleWriter.ToString()); } using (StreamWriter writer = File.CreateText(debugPath)) { writer.WriteLine("Unique Modified Lines:"); foreach (string s in modified_lines) { writer.WriteLine(s.ToString()); } writer.WriteLine("\n\n"); writer.WriteLine("Possibly Missed lines:"); writer.Write(debugBuilder.ToString()); } #else RenpyScriptBuilder scriptBuilder = new RenpyScriptBuilder(); TreeWalker walker = new TreeWalker(scriptBuilder); CodeBlocks cbs = ReadSegments(lines); StringBuilder simpleWriter = new StringBuilder(); StringBuilder debugBuilder = new StringBuilder(); HashSet <string> modified_lines = new HashSet <string>(); // Write to Init Region //scriptBuilder.SetBodyRegion(); foreach (string line in cbs.header) { AppendSLToForwardSlashAndBlankLine(line, subroutineDatabase, scriptBuilder, walker, simpleWriter, debugBuilder, modified_lines, isProgramBlock: true); } foreach (string line in cbs.definition) { AppendSLToForwardSlashAndBlankLine(line, subroutineDatabase, scriptBuilder, walker, simpleWriter, debugBuilder, modified_lines, isProgramBlock: true); } // Write to Body Region //scriptBuilder.SetBodyRegion(); foreach (string line in cbs.program) { AppendSLToForwardSlashAndBlankLine(line, subroutineDatabase, scriptBuilder, walker, simpleWriter, debugBuilder, modified_lines, isProgramBlock: true); } string debugPath = @"C:\drojf\large_projects\ponscripter_parser\renpy\ponscripty\game\debug.txt"; string savePath = @"C:\drojf\large_projects\ponscripter_parser\renpy\ponscripty\game\script.rpy"; scriptBuilder.SaveFile("prelude.rpy", savePath); using (StreamWriter writer = File.CreateText(savePath)) { writer.Write(simpleWriter.ToString()); } using (StreamWriter writer = File.CreateText(debugPath)) { writer.WriteLine("Unique Modified Lines:"); foreach (string s in modified_lines) { writer.WriteLine(s.ToString()); } writer.WriteLine("\n\n"); writer.WriteLine("Possibly Missed lines:"); writer.Write(debugBuilder.ToString()); } #endif return(true); }
public Parser(List <Lexeme> lexemes, SubroutineDatabase subroutineDatabase) { this.lexemes = lexemes; this.subroutineDatabase = subroutineDatabase; }
/// <summary> /// Scan the entire script for subroutine declarations and definitions /// </summary> /// <param name="allLines"></param> public static SubroutineDatabase buildInitialUserList(string[] allLines, SubroutineDatabase database) { List <LabelInfo> labels = new List <LabelInfo>(); //scan for subroutines definitions and labels for (int i = 0; i < allLines.Length; i++) { string s = allLines[i]; Match defsubMatch = defsubRegex.Match(s); Match labelMatch = labelRegex.Match(s); if (defsubMatch.Success) { //search the script for "defsub ut_mld2" type calls. store each in a hashmap string subName = defsubMatch.Groups[1].Value; Console.WriteLine($"Got sub definition {subName}"); if (!database.TryAdd(subName, null)) { Console.WriteLine($"WARNING: duplicate defsub found for {subName}"); } } else if (labelMatch.Success) { string labelName = labelMatch.Groups[1].Value; Console.WriteLine($"Got label definition {labelName}"); labels.Add(new LabelInfo(labelName, i)); } } //iterate starting at the label definition until (worst case) the end of file //when a getparam or return is found, the number of parameters is recorded SubroutineInformation GetSubroutineInformation(string[] linesToSearch, LabelInfo label) { for (int i = label.labelLineIndex; i < linesToSearch.Length; i++) { string s = linesToSearch[i]; Match getParamMatch = getParamRegex.Match(s); Match returnMatch = returnRegex.Match(s); if (getParamMatch.Success) { Console.WriteLine($"{label.labelName} arguments are {getParamMatch.Groups[0].Value} [{s}]"); return(new SubroutineInformation(getParamMatch.Groups[0].Value)); } else if (returnMatch.Success) { Console.WriteLine($"{label.labelName} takes no arguments"); return(new SubroutineInformation(hasArguments: false)); } } //somehow reached end of the document return(null); } //For reach subroutine, determine what arguments (if any) it uses //May not work for all cases, but usually getparam is the first call //after the label def so use this method for now. foreach (LabelInfo label in labels) { //Only process subroutine definitions, not ordinary labels if (!database.InnerTryGetValue(label.labelName, out _)) { continue; } Console.WriteLine($"{label.labelName} is a subroutine"); SubroutineInformation subInfo = GetSubroutineInformation(allLines, label); if (subInfo == null) { Console.WriteLine($"ERROR: no return or getparam for subroutine {label.labelName}"); } else { database[label.labelName] = subInfo; } } return(database); }