public TokenStream ParseFile(string sourcefile) { var fs = new TextFileScanner(sourcefile); return(Parse(fs)); }
private TokenStream Parse(TextFileScanner fs) { stream = new TokenStream(); var tokenContent = new StringBuilder(); int startLine = 0; int startColumn = 0; while (fs.CanRead && fs.Peek() != TextFileScanner.EOF) { tokenContent.Length = 0; startLine = fs.Line; startColumn = fs.Column; Token?token = null; var currentStates = this.nfaTable.ComputeNodeEClosure(this.nfaTable.StartState); do { string symbol = fs.Peek().ToString(); if (symbol == TextFileScanner.EOF.ToString()) { break; } tokenContent.Append(symbol); currentStates = this.nfaTable.ApplyTransition(currentStates, symbol); var finalNodes = currentStates.Where(node => node.IsFinal); var tags = new HashSet <(int priority, string tag)>(); foreach (var node in finalNodes) { foreach (var fullTag in node.GetTags(Node.LABEL)) { var data = fullTag.Split('~'); string tag = data[0]; int priority = int.Parse(data[1]); tags.Add((priority, tag)); } } var orderedTags = tags.OrderByDescending(val => val.priority); var highestPriority = orderedTags.FirstOrDefault(); var secondHighestPriority = orderedTags.ElementAtOrDefault(1); string highestPriorityTag = highestPriority.tag; if (highestPriorityTag != null) { Debug.Assert(secondHighestPriority.tag == null || highestPriority.priority > secondHighestPriority.priority, $"Conflict of priority found between {highestPriority.tag}:{highestPriority.priority} and {secondHighestPriority.tag}:{secondHighestPriority.priority}"); token = new Token(highestPriorityTag, tokenContent.ToString(), startLine + 1, startColumn + 1); } if (!fs.CanRead && fs.Peek() == TextFileScanner.EOF) { break; } fs.GoNext(); } while (currentStates.Count > 0); Debug.Assert(token != null, $"Invalid symbol found during token parsing: '{tokenContent.ToString().Last()}' Ascii value:{(int)tokenContent.ToString().Last()} around {startLine}:{startColumn}"); int goBackN = tokenContent.Length - token.Content.Length; for (int i = 0; i < goBackN; i++) { fs.GoPrevious(); } stream.Add(token); } return(new TokenStream(stream)); }