private RuleMatchResult ValidateType(Lexer.Lexme lexme) { foreach (Lexer.LexmeType type in Types) { if (lexme.Type == type) { StopAutoValidate = true; return(RuleMatchResult.Match); } } if (this.Type == RuleType.ValidateAllUntilMatch && !StopAutoValidate) { return(RuleMatchResult.Waiting); } //no matches, is it still valid? if (this.Optional) { return(RuleMatchResult.Waiting); } else { return(RuleMatchResult.Invalid); } }
public Dictionary <string, Parser.Token> GetAllInherits() { //List<Parser.Token> calls = new List<Parser.Token>(); Dictionary <string, Parser.Token> calls = new Dictionary <string, Parser.Token>(); foreach (Parser.Token token in this.Tokens) { if (token.Type == Parser.TokenType.InheritCommand) { string funcName = ""; //gotta find out the form of the function call with respect to //the scope resolution operator for (int index = 0; index < token.Lexmes.Count; index++) { Lexer.Lexme lexme = token.Lexmes[index]; if (lexme.Type == Lexer.LexmeType.Instruction && lexme.Data.ToString() == "inherit") { funcName = token.Code; break; } } if (funcName.Length < 1 || funcName == "") { throw new Exception("Unidentifiable form for inherit."); } calls.Add(funcName, token); } } return(calls); }
private RuleMatchResult ValidateValue(Lexer.Lexme lexme) { foreach (string value in Values) { if (lexme.Data.ToString() == value) { StopAutoValidate = true; return(RuleMatchResult.Match); } } if (this.Type == RuleType.ValidateAllUntilMatch && !StopAutoValidate) { return(RuleMatchResult.Waiting); } //no matches, is it still valid? if (this.Optional) { return(RuleMatchResult.Waiting); } else { return(RuleMatchResult.Invalid); } }
public Dictionary <string, Parser.Token> GetAllFunctionCalls(Parser.Token root) { //List<Parser.Token> calls = new List<Parser.Token>(); Dictionary <string, Parser.Token> calls = new Dictionary <string, Parser.Token>(); Dictionary <string, int> FuncInstanceCount = new Dictionary <string, int>(); foreach (Parser.Token token in root.Children[1]) { if (token.Type == Parser.TokenType.FunctionCall) { string funcName = ""; //gotta find out the form of the function call with respect to //the scope resolution operator for (int index = 0; index < token.Lexmes.Count; index++) { Lexer.Lexme lexme = token.Lexmes[index]; if (lexme.Data.ToString() == "::") { funcName = token.Lexmes[index + 1].Data.ToString(); break; } if (lexme.Type == Lexer.LexmeType.Identifier) { //look ahead if (token.Lexmes[index + 1].Data.ToString() == "::") { funcName = token.Lexmes[index + 2].Data.ToString(); break; } else { funcName = lexme.Data.ToString(); break; } } } if (funcName.Length < 1 || funcName == "") { throw new Exception("Unidentifiable form for function name."); } if (calls.ContainsKey(funcName)) { calls.Add(funcName + Globals.Generator.FunctionHashSymbol + FuncInstanceCount[funcName], token); FuncInstanceCount[funcName] += 1; } else { calls.Add(funcName, token); FuncInstanceCount.Add(funcName, 1); } } } return(calls); }
public bool CombineGroupedTokens(Lexme lexme, List <Lexme> tokenList, int startLine) { if (ConsumingFlag) { //we previously started consuming, now we need to know if we can stop if (PairStateMachine.ValidMatchingCloseLexme(StartLexme.Type, lexme.Type)) { ConsumingFlag = false; //HACK CITY! if (StartLexme.Type == LexmeType.CommentLine) { tokenList.Add(StartLexme); tokenList.Add(lexme); } else { if (StartLexme.Type == LexmeType.OpenComment) { StartLexme.Type = LexmeType.Comment; } else if (StartLexme.Type == LexmeType.Quote) { StartLexme.Type = LexmeType.StringLiteral; } StartLexme.Data.Append(lexme.Data); tokenList.Add(StartLexme); } StartLexme = null; } else { StartLexme.Data.Append(lexme.Data); } } else { //we have not opened consumption yet. Do we start? if (PairStateMachine.RequiresClosing(lexme.Type)) { //yes ConsumingFlag = true; Begin(lexme, startLine); } else { tokenList.Add(lexme); } } return(ConsumingFlag); }
/// <summary> /// /// </summary> /// <param name="spawn"></param> private void ScanSpawnedGrammars(Lexer.Lexme activeLexme, ref LinkedList <Syntax> spawned, List <Token> identifiedTokens) { int spawnedCount = 0; LinkedList <Syntax> removalList = new LinkedList <Syntax>(); foreach (Syntax spawn in spawned) { //if a spawned item is complete and it is the front of the list, we have our man if (spawn.IsComplete() && spawn == spawned.First.Value) { //we got ooooooonnne! *riiiiiiing* int startIndex = spawn.StartIndex; identifiedTokens.Add(new Token(spawn.LexmeToken.Lexmes, spawn.TokenID)); //ok, now every spawned syntax that overlaps with this one's lexme indices //can also be ruled out. However, non overlaping ones naturally come afterward so //we just leave them alone. They may be validating our next token already. //(Note this also removes the spawn we just identified) foreach (Syntax de in spawned) { if (de.StartIndex <= startIndex) { removalList.AddLast(de); } } spawnedCount = 0; } //nope, continue validating any noncompleted ones until they are ruled out or completed else { if (!spawn.IsAcceptableInput(activeLexme) && !spawn.IsComplete()) { //this puppy is ruled out, remove from list //spawned.Remove(spawn); removalList.AddLast(spawn); } } spawnedCount++; } //finally, remove the unwanted fluff foreach (Syntax syn in removalList) { spawned.Remove(syn); } }
public Lexer(Stellarmass.LPC.Scanner.Scanner scannedData) { int lineCount = 0; int tokenCount = 0; GreedyLexmePair greedy = new GreedyLexmePair(); List <Lexme> tempTokensList = new List <Lexme>(); foreach (Scanner.CodeLine line in scannedData.Lines) { foreach (Scanner.Token scanned in line.Tokens) { try{ Lexme lexme = new Lexme(); lexme.Data = scanned.Data; lexme.Type = IdentifyType(scanned, lineCount + 1, greedy); //this will automatically combine lexme with any previous lexmes if they are supposed to //be grouped together as a string or comment greedy.CombineGroupedTokens(lexme, tempTokensList, lineCount + 1); } catch (LexerException e) { throw new LexerException(e.Message, lineCount); } tokenCount++; } //end foreach //OLD POSITION lineCount++; tokenCount = 0; } //NEW POSITION //just in case the line ended before we could finish the token list if (greedy.FlushGroup() != null) { tempTokensList.AddRange(greedy.FlushGroup()); } Lines.Add(new Line(tempTokensList, Lines.Count)); }
public int SyntaxID = 0; //0 means unknown token public LexmeContainer(Lexer.Lexme lexme, int index) { Lexme = lexme; Index = index; }
public LexmeHierarchy(List <Lexer.Lexme> lexmes, LexmeHierarchy parent, int index, int recursionDepth) { LexmeContainer ActiveContainer; LexmeContainers = new List <LexmeContainer>(); Parent = parent; //for(int index = 0; index < lexmes.Count; index++) while (index < lexmes.Count) { Lexer.Lexme lexme = lexmes[index]; //shortcut to convert from a foreach loop that was previously used string lexstr = lexme.Data.ToString(); if (LPC.LPCKeywords.IsClosingPair(lexstr)) { if (recursionDepth < 1) { throw new ParserException("Unmatched " + lexstr + "found."); } recursionDepth--; return; } LexmeContainers.Add(new LexmeContainer(lexme, index)); ActiveContainer = LexmeContainers[LexmeContainers.Count - 1]; this.LexmeCount++; if (LPC.LPCKeywords.IsOpenningPair(lexstr)) { //we have stepped into the realm of recursion, watch your step! ActiveContainer.Children = new LexmeHierarchy(lexmes, this, index + 1, recursionDepth + 1); //in order to track indexing correctly through recursion each hierarchy //needs to store not only the number of lexmecontainers it has but the //recursed number within each sub-hierarhchy within each lexmecontainer. //If there were no consumed lexmes in the recursion we can just remove the children stub. //(UPDATE: Actually, we can't remove that stub, I need it XD ) this.LexmeCount += ActiveContainer.Children.LexmeCount; index += ActiveContainer.Children.LexmeCount; //we know now that the next lexme is ending the recursion so we can just consume it now this.LexmeCount++; index++; LexmeContainers.Add(new LexmeContainer(lexmes[index], index)); //remove empty children stub //UPDATE: nevermind, I need it /*if(ActiveContainer.Children.LexmeCount <= 0) * { * ActiveContainer.Children = null; * }*/ } index++; } //end foreach return; }
private void Begin(Lexme start, int startLine) { StartLine = startLine; StartLexme = start; }
public GreedyLexmePair() { StartLexme = new Lexme(); StartLexme.Type = LexmeType.none; }
/// <summary> /// This function /// </summary> /// <param name="name"></param> /// <param name="root"></param> private void RemoveFunctionCallFromCode(string name, Parser.Token root) { const int FUNC_BODY_TOKENS = 1; //foreach(Parser.Token token in root.Children[FUNC_BODY_TOKENS]) for (int count = 0; count < root.Children[FUNC_BODY_TOKENS].Count; count++) { Parser.Token token = root.Children[FUNC_BODY_TOKENS][count]; if (token.Type == Parser.TokenType.FunctionCall) { string funcName = ""; //gotta find out the form of the function call with respect to //the scope resolution operator for (int index = 0; index < token.Lexmes.Count; index++) { Lexer.Lexme lexme = token.Lexmes[index]; if (lexme.Data.ToString() == "::") { funcName = token.Lexmes[index + 1].Data.ToString(); break; } if (lexme.Type == Lexer.LexmeType.Identifier) { //look ahead if (token.Lexmes[index + 1].Data.ToString() == "::") { funcName = token.Lexmes[index + 2].Data.ToString(); break; } else { //found the name of the currently investigated grammar token. //is it the one we are looking for? funcName = lexme.Data.ToString(); if (name == funcName) { Parser.Token funccall = root.Children[FUNC_BODY_TOKENS][count]; Parser.Token pretab = null, newline = null; if (count > 0) { pretab = root.Children[FUNC_BODY_TOKENS][count - 1]; } if (root.Children[FUNC_BODY_TOKENS].Count - 1 <= count + 1) { newline = root.Children[FUNC_BODY_TOKENS][count + 1]; } //WARNING - hackish! //This parameter is used to remove the tab token and newline token that were //placed after the function back in 'PushDataToCode'. It should only be set //if you knoew for sure there is a tab before this function and a newline after. //It is used to remove all that exra space generated when a new function is created and then never used. //yup root.Children[FUNC_BODY_TOKENS].Remove(funccall); //if(pretab != null) {root.Children[FUNC_BODY_TOKENS].Remove(pretab);} //if(newline != null) {root.Children[FUNC_BODY_TOKENS].Remove(newline);} } break; } } } } } return; }
public RuleMatchResult Validate(Lexer.Lexme lexme) { //check for the potential that the rule itself is whitespace! if (this.Types.Count == 1 && this.Types[0] == Lexer.LexmeType.Whitespace && lexme.Type == Lexer.LexmeType.Whitespace) { return(RuleMatchResult.Match); } //make sure any whitespace, line feeds and comments are handled properly. //They can never be a match but they can be flagged as valid or invalid depending //on the rule's settings. if (lexme.Type == Lexer.LexmeType.NewLine) { if ((Flags & (int)RuleFlags.DontAllowLinefeeds) != 0) { return(RuleMatchResult.Invalid); } else { return(RuleMatchResult.Valid); } } if (lexme.Type == Lexer.LexmeType.Whitespace) { if ((Flags & (int)RuleFlags.DontAllowSpaces) != 0) { return(RuleMatchResult.Invalid); } else { return(RuleMatchResult.Valid); } } if (lexme.Type == Lexer.LexmeType.Comment) { if ((Flags & (int)RuleFlags.DontAllowComments) != 0) { return(RuleMatchResult.Invalid); } else { return(RuleMatchResult.Valid); } } RuleMatchResult result = RuleMatchResult.Invalid; switch (MatchType) { case RuleMatchType.ByValue: { result = ValidateValue(lexme); break; } case RuleMatchType.ByType: { result = ValidateType(lexme); break; } } return(result); }
public bool IsAcceptableInput(Lexer.Lexme lexme) { if (Complete) { return(false); } Token.Lexmes.Add(lexme); //Here, we are systematically checking every rule until we have a match or an invalid lexme. //In the event of a match we can increment the rules-index so that we start with the next //rule the next time this function is called. If we have a match and it was the final rule //we set the completed flag. //Keep in mind that optional rules will still return valid if we don't have a match, this allows us //to keep moving forward until we match a rule or we come across a required rule that cannot be //matched at which point it is invalid. Thus, we only need to return a success or fail state based //on a found match or an invalid lexme. for (int i = RuleIndex; i < Rules.Count; i++) { //-slight hack- //we are going to remove any leading whitespace symbols by returning invalid if (lexme.Type == Lexer.LexmeType.Whitespace || lexme.Type == Lexer.LexmeType.NewLine) { if (this.Length < 1) { return(false); } } switch (Rules[i].Validate(lexme)) { case RuleMatchResult.Match: { //if we've matched all required rules, flag the construct as completed if (i == Rules.Count - 1) { Complete = true; } //otherwise, we are at least one step closer RuleIndex = i + 1; Length++; return(true); } case RuleMatchResult.Valid: { //no match but we are still in the game Length++; return(true); } case RuleMatchResult.Waiting: { RuleIndex++; break; } case RuleMatchResult.Invalid: { /*if(Rules[i].Optional) * { * RuleIndex = i + 1; * Length++; * return true; * }*/ ResetValidation(true); return(false); } } } return(false); }