/// <summary> /// Creates buffered token. /// </summary> private void CreateBuffered() { string tokenName = buffer.ToString(); ParsedToken token = new ParsedToken { Name = tokenName, InRowPosition = inRowPosition - buffer.Length, RowIndex = rowIndex, InStringPosition = inStringPosition - buffer.Length, }; // If the state doesn't have a link for the char // but has token name then the state // returns token of defined in the state type. // Otherwise if buffer was not empty // but there is no token name then // the automaton was interrupted where it was not meant // to be so we create an undefined token // and return to start state. if (currentState.TokenName != null) { IDefinedToken definedToken; if (currentState.TokenName == classTable.UnclassifiedTokenClassName || nodes.Terminals.Any(n => n.Name == tokenName)) { definedToken = (IDefinedToken)nodes.Terminals.Single(n => n.Name == tokenName); } else { definedToken = nodes.Tokens.Single(n => n.Name == currentState.TokenName && n.TokenClass == currentState.TokenClass); } token.Id = definedToken.Id; token.TokenClassId = definedToken.TokenClassId; } else { token.Id = UndefinedTokenClassId; token.TokenClassId = UndefinedTokenClassId; } buffer.Clear(); currentState = StartState; tokens.Add(token); Logger.Add("tokenParser", new TokenLog { TokenClass = token.TokenClassId.ToString(), TokenName = token.Name, Message = "created token", }); }
/// <summary> /// Determines whether should move on to the next state /// or create new token and then return. /// Also fills respective tables with newly created tokens. /// </summary> /// <param name="ch">Current symbol to process.</param> /// <param name="symbolCategory">Category of current symbol.</param> /// <param name="symbolClass">Class of current symbol.</param> /// <param name="isBufferedTokenCreated">Whether token was created from buffer or not.</param> private void ProcessSymbol(char ch, SymbolCategory symbolCategory, string symbolClass, out bool isBufferedTokenCreated) { isBufferedTokenCreated = false; // If current state has link for this symbol // then move to the next state and fill buffer with the character. if (symbolClass != null && currentState.Links != null && currentState.Links.ContainsKey(symbolClass)) { buffer.Append(ch); currentState = GetState(currentState.Links[symbolClass]); return; } // if the buffer is empty and symbol is undefined // then an undefined token will be created. if (string.IsNullOrWhiteSpace(buffer.ToString())) { if (symbolCategory == SymbolCategory.Undefined) { ParsedToken undefinedToken = new ParsedToken { Id = -1, TokenClassId = -1, Name = ch.ToString(), InRowPosition = inRowPosition, RowIndex = rowIndex, InStringPosition = inStringPosition, }; tokens.Add(undefinedToken); } return; } CreateBuffered(); isBufferedTokenCreated = true; }