/// <summary>
/// Creates buffered token.
/// </summary>
private void CreateBuffered()
{
string tokenName = buffer.ToString();
ParsedToken token = new ParsedToken
{
Name = tokenName,
InRowPosition = inRowPosition - buffer.Length,
RowIndex = rowIndex,
InStringPosition = inStringPosition - buffer.Length,
};
// If the state doesn't have a link for the char
// but has token name then the state
// returns token of defined in the state type.
// Otherwise if buffer was not empty
// but there is no token name then
// the automaton was interrupted where it was not meant
// to be so we create an undefined token
// and return to start state.
if (currentState.TokenName != null)
{
IDefinedToken definedToken;
if (currentState.TokenName == classTable.UnclassifiedTokenClassName ||
nodes.Terminals.Any(n => n.Name == tokenName))
{
definedToken = (IDefinedToken)nodes.Terminals.Single(n => n.Name == tokenName);
}
else
{
definedToken = nodes.Tokens.Single(n => n.Name == currentState.TokenName && n.TokenClass == currentState.TokenClass);
}
token.Id = definedToken.Id;
token.TokenClassId = definedToken.TokenClassId;
}
else
{
token.Id = UndefinedTokenClassId;
token.TokenClassId = UndefinedTokenClassId;
}
buffer.Clear();
currentState = StartState;
tokens.Add(token);
Logger.Add("tokenParser", new TokenLog
{
TokenClass = token.TokenClassId.ToString(),
TokenName = token.Name, Message = "created token",
});
}
/// <summary>
/// Determines whether should move on to the next state
/// or create new token and then return.
/// Also fills respective tables with newly created tokens.
/// </summary>
/// <param name="ch">Current symbol to process.</param>
/// <param name="symbolCategory">Category of current symbol.</param>
/// <param name="symbolClass">Class of current symbol.</param>
/// <param name="isBufferedTokenCreated">Whether token was created from buffer or not.</param>
private void ProcessSymbol(char ch, SymbolCategory symbolCategory, string symbolClass, out bool isBufferedTokenCreated)
{
isBufferedTokenCreated = false;
// If current state has link for this symbol
// then move to the next state and fill buffer with the character.
if (symbolClass != null &&
currentState.Links != null &&
currentState.Links.ContainsKey(symbolClass))
{
buffer.Append(ch);
currentState = GetState(currentState.Links[symbolClass]);
return;
}
// if the buffer is empty and symbol is undefined
// then an undefined token will be created.
if (string.IsNullOrWhiteSpace(buffer.ToString()))
{
if (symbolCategory == SymbolCategory.Undefined)
{
ParsedToken undefinedToken = new ParsedToken
{
Id = -1,
TokenClassId = -1,
Name = ch.ToString(),
InRowPosition = inRowPosition,
RowIndex = rowIndex,
InStringPosition = inStringPosition,
};
tokens.Add(undefinedToken);
}
return;
}
CreateBuffered();
isBufferedTokenCreated = true;
}
