protected LexerPath ConfigureNewLexerPath(LexerPath lexerPath)
{
CurrentLexerPathId += 1;
lexerPath.CurrentToken.LexerPathId = CurrentLexerPathId;
lexerPath.LexerPathID = CurrentLexerPathId;
LexerPathMap.Add(CurrentLexerPathId, lexerPath);
return(lexerPath);
}
protected List <IToken> NextTokens(int lexerPathId)
{
List <IToken> tokenList = new List <IToken>();
LexerPath lexerPath;
if (LexerPathMap.TryGetValue(lexerPathId, out lexerPath))
{
NextTokens(lexerPath, tokenList);
}
else
{
throw new StepLexerException(string.Format("LexerPath with id {0} does not exist.", lexerPathId));
}
return(tokenList);
}
protected void NextTokens(LexerPath lexerPath, List <IToken> tokenList)
{
if (lexerPath.CurrentToken.Terminal.Equals(Token.NULL))
{
if (_sourceLines.Count != 0)
{
lexerPath.ActiveLineNumber = Token.LINEPOSITION_START;
}
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.BOF, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
tokenList.Add(lexerPath.CurrentToken);
}
// Check if EOF (sourceline empty)
else if (_sourceLines.Count == 0 && lexerPath.CurrentToken.Terminal.Equals(Token.BOF))
{
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.EOF, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
tokenList.Add(lexerPath.CurrentToken);
}
// Check if EOF (lastline EOL)
else if (lexerPath.CurrentToken.Terminal.Equals(Token.EOL) && _sourceLines.Count <= lexerPath.ActiveLineNumber)
{
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.EOF, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
tokenList.Add(lexerPath.CurrentToken);
}
else if (lexerPath.CurrentToken.Terminal.Equals(Token.BOF))
{
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.BOL, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
lexerPath.ActiveCharacterNumber = Token.CHARPOSITION_START;
tokenList.Add(lexerPath.CurrentToken);
}
// Check if BOL (lastline EOL)
else if (lexerPath.CurrentToken.Terminal.Equals(Token.EOL) && _sourceLines.Count > lexerPath.ActiveLineNumber)
{
lexerPath.ActiveLineNumber += 1;
lexerPath.ActiveCharacterNumber = Token.CHARPOSITION_START;
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.BOL, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
tokenList.Add(lexerPath.CurrentToken);
}
// check if EOL
else if (_sourceLines[lexerPath.ActiveLineNumber].Length < lexerPath.ActiveCharacterNumber && !lexerPath.CurrentToken.Terminal.Equals(Token.EOL))
{
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.EOL, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
tokenList.Add(lexerPath.CurrentToken);
}
// check indent level change after EOL
else if (_lexerOptions.ReturnIndentToken)
{
int indentCount = CountIndent(lexerPath, _sourceLines[lexerPath.ActiveLineNumber]);
if (indentCount != lexerPath.ActiveIndentNumber)
{
if (indentCount > lexerPath.ActiveIndentNumber)
{
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.INDENT_INCREASED, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
tokenList.Add(lexerPath.CurrentToken);
}
else
{
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.INDENT_DECREASED, string.Empty, lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber);
tokenList.Add(lexerPath.CurrentToken);
}
lexerPath.ActiveIndentNumber = indentCount;
}
}
else
{
List <TerminalMatch> terminalMatches = new List <TerminalMatch>();
// TODO Redo to make it more efficient than match all terminals
foreach (Terminal terminal in lexerPath.Terminals)
{
Match singleMatch = terminal.Match(_sourceLines[lexerPath.ActiveLineNumber].Substring(lexerPath.ActiveCharacterNumber));
if (singleMatch.Success)
{
terminalMatches.Add(new TerminalMatch(terminal, singleMatch.Value));
}
}
if (terminalMatches.Count == 0)
{
if (LexerPathMap.Count > 1)
{
/* remove lexer path as it was earlier dublicated */
LexerPathMap.Remove(lexerPath.LexerPathID);
NotifyCollapsingLexerPath(lexerPath);
}
else
{
// if no match progress ActiveCharacterNumber 1 and return unknownterminal
lexerPath.CurrentToken = new Token(lexerPath.LexerPathID, Token.UNKNOWN_TERMINAL, _sourceLines[lexerPath.ActiveLineNumber].Substring(lexerPath.ActiveCharacterNumber, 1), lexerPath.ActiveLineNumber, lexerPath.ActiveCharacterNumber + 1);
lexerPath.ActiveCharacterNumber += 1;
tokenList.Add(lexerPath.CurrentToken);
}
}
else
{
int matchCounter = 0;
// if match is ok find terminal, progress _activeCharacterNumber and add token
foreach (TerminalMatch terminalMatch in terminalMatches)
{
matchCounter += 1;
LexerPath newLexerPath = lexerPath;
if (terminalMatches.Count > 1 && matchCounter > 1)
{
newLexerPath = ConfigureNewLexerPath(newLexerPath.Clone());
NotifySplitLexerPath(newLexerPath);
}
if (!terminalMatch.IgnoreTerminal)
{
newLexerPath.CurrentToken = new Token(newLexerPath.LexerPathID, terminalMatch.Terminal.TerminalName, terminalMatch.Capture, newLexerPath.ActiveLineNumber, newLexerPath.ActiveCharacterNumber);
}
else
{
NotifyIgnoreTerminal(new Token(newLexerPath.LexerPathID, terminalMatch.Terminal.TerminalName, terminalMatch.Capture, newLexerPath.ActiveLineNumber, newLexerPath.ActiveCharacterNumber));
}
newLexerPath.ActiveCharacterNumber += terminalMatch.Capture.Length;
if (terminalMatch.IgnoreTerminal)
{
NextTokens(newLexerPath, tokenList);
}
else
{
tokenList.Add(newLexerPath.CurrentToken);
}
}
}
}
}
protected void Reset()
{
LexerPathMap.Clear();
CurrentLexerPathId = LexerPath.NOTSET;
NewStartLexerPath();
}
