public FSMMatch <N> Run(ReadOnlyMemory <char> source, int start)
{
var value = "";
int tokenStartIndex = start;
int tokenLength = 0;
var result = new FSMMatch <N>(false);
var successes = new Stack <FSMMatch <N> >();
CurrentPosition = start;
var currentNode = Nodes[0];
var lastNode = 0;
TokenPosition position = null;
var tokenStarted = false;
if (CurrentPosition < source.Length)
{
var currentCharacter = source.At(CurrentPosition);
while (CurrentPosition < source.Length && currentNode != null)
{
currentCharacter = source.Span[CurrentPosition];
var consumeSkipped = true;
while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
{
currentCharacter = source.At(CurrentPosition);
if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentCharacter))
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition++;
CurrentColumn++;
}
else
{
var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
if (IgnoreEOL && eol != EOLType.No)
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition += eol == EOLType.Windows ? 2 : 1;
CurrentColumn = 0;
CurrentLine++;
}
else
{
consumeSkipped = false;
}
}
}
currentNode = Move(currentNode, currentCharacter, value);
if (currentNode != null)
{
lastNode = currentNode.Id;
value += currentCharacter;
tokenLength++;
if (!tokenStarted)
{
tokenStarted = true;
position = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
}
if (currentNode.IsEnd)
{
var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position, currentNode.Id);
successes.Push(resultInter);
}
if (HasAction(currentNode.Id))
{
value = Actions[currentNode.Id](value);
}
CurrentPosition++;
CurrentColumn++;
}
else
{
if (!successes.Any() && CurrentPosition < source.Length)
{
throw new LexerException(new LexicalError(CurrentLine, CurrentColumn,
source.At(CurrentPosition)));
}
}
}
}
if (successes.Any())
{
result = successes.Pop();
if (HasCallback(result.NodeId))
{
result = Callbacks[result.NodeId](result);
}
}
return(result);
}
public FSMMatch <N> Run(ReadOnlyMemory <char> source, LexerPosition lexerPosition)
{
ConsumeIgnored(source, lexerPosition);
// End of token stream
if (lexerPosition.Index >= source.Length)
{
return(new FSMMatch <N>(false));
}
// Make a note of where current token starts
var position = lexerPosition.Clone();
FSMMatch <N> result = null;
var currentNode = Nodes[0];
while (lexerPosition.Index < source.Length)
{
var currentCharacter = source.At(lexerPosition);
var currentValue = source.Slice(position.Index, lexerPosition.Index - position.Index + 1);
currentNode = Move(currentNode, currentCharacter, currentValue);
if (currentNode == null)
{
// No more viable transitions, so exit loop
break;
}
if (currentNode.IsEnd)
{
// Remember the possible match
result = new FSMMatch <N>(true, currentNode.Value, currentValue, position, currentNode.Id, lexerPosition, currentNode.IsLineEnding);
}
lexerPosition.Index++;
lexerPosition.Column++;
}
if (result != null)
{
// Backtrack
var length = result.Result.Value.Length;
lexerPosition.Index = result.Result.Position.Index + length;
lexerPosition.Column = result.Result.Position.Column + length;
if (HasCallback(result.NodeId))
{
result = Callbacks[result.NodeId](result);
}
return(result);
}
if (lexerPosition.Index >= source.Length)
{
// Failed on last character, so need to backtrack
lexerPosition.Index -= 1;
lexerPosition.Column -= 1;
}
var errorChar = source.Slice(lexerPosition.Index, 1);
var ko = new FSMMatch <N>(false, default(N), errorChar, lexerPosition, -1, lexerPosition, false);
return(ko);
}
public FSMMatch <N> Run(string source, int start)
{
string value = "";
var result = new FSMMatch <N>(false);
var successes = new Stack <FSMMatch <N> >();
CurrentPosition = start;
FSMNode <N> currentNode = Nodes[0];
int lastNode = 0;
TokenPosition position = null;
bool tokenStarted = false;
if (CurrentPosition < source.Length)
{
char currentToken = source[CurrentPosition];
while (CurrentPosition < source.Length && currentNode != null)
{
currentToken = source[CurrentPosition];
bool consumeSkipped = true;
while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
{
currentToken = source[CurrentPosition];
if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentToken))
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition++;
CurrentColumn++;
}
else
{
var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
if (IgnoreEOL && eol != EOLType.No)
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition += (eol == EOLType.Windows ? 2 : 1);
CurrentColumn = 0;
CurrentLine++;
}
else
{
consumeSkipped = false;
}
}
}
currentNode = Move(currentNode, currentToken, value);
if (currentNode != null)
{
lastNode = currentNode.Id;
value += currentToken;
if (!tokenStarted)
{
tokenStarted = true;
position = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
}
if (currentNode.IsEnd)
{
var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position);
successes.Push(resultInter);
}
if (HasAction(currentNode.Id))
{
value = Actions[currentNode.Id](value);
}
CurrentPosition++;
CurrentColumn++;
}
else
{
if (lastNode == 0 && !tokenStarted && !successes.Any() && CurrentPosition < source.Length)
{
throw new LexerException(new LexicalError(CurrentLine, CurrentColumn, source[CurrentPosition]));
}
;
}
}
}
if (successes.Any())
{
result = successes.Pop();
if (HasCallback(lastNode))
{
result = Callbacks[lastNode](result);
}
}
return(result);
}