private void ConsumeIgnored(ReadOnlyMemory <char> source, LexerPosition position)
{
while (position.Index < source.Length)
{
if (IgnoreWhiteSpace)
{
var currentCharacter = source.At(position.Index);
if (WhiteSpaces.Contains(currentCharacter))
{
position.Index++;
position.Column++;
continue;
}
}
if (IgnoreEOL)
{
var eol = EOLManager.IsEndOfLine(source, position.Index);
if (eol != EOLType.No)
{
position.Index += eol == EOLType.Windows ? 2 : 1;
position.Column = 0;
position.Line++;
continue;
}
}
break;
}
}
private void ConsumeIgnored(ReadOnlyMemory <char> source)
{
while (CurrentPosition < source.Length)
{
if (IgnoreWhiteSpace)
{
var currentCharacter = source.At(CurrentPosition);
if (WhiteSpaces.Contains(currentCharacter))
{
CurrentPosition++;
CurrentColumn++;
continue;
}
}
if (IgnoreEOL)
{
var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
if (eol != EOLType.No)
{
CurrentPosition += eol == EOLType.Windows ? 2 : 1;
CurrentColumn = 0;
CurrentLine++;
continue;
}
}
break;
}
}
public static string RemoveWhiteSpaces(this string value) => value == null ? null : WhiteSpaces.Replace(value, string.Empty);
public FSMMatch <N> Run(ReadOnlyMemory <char> source, int start)
{
var value = "";
int tokenStartIndex = start;
int tokenLength = 0;
var result = new FSMMatch <N>(false);
var successes = new Stack <FSMMatch <N> >();
CurrentPosition = start;
var currentNode = Nodes[0];
var lastNode = 0;
TokenPosition position = null;
var tokenStarted = false;
if (CurrentPosition < source.Length)
{
var currentCharacter = source.At(CurrentPosition);
while (CurrentPosition < source.Length && currentNode != null)
{
currentCharacter = source.Span[CurrentPosition];
var consumeSkipped = true;
while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
{
currentCharacter = source.At(CurrentPosition);
if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentCharacter))
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition++;
CurrentColumn++;
}
else
{
var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
if (IgnoreEOL && eol != EOLType.No)
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition += eol == EOLType.Windows ? 2 : 1;
CurrentColumn = 0;
CurrentLine++;
}
else
{
consumeSkipped = false;
}
}
}
currentNode = Move(currentNode, currentCharacter, value);
if (currentNode != null)
{
lastNode = currentNode.Id;
value += currentCharacter;
tokenLength++;
if (!tokenStarted)
{
tokenStarted = true;
position = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
}
if (currentNode.IsEnd)
{
var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position, currentNode.Id);
successes.Push(resultInter);
}
if (HasAction(currentNode.Id))
{
value = Actions[currentNode.Id](value);
}
CurrentPosition++;
CurrentColumn++;
}
else
{
if (!successes.Any() && CurrentPosition < source.Length)
{
throw new LexerException(new LexicalError(CurrentLine, CurrentColumn,
source.At(CurrentPosition)));
}
}
}
}
if (successes.Any())
{
result = successes.Pop();
if (HasCallback(result.NodeId))
{
result = Callbacks[result.NodeId](result);
}
}
return(result);
}
public FSMMatch <N> Run(string source, int start)
{
string value = "";
var result = new FSMMatch <N>(false);
var successes = new Stack <FSMMatch <N> >();
CurrentPosition = start;
FSMNode <N> currentNode = Nodes[0];
int lastNode = 0;
TokenPosition position = null;
bool tokenStarted = false;
if (CurrentPosition < source.Length)
{
char currentToken = source[CurrentPosition];
while (CurrentPosition < source.Length && currentNode != null)
{
currentToken = source[CurrentPosition];
bool consumeSkipped = true;
while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
{
currentToken = source[CurrentPosition];
if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentToken))
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition++;
CurrentColumn++;
}
else
{
var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
if (IgnoreEOL && eol != EOLType.No)
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition += (eol == EOLType.Windows ? 2 : 1);
CurrentColumn = 0;
CurrentLine++;
}
else
{
consumeSkipped = false;
}
}
}
currentNode = Move(currentNode, currentToken, value);
if (currentNode != null)
{
lastNode = currentNode.Id;
value += currentToken;
if (!tokenStarted)
{
tokenStarted = true;
position = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
}
if (currentNode.IsEnd)
{
var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position);
successes.Push(resultInter);
}
if (HasAction(currentNode.Id))
{
value = Actions[currentNode.Id](value);
}
CurrentPosition++;
CurrentColumn++;
}
else
{
if (lastNode == 0 && !tokenStarted && !successes.Any() && CurrentPosition < source.Length)
{
throw new LexerException(new LexicalError(CurrentLine, CurrentColumn, source[CurrentPosition]));
}
;
}
}
}
if (successes.Any())
{
result = successes.Pop();
if (HasCallback(lastNode))
{
result = Callbacks[lastNode](result);
}
}
return(result);
}