private void ConsumeIgnored(ReadOnlyMemory <char> source)
{
while (CurrentPosition < source.Length)
{
if (IgnoreWhiteSpace)
{
var currentCharacter = source.At(CurrentPosition);
if (WhiteSpaces.Contains(currentCharacter))
{
CurrentPosition++;
CurrentColumn++;
continue;
}
}
if (IgnoreEOL)
{
var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
if (eol != EOLType.No)
{
CurrentPosition += eol == EOLType.Windows ? 2 : 1;
CurrentColumn = 0;
CurrentLine++;
continue;
}
}
break;
}
}
private void ConsumeIgnored(ReadOnlyMemory <char> source, LexerPosition position)
{
while (position.Index < source.Length)
{
if (IgnoreWhiteSpace)
{
var currentCharacter = source.At(position.Index);
if (WhiteSpaces.Contains(currentCharacter))
{
position.Index++;
position.Column++;
continue;
}
}
if (IgnoreEOL)
{
var eol = EOLManager.IsEndOfLine(source, position.Index);
if (eol != EOLType.No)
{
position.Index += eol == EOLType.Windows ? 2 : 1;
position.Column = 0;
position.Line++;
continue;
}
}
break;
}
}
public static EOLType IsEndOfLine(ReadOnlyMemory <char> value, int position)
{
var end = EOLType.No;
var n = value.At(position);
if (n == '\n')
{
end = EOLType.Nix;
}
else if (n == '\r')
{
if (value.At(position + 1) == '\n')
{
end = EOLType.Windows;
}
else
{
end = EOLType.Mac;
}
}
return(end);
}
public static bool CheckDate(ReadOnlyMemory <char> value)
{
var ok = false;
if (value.Length == 6)
{
ok = char.IsDigit(value.At(0));
ok = ok && char.IsDigit(value.At(1));
ok = ok && value.At(2) == '.';
ok = ok && char.IsDigit(value.At(3));
ok = ok && char.IsDigit(value.At(4));
ok = ok && value.At(5) == '.';
}
return(ok);
}
public FSMMatch <N> Run(ReadOnlyMemory <char> source, LexerPosition lexerPosition)
{
ConsumeIgnored(source, lexerPosition);
// End of token stream
if (lexerPosition.Index >= source.Length)
{
return(new FSMMatch <N>(false));
}
// Make a note of where current token starts
var position = lexerPosition.Clone();
FSMMatch <N> result = null;
var currentNode = Nodes[0];
while (lexerPosition.Index < source.Length)
{
var currentCharacter = source.At(lexerPosition);
var currentValue = source.Slice(position.Index, lexerPosition.Index - position.Index + 1);
currentNode = Move(currentNode, currentCharacter, currentValue);
if (currentNode == null)
{
// No more viable transitions, so exit loop
break;
}
if (currentNode.IsEnd)
{
// Remember the possible match
result = new FSMMatch <N>(true, currentNode.Value, currentValue, position, currentNode.Id, lexerPosition, currentNode.IsLineEnding);
}
lexerPosition.Index++;
lexerPosition.Column++;
}
if (result != null)
{
// Backtrack
var length = result.Result.Value.Length;
lexerPosition.Index = result.Result.Position.Index + length;
lexerPosition.Column = result.Result.Position.Column + length;
if (HasCallback(result.NodeId))
{
result = Callbacks[result.NodeId](result);
}
return(result);
}
if (lexerPosition.Index >= source.Length)
{
// Failed on last character, so need to backtrack
lexerPosition.Index -= 1;
lexerPosition.Column -= 1;
}
var errorChar = source.Slice(lexerPosition.Index, 1);
var ko = new FSMMatch <N>(false, default(N), errorChar, lexerPosition, -1, lexerPosition, false);
return(ko);
}
public FSMMatch <N> Run(ReadOnlyMemory <char> source, int start)
{
var value = "";
int tokenStartIndex = start;
int tokenLength = 0;
var result = new FSMMatch <N>(false);
var successes = new Stack <FSMMatch <N> >();
CurrentPosition = start;
var currentNode = Nodes[0];
var lastNode = 0;
TokenPosition position = null;
var tokenStarted = false;
if (CurrentPosition < source.Length)
{
var currentCharacter = source.At(CurrentPosition);
while (CurrentPosition < source.Length && currentNode != null)
{
currentCharacter = source.Span[CurrentPosition];
var consumeSkipped = true;
while (consumeSkipped && !tokenStarted && CurrentPosition < source.Length)
{
currentCharacter = source.At(CurrentPosition);
if (IgnoreWhiteSpace && WhiteSpaces.Contains(currentCharacter))
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition++;
CurrentColumn++;
}
else
{
var eol = EOLManager.IsEndOfLine(source, CurrentPosition);
if (IgnoreEOL && eol != EOLType.No)
{
if (successes.Any())
{
currentNode = null;
}
else
{
currentNode = Nodes[0];
}
CurrentPosition += eol == EOLType.Windows ? 2 : 1;
CurrentColumn = 0;
CurrentLine++;
}
else
{
consumeSkipped = false;
}
}
}
currentNode = Move(currentNode, currentCharacter, value);
if (currentNode != null)
{
lastNode = currentNode.Id;
value += currentCharacter;
tokenLength++;
if (!tokenStarted)
{
tokenStarted = true;
position = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
}
if (currentNode.IsEnd)
{
var resultInter = new FSMMatch <N>(true, currentNode.Value, value, position, currentNode.Id);
successes.Push(resultInter);
}
if (HasAction(currentNode.Id))
{
value = Actions[currentNode.Id](value);
}
CurrentPosition++;
CurrentColumn++;
}
else
{
if (!successes.Any() && CurrentPosition < source.Length)
{
throw new LexerException(new LexicalError(CurrentLine, CurrentColumn,
source.At(CurrentPosition)));
}
}
}
}
if (successes.Any())
{
result = successes.Pop();
if (HasCallback(result.NodeId))
{
result = Callbacks[result.NodeId](result);
}
}
return(result);
}
public LexerResult <JsonToken> Tokenize(ReadOnlyMemory <char> source)
{
var tokens = new List <Token <JsonToken> >();
var position = 0;
var currentLine = 1;
var currentColumn = 1;
var length = source.Length;
var currentTokenLine = 1;
var currentTokenColumn = 1;
var currentTokenPosition = 1;
var currentValue = "";
var InString = false;
var InNum = false;
var InNull = false;
var InTrue = false;
var InFalse = false;
var NumIsDouble = false;
int tokenStartIndex = 0;
int tokenLength = 0;
Func <JsonToken, Token <JsonToken> > NewToken = tok =>
{
var token = new Token <JsonToken>();
token.Position = new TokenPosition(currentTokenPosition, currentTokenLine, currentTokenColumn);
token.SpanValue = source.Slice(tokenStartIndex, tokenLength);
tokenStartIndex = tokenStartIndex + tokenLength;
token.TokenID = tok;
tokens.Add(token);
currentValue = "";
return(token);
};
while (position < length)
{
var current = source.At(position);
if (InString)
{
currentValue += current;
if (current == '"')
{
InString = false;
NewToken(JsonToken.STRING);
position++;
}
else
{
position++;
}
}
else if (InNum)
{
if (current == '.')
{
NumIsDouble = true;
currentValue += current;
}
else if (char.IsDigit(current))
{
currentValue += current;
var type = NumIsDouble ? JsonToken.DOUBLE : JsonToken.INT;
if (position == length - 1)
{
NewToken(type);
}
}
else
{
InNum = false;
var type = NumIsDouble ? JsonToken.DOUBLE : JsonToken.INT;
NewToken(type);
position--;
}
position++;
}
else if (InNull)
{
if (current == "null"[currentValue.Length])
{
currentValue += current;
if (currentValue.Length == 4)
{
NewToken(JsonToken.NULL);
InNull = false;
}
}
position++;
}
else if (InFalse)
{
if (current == "false"[currentValue.Length])
{
currentValue += current;
if (currentValue.Length == 5)
{
NewToken(JsonToken.BOOLEAN);
InFalse = false;
}
else
{
position++;
}
}
}
else if (InTrue)
{
if (current == "true"[currentValue.Length])
{
currentValue += current;
if (currentValue.Length == 5)
{
NewToken(JsonToken.BOOLEAN);
InFalse = false;
}
}
position++;
}
else
{
currentValue += current;
if (current == '"')
{
InString = true;
currentValue += current;
currentTokenColumn = currentColumn;
currentTokenLine = currentLine;
}
else if (char.IsDigit(current))
{
InNum = true;
}
else if (current == 't')
{
InTrue = true;
}
else if (current == 'f')
{
InFalse = true;
}
else if (current == 'n')
{
InNull = true;
}
else if (current == '[')
{
NewToken(JsonToken.CROG);
}
else if (current == ']')
{
NewToken(JsonToken.CROD);
}
else if (current == '{')
{
NewToken(JsonToken.ACCG);
}
else if (current == '}')
{
NewToken(JsonToken.ACCD);
}
else if (current == ':')
{
NewToken(JsonToken.COLON);
}
else if (current == ',')
{
NewToken(JsonToken.COMMA);
}
else if (char.IsWhiteSpace(current))
{
currentValue = "";
}
else if (current == '\n' || current == '\r')
{
currentLine++;
currentValue = ";;";
currentColumn = 1;
}
currentColumn++;
position++;
}
}
return(new LexerResult <JsonToken>(tokens));
}