public static Token Parse(Tokenizer.State state)
{
if (!Char.IsWhiteSpace(state.CurrentCharacter))
{
return(null);
}
var c = state.CurrentCharacter;
var isNewLine = c == '\n';
if (isNewLine)
{
state.Line += 1;
state.LineBegin = state.Index + 1;
}
var token = new Token(
value: c.ToString(),
type: TokenType.Whitespace,
subType: isNewLine
? TokenSubType.NewLine
: TokenSubType.Space
);
state.Index += 1;
return(token);
}
private static Token ParseNormalCharacter(Tokenizer.State state)
{
switch (state.CurrentCharacter)
{
case ' ':
case '\t':
case '\n':
case '\r':
state.ErrorCode = (uint)ErrorCodes.T_MisleadingCharacter;
state.ErrorMessage =
"Character definition is misleading\n" +
"Whitespace symbols should be written using escape sequences:\n" +
" \\s - space\n" +
" \\t - tabulation\n" +
" \\n - new line\n" +
" \\r - carriage return";
break;
case '\0':
state.ErrorCode = (uint)ErrorCodes.T_UnexpectedEndOfFile;
state.ErrorMessage = "Character is expected but 'End of file' was found instead";
break;
default: break;
}
return(GenerateTokenFor(state.CurrentCharacter, state));
}
public static Token Parse(Tokenizer.State state)
{
if (state.CurrentCharacter != '#')
{
return(null);
}
int index = state.Index;
state.Index += 1;
while (IsMatching(state.CurrentCharacter, state))
{
state.Index += 1;
}
int length = state.Index - index;
if (length == 1)
{
state.ErrorCode = (uint)ErrorCodes.T_CompilerDirectiveNameIsNotStated;
state.ErrorMessage = "Compiler directive name is not stated";
}
return(new Token(
value: state.Code.Substring(index, length),
type: TokenType.CompilerDirective,
subType: TokenSubType.CompilerDirective
));
}
public static Token Parse(Tokenizer.State state)
{
if (state.CurrentCharacter != '"')
{
return(null);
}
var begin = state.Index + 1;
do
{
state.Index += 1;
while (state.CurrentCharacter == '\\')
{
state.Index += 2;
}
} while (!"\"\0".Contains(state.CurrentCharacter));
if (state.CurrentCharacter == '\0')
{
state.Index -= 1;
state.ErrorCode = (uint)ErrorCodes.T_UnexpectedEndOfFile;
state.ErrorMessage = "End of string was not found";
}
return(new Token(
value: state.Code.Substring(begin, state.Index++ - begin),
type: TokenType.String,
subType: TokenSubType.String
));
}
private static void ParseInlineCommentary(Tokenizer.State state)
{
while (state.CurrentCharacter != '\0' && !"\r\n".Contains(state.CurrentCharacter))
{
state.Index += 1;
}
}
private static void ParseMultilineCommentary(Tokenizer.State state)
{
while (true)
{
if (state.CurrentCharacter == '\0')
{
state.ErrorCode = (uint)ErrorCodes.T_UnexpectedEndOfFile;
state.ErrorMessage = "Commentary was not closed";
return;
}
if (state.CurrentCharacter == '*')
{
state.Index += 1;
if (state.CurrentCharacter == '/')
{
state.Index += 1;
return;
}
}
if (state.CurrentCharacter == '\n')
{
state.Line += 1;
state.LineBegin = state.Index + 1;
}
state.Index += 1;
}
}
private static bool TryParse
(Tokenizer.State state, Func <Tokenizer.State, Token> parser, out Token token)
{
state.Save();
Tokenizer.State stateCopy = new Tokenizer.State(state);
token = parser(state);
if (state.IsErrorOccured())
{
state.Drop();
return(true);
}
else if (token != null)
{
token.Index = stateCopy.Index;
token.Line = stateCopy.Line;
token.Position = stateCopy.Position;
state.Drop();
return(true);
}
state.Restore();
return(false);
}
private static bool IsMatching(Tokenizer.State state, ref int decimalPoint)
{
if (Char.IsDigit(state.CurrentCharacter))
{
return(true);
}
return(IsMatchingDecimal(state, ref decimalPoint));
}
private static Token GenerateTokenFor(char character, Tokenizer.State state)
{
state.Index += 1;
return(new Token(
value: character.ToString(),
type: TokenType.Character,
subType: TokenSubType.Character
));
}
private static bool IsMatchingDecimal(Tokenizer.State state, ref int decimalPoint)
{
if (state.CurrentCharacter != '.' || decimalPoint >= 0)
{
return(false);
}
decimalPoint = state.Index;
return(true);
}
public static Token Parse(Tokenizer.State state)
{
foreach (Func <Tokenizer.State, Token> parser in parsers)
{
if (TryParse(state, parser, out Token token))
{
return(token);
}
}
return(null);
}
public static Token Parse(Tokenizer.State state)
{
foreach (var parser in parsers)
{
Token token;
if (TryParse(state, parser, out token))
{
return(token);
}
}
return(null);
}
private static void ParseNumber(Tokenizer.State state, out int decimalPoint)
{
decimalPoint = -2;
while (IsMatching(state, ref decimalPoint))
{
state.Index += 1;
}
if (decimalPoint == state.Index - 1)
{
decimalPoint = -2;
state.Index -= 1;
}
}
public static Token Parse(Tokenizer.State state)
{
if (state.CurrentCharacter != '\'')
{
return(null);
}
state.Index += 1;
if (state.CurrentCharacter == '\\')
{
state.Index += 1;
return(ParseSpecialCharacter(state));
}
return(ParseNormalCharacter(state));
}
private static Token ParseSpecialCharacter(Tokenizer.State state)
{
switch (state.CurrentCharacter)
{
case 'n': return(GenerateTokenFor('\n', state));
case 't': return(GenerateTokenFor('\t', state));
case 's': return(GenerateTokenFor(' ', state));
case 'r': return(GenerateTokenFor('\r', state));
default:
state.ErrorCode = (uint)ErrorCodes.T_SpecialCharacterDoesNotExists;
state.ErrorMessage = $"Special character identifier '{state.CurrentCharacter}' does not exists";
return(GenerateTokenFor(state.CurrentCharacter, state));
}
}
public static Token Parse(Tokenizer.State state)
{
Tokenizer.State s = new Tokenizer.State(state);
ParseNumber(state, out int decimalPoint);
if (state.Index == s.Index)
{
return(null);
}
return(new Token(
value: state.Code.Substring(s.Index, state.Index - s.Index),
type: TokenType.Number,
subType: decimalPoint >= 0
? TokenSubType.Decimal
: TokenSubType.Integer
));
}
public static Token Parse(Tokenizer.State state)
{
var st = GetTypeFor(state.CurrentCharacter);
if (st == TokenSubType.Unknown)
{
return(null);
}
var t = new Token(
value: state.CurrentCharacter.ToString(),
type: TokenType.Operator,
subType: st
);
state.Index += 1;
return(t);
}
public static Token Parse(Tokenizer.State state)
{
if (state.CurrentCharacter == '/')
{
int begin = state.Index;
bool inline = false;
state.Index += 1;
if (state.CurrentCharacter == '/')
{
state.Index += 1;
ParseInlineCommentary(state);
inline = true;
}
else if (state.CurrentCharacter == '*')
{
state.Index += 1;
ParseMultilineCommentary(state);
}
else
{
return(null);
}
if (state.IsErrorOccured())
{
state.Index = begin + 2;
}
return(new Token(
value: state.Code.Substring(begin, state.Index - begin),
type: TokenType.Commentary,
subType: inline
? TokenSubType.InlineCommentary
: TokenSubType.MultilineCommentary
));
}
return(null);
}
private static bool IsMatching(char c, Tokenizer.State state)
{
if (Char.IsLetter(c))
{
if (c > 127)
{
// If compiler directive name contains non-ascii letters
// it will be written as error to state
state.ErrorCode = (uint)ErrorCodes.T_InvalidCompilerDirectiveName;
state.ErrorMessage =
"Compiler directive name should consist of:\n" +
" - latin letters\n" +
" - '_' symbols";
}
return(true);
}
return(c == '_');
}
private static bool IsMatching(char c, Tokenizer.State state)
{
const string possible = @"!?@#$%,^&|*/\_+~-<=>";
if (Char.IsLetter(c) || possible.Contains(c))
{
if (c > 127)
{
// If identifier contains non-ascii letters
// it will be written as error to state
state.ErrorCode = (uint)ErrorCodes.T_InvalidIdentifier;
state.ErrorMessage =
"Identifier can contains only:\n" +
" - latin letters\n" +
$" - '{possible}' symbols";
}
return(true);
}
return(false);
}
public static Token Parse(Tokenizer.State state)
{
if (!IsMatching(state.CurrentCharacter, state))
{
return(null);
}
int begin = state.Index;
state.Index += 1;
while (IsMatching(state.CurrentCharacter, state))
{
state.Index += 1;
}
return(new Token(
value: state.Code.Substring(begin, state.Index - begin),
type: TokenType.Identifier,
subType: TokenSubType.Identifier
));
}
public static Token Parse(Tokenizer.State state)
{
if (state.CurrentCharacter == '`')
{
var begin = state.Index;
state.Index += 1;
ParseInlineCommentary(state);
if (state.IsErrorOccured())
{
state.Index = begin + 2;
}
return(new Token(
value: state.Code.Substring(begin, state.Index - begin),
type: TokenType.Commentary,
subType: TokenSubType.InlineCommentary
));
}
return(null);
}
public static Token Parse(Tokenizer.State state)
{
return(null);
}
