public static Token Parse(Tokenizer.State state) { if (!Char.IsWhiteSpace(state.CurrentCharacter)) { return(null); } var c = state.CurrentCharacter; var isNewLine = c == '\n'; if (isNewLine) { state.Line += 1; state.LineBegin = state.Index + 1; } var token = new Token( value: c.ToString(), type: TokenType.Whitespace, subType: isNewLine ? TokenSubType.NewLine : TokenSubType.Space ); state.Index += 1; return(token); }
private static Token ParseNormalCharacter(Tokenizer.State state) { switch (state.CurrentCharacter) { case ' ': case '\t': case '\n': case '\r': state.ErrorCode = (uint)ErrorCodes.T_MisleadingCharacter; state.ErrorMessage = "Character definition is misleading\n" + "Whitespace symbols should be written using escape sequences:\n" + " \\s - space\n" + " \\t - tabulation\n" + " \\n - new line\n" + " \\r - carriage return"; break; case '\0': state.ErrorCode = (uint)ErrorCodes.T_UnexpectedEndOfFile; state.ErrorMessage = "Character is expected but 'End of file' was found instead"; break; default: break; } return(GenerateTokenFor(state.CurrentCharacter, state)); }
public static Token Parse(Tokenizer.State state) { if (state.CurrentCharacter != '#') { return(null); } int index = state.Index; state.Index += 1; while (IsMatching(state.CurrentCharacter, state)) { state.Index += 1; } int length = state.Index - index; if (length == 1) { state.ErrorCode = (uint)ErrorCodes.T_CompilerDirectiveNameIsNotStated; state.ErrorMessage = "Compiler directive name is not stated"; } return(new Token( value: state.Code.Substring(index, length), type: TokenType.CompilerDirective, subType: TokenSubType.CompilerDirective )); }
public static Token Parse(Tokenizer.State state) { if (state.CurrentCharacter != '"') { return(null); } var begin = state.Index + 1; do { state.Index += 1; while (state.CurrentCharacter == '\\') { state.Index += 2; } } while (!"\"\0".Contains(state.CurrentCharacter)); if (state.CurrentCharacter == '\0') { state.Index -= 1; state.ErrorCode = (uint)ErrorCodes.T_UnexpectedEndOfFile; state.ErrorMessage = "End of string was not found"; } return(new Token( value: state.Code.Substring(begin, state.Index++ - begin), type: TokenType.String, subType: TokenSubType.String )); }
private static void ParseInlineCommentary(Tokenizer.State state) { while (state.CurrentCharacter != '\0' && !"\r\n".Contains(state.CurrentCharacter)) { state.Index += 1; } }
private static void ParseMultilineCommentary(Tokenizer.State state) { while (true) { if (state.CurrentCharacter == '\0') { state.ErrorCode = (uint)ErrorCodes.T_UnexpectedEndOfFile; state.ErrorMessage = "Commentary was not closed"; return; } if (state.CurrentCharacter == '*') { state.Index += 1; if (state.CurrentCharacter == '/') { state.Index += 1; return; } } if (state.CurrentCharacter == '\n') { state.Line += 1; state.LineBegin = state.Index + 1; } state.Index += 1; } }
private static bool TryParse (Tokenizer.State state, Func <Tokenizer.State, Token> parser, out Token token) { state.Save(); Tokenizer.State stateCopy = new Tokenizer.State(state); token = parser(state); if (state.IsErrorOccured()) { state.Drop(); return(true); } else if (token != null) { token.Index = stateCopy.Index; token.Line = stateCopy.Line; token.Position = stateCopy.Position; state.Drop(); return(true); } state.Restore(); return(false); }
private static bool IsMatching(Tokenizer.State state, ref int decimalPoint) { if (Char.IsDigit(state.CurrentCharacter)) { return(true); } return(IsMatchingDecimal(state, ref decimalPoint)); }
private static Token GenerateTokenFor(char character, Tokenizer.State state) { state.Index += 1; return(new Token( value: character.ToString(), type: TokenType.Character, subType: TokenSubType.Character )); }
private static bool IsMatchingDecimal(Tokenizer.State state, ref int decimalPoint) { if (state.CurrentCharacter != '.' || decimalPoint >= 0) { return(false); } decimalPoint = state.Index; return(true); }
public static Token Parse(Tokenizer.State state) { foreach (Func <Tokenizer.State, Token> parser in parsers) { if (TryParse(state, parser, out Token token)) { return(token); } } return(null); }
public static Token Parse(Tokenizer.State state) { foreach (var parser in parsers) { Token token; if (TryParse(state, parser, out token)) { return(token); } } return(null); }
private static void ParseNumber(Tokenizer.State state, out int decimalPoint) { decimalPoint = -2; while (IsMatching(state, ref decimalPoint)) { state.Index += 1; } if (decimalPoint == state.Index - 1) { decimalPoint = -2; state.Index -= 1; } }
public static Token Parse(Tokenizer.State state) { if (state.CurrentCharacter != '\'') { return(null); } state.Index += 1; if (state.CurrentCharacter == '\\') { state.Index += 1; return(ParseSpecialCharacter(state)); } return(ParseNormalCharacter(state)); }
private static Token ParseSpecialCharacter(Tokenizer.State state) { switch (state.CurrentCharacter) { case 'n': return(GenerateTokenFor('\n', state)); case 't': return(GenerateTokenFor('\t', state)); case 's': return(GenerateTokenFor(' ', state)); case 'r': return(GenerateTokenFor('\r', state)); default: state.ErrorCode = (uint)ErrorCodes.T_SpecialCharacterDoesNotExists; state.ErrorMessage = $"Special character identifier '{state.CurrentCharacter}' does not exists"; return(GenerateTokenFor(state.CurrentCharacter, state)); } }
public static Token Parse(Tokenizer.State state) { Tokenizer.State s = new Tokenizer.State(state); ParseNumber(state, out int decimalPoint); if (state.Index == s.Index) { return(null); } return(new Token( value: state.Code.Substring(s.Index, state.Index - s.Index), type: TokenType.Number, subType: decimalPoint >= 0 ? TokenSubType.Decimal : TokenSubType.Integer )); }
public static Token Parse(Tokenizer.State state) { var st = GetTypeFor(state.CurrentCharacter); if (st == TokenSubType.Unknown) { return(null); } var t = new Token( value: state.CurrentCharacter.ToString(), type: TokenType.Operator, subType: st ); state.Index += 1; return(t); }
public static Token Parse(Tokenizer.State state) { if (state.CurrentCharacter == '/') { int begin = state.Index; bool inline = false; state.Index += 1; if (state.CurrentCharacter == '/') { state.Index += 1; ParseInlineCommentary(state); inline = true; } else if (state.CurrentCharacter == '*') { state.Index += 1; ParseMultilineCommentary(state); } else { return(null); } if (state.IsErrorOccured()) { state.Index = begin + 2; } return(new Token( value: state.Code.Substring(begin, state.Index - begin), type: TokenType.Commentary, subType: inline ? TokenSubType.InlineCommentary : TokenSubType.MultilineCommentary )); } return(null); }
private static bool IsMatching(char c, Tokenizer.State state) { if (Char.IsLetter(c)) { if (c > 127) { // If compiler directive name contains non-ascii letters // it will be written as error to state state.ErrorCode = (uint)ErrorCodes.T_InvalidCompilerDirectiveName; state.ErrorMessage = "Compiler directive name should consist of:\n" + " - latin letters\n" + " - '_' symbols"; } return(true); } return(c == '_'); }
private static bool IsMatching(char c, Tokenizer.State state) { const string possible = @"!?@#$%,^&|*/\_+~-<=>"; if (Char.IsLetter(c) || possible.Contains(c)) { if (c > 127) { // If identifier contains non-ascii letters // it will be written as error to state state.ErrorCode = (uint)ErrorCodes.T_InvalidIdentifier; state.ErrorMessage = "Identifier can contains only:\n" + " - latin letters\n" + $" - '{possible}' symbols"; } return(true); } return(false); }
public static Token Parse(Tokenizer.State state) { if (!IsMatching(state.CurrentCharacter, state)) { return(null); } int begin = state.Index; state.Index += 1; while (IsMatching(state.CurrentCharacter, state)) { state.Index += 1; } return(new Token( value: state.Code.Substring(begin, state.Index - begin), type: TokenType.Identifier, subType: TokenSubType.Identifier )); }
public static Token Parse(Tokenizer.State state) { if (state.CurrentCharacter == '`') { var begin = state.Index; state.Index += 1; ParseInlineCommentary(state); if (state.IsErrorOccured()) { state.Index = begin + 2; } return(new Token( value: state.Code.Substring(begin, state.Index - begin), type: TokenType.Commentary, subType: TokenSubType.InlineCommentary )); } return(null); }
public static Token Parse(Tokenizer.State state) { return(null); }