private void AddToken(Token.Type type) { idToken++; ListToken.Add(new Token(idToken, row, column - auxiliary.Length, type, auxiliary)); auxiliary = ""; state = 0; }
public Token(string value, Token.Type type, int startPosition, int line) { Value = value; TokenType = type; StartPosition = startPosition; Line = line; }
public void parea(Token.Type type) { if (syntacticError) { if (index < ListToken.Count - 1) { index++; preAnalysis = ListToken[index]; if (preAnalysis.TypeToken == Token.Type.SIMBOLO_PUNTO_Y_COMA) { syntacticError = false; } } } else { if (index < ListToken.Count - 1) { if (preAnalysis.TypeToken == type) { index++; preAnalysis = ListToken[index]; } else { addError(preAnalysis.Row, preAnalysis.Column, preAnalysis.toStringTypeToken, "Was expected '" + type + "'"); syntacticError = true; } } } }
void Require(out Token t, Token.Type tt) { if (!Consume(out t, tt)) { throw Expected(tt); } }
/// <summary> /// Verifies that the current token is of the expected type, and generates /// an error if not. /// </summary> /// <param name="type"></param> private void Expect(Token.Type type) { if (CrntCode != type) { SyntaxError($"Unexpected {lexer.Crnt}; expected {type}"); } }
public static void CheckToken(Token token, Token.Type type, string text) { if (token.type != type) { throw new Syntax.Exception(text, token.GetIndex(), token.GetLine()); } }
public Token(Token.Type type, string raw, int row, int col) { this.type = type; this.raw = raw; this.row = row; this.col = col; }
/// <summary> /// Parse binary operator expression /// </summary> /// <param name="type">Type of binary operation</param> /// <returns>Expression with binary operator</returns> private Func <Expression, Expression> ParseBinaryOperatorExpression(Token.Type type) { return((Expression left) => { if (PeekedToken == type) { Next(); } else { return null; } var expression = new BinaryOperatorExpression(CurrentToken) { LeftExpression = left }; Next(); expression.RightExpression = ParseExpression(); return expression; }); }
void Require(Token.Type tt) { if (!Consume(tt)) { throw Expected(tt); } }
private void AddToken(Token.Type type) { int end = current - start; if (end < 0) { end = 0; } if (current + end < 1) { return; } // delimiter redeclaration string value = script.Substring(start, end); if (value.ToLowerInvariant().Equals(DELIMITER_DECLARE)) { DelimiterDeclaration(); return; } // typical token addition tokens.Add(new Token(type, value, line, start)); }
private void Parser(Token.Type type) { if (syntacticError) { if (index < ListToken.Count - 1) { index++; preAnalysis = ListToken[index]; if (preAnalysis.TypeToken == Token.Type.SYMBOL_SEMICOLON) { syntacticError = false; } } } else { if (preAnalysis.TypeToken != Token.Type.END) { if (preAnalysis.TypeToken == type) { index++; preAnalysis = ListToken[index]; } else { AddError(preAnalysis.Row, preAnalysis.Column, preAnalysis.toStringTypeToken, "Was expected '" + type + "'"); syntacticError = true; } } } }
// Consume the expected token, panic if we dont get it private Token consume(Token.Type type, string error_msg) { if (expect(type, error_msg)) { return(advance()); } throw new ParseExceptionUnexpectedToken(error_msg); }
// Check to see if what we are expecting is correct private bool expect(Token.Type type) { if (peek().type == type) { return(true); } return(false); }
private Token CreateToken(Token.Type type, string text) { Token token = new Token(text, type); token._meta_lineNumber = _lineNumber; token._meta_index = _index; return(token); }
// Check to see if what we are expecting is correct, if not throw an error private bool expect(Token.Type type, string error_msg) { if (peek().type == type) { return(true); } return(false); }
public Token(Token.Type Kind, int StartPosition, int StartLine, int StartColumn, bool FirstOnLine) { this.Kind = Kind; this.StartPosition = StartPosition; this.StartLine = StartLine; this.StartColumn = StartColumn; this.firstOnLine = FirstOnLine; }
private Token CreateToken(Token.Type type, params char[] chars) { Token token = new Token(new string(chars), type); token._meta_lineNumber = _lineNumber; token._meta_index = _index; return(token); }
/// <summary> /// Checks to see if the current token type is the spefified type /// </summary> private bool CheckType(Token.Type type) { if (isAtEnd) { return(false); } return(Peek().type == type); }
// Consume the expected token, don't panic if we dont get it private Token consume(Token.Type type) { if (expect(type)) { return(advance()); } return(null); }
public TokenExpression(string pattern, Token.Type tokenType) { Contract.NotNull(pattern, nameof(pattern)); _tokenType = tokenType; _compiledRegEx = new Lazy <Regex>( () => new Regex("^" + pattern, RegexOptions.Compiled)); }
public Tokenizer(string input) { start = 0; end = input.Length; this.output = new LinkedList <object>(); this.input = input; this.carry = ""; carryType = Token.Type.EOS; }
public Tokenizer(string input) { start = 0; end = input.Length; this.output = new LinkedList<object>(); this.input = input; this.carry = ""; carryType = Token.Type.EOS; }
/// <summary> /// Performs actions on a new token creating. /// </summary> /// <returns>A new created token.</returns> private Token OnNewToken(Token.Type tokenType) { state = State.None; var value = tokenValue.ToString(); tokenValue.Clear(); return(new Token(value, tokenType, tokenStartPosition, currentLine)); }
private void addOperatorToken(char ch, Token.Type type) { if (_curToken.Length > 0) { _tokens.Add(new Token(_curToken, Token.Type.Data)); _curToken = ""; } _tokens.Add(new Token(ch.ToString(), type)); }
/// <summary> /// Checks whether the current token has the given type or not. /// </summary> /// <param name="value">The comparing type.</param> public bool CurrentTokenTypeIs(Token.Type type) { if (IsFinished) { return(false); } return(CurrentOrLast.TokenType == type); }
/// <summary> /// 태그를 더함 /// </summary> private static void AddTag(List <Token> tokens, string value) { if (value.Length < 3 || value[0] != '<' || value[value.Length - 1] != '>') { return; } Token.Type type = Token.Type.text; if (value.Contains("<b>")) { type = Token.Type.boldStart; } else if (value.Contains("</b>")) { type = Token.Type.boldEnd; } else if (value.Contains("<i>")) { type = Token.Type.italicStart; } else if (value.Contains("</i>")) { type = Token.Type.italicEnd; } else if (value.Contains("<size=")) { type = Token.Type.sizeStart; } else if (value.Contains("</size>")) { type = Token.Type.sizeEnd; } else if (value.Contains("<color=")) { type = Token.Type.colorStart; value = value.Replace("'", string.Empty); if (!value.Contains("#")) { value = StringToHexValue(value); } else if ((value.Substring(8, value.Length - 9)).Length == 6) { value = SetAlphaValue(value); } } else if (value.Contains("</color>")) { type = Token.Type.colorEnd; } tokens.Add(new Token(type, value)); }
public MatchableElement( Token.Type type, ElementFactory factory, string name ) { TokenType = type; this.factory = factory; this.name = name; }
bool Pat(string s, Token.Type tt) { if (str.StartsWith(s)) { result.Add(new Token(tt, s)); str = str.Substring(s.Length); return(true); } return(false); }
private void SetState(State state, Token.Type type, bool updateLocation) { activeState = state; token.TypeId = type; token.RawValue += ((char)stream.Symbol).ToString(); if (updateLocation) { UpdateTokenLocation(); } }
bool Consume(Token.Type tt) { var res = Is(tt); if (res) { ++pos; } return(res); }
public ExpressionParser(SharedParseData parseData, Token.Type terminator, SymbolTable scope, DefineMode defineMode, int end) { contextStack.Push(new Context(parseData.ast.Count, Context.Kind.STATEMENT)); this.terminator = terminator; this.parseData = parseData; this.defineMode = defineMode; this.canDefine = true; this.scope = scope; this.end = end; }
public Token(string lexema, Token.Type type) { this.type = type; this.lexema = lexema; }
private Token Next() { type = Token.Type.NONE; val = ""; while (IsWhite(buf.Peek())) { buf.Read(); } line = buf.GetLine(); pos = buf.getPos(); int ch = buf.Peek(); if (ch == Buffer.EOF) { type = Token.Type.EOF; } else if (IsDecimal(ch)) { GetScalar(); } else if (IsAlpha(ch)) { GetIdentificator(); Object keyword = Token.Terms[val]; type = keyword == null? Token.Type.IDENTIFICATOR: (Token.Type)keyword; } else if (ch == '.') { type = Token.Type.CONST_DOUBLE; GetFloat(); } else if (IsSeparator(ch)) { type = Token.Type.SEPARATOR; ReadInValue(); } else if (IsOperator(ch)) { type = Token.Type.OPERATOR; GetOperator(); } else if (ch == '\'') { type = Token.Type.CONST_CHAR; GetChar(); } else if (ch == '\"') { type = Token.Type.CONST_STRING; GetString(); } else { throw_exception("недопустимый символ \"" + (char)ch + "\""); } return new Token(pos, line, type, val);; }
private void GetScalar() { int platform = GetInteger(); if (IsEnter(".eEpP", buf.Peek())) { GetFloat(true, platform); val = StringToDouble(val).ToString(); type = Token.Type.CONST_DOUBLE; } else { type = Token.Type.CONST_INT; } if (IsAlpha(buf.Peek())) { throw_exception("недопустимый символ \"" + (char)buf.Peek() + "\" в записи числа."); } else if ((platform == 8 && IsDecimal(buf.Peek()))) { throw_exception("недопустимая цифра \"" + (char)buf.Peek() + "\" для основания \"8\""); } }
private void GetIdentificator() { while (IsAlpha(buf.Peek()) || IsDecimal(buf.Peek())) { ReadInValue(); } if (!IsPermissed(buf.Peek())) { throw_exception("недопустимый символ \"" + (char)buf.Peek() + "\""); } type = Token.Type.IDENTIFICATOR; }
private int GetFloat(bool is_exist_integer = false, int pl = 0) { int platform = 0; if (buf.Peek() == '.') { ReadInValue(); if (IsDecimal(buf.Peek())) platform = GetInteger(pl, true); else if(!is_exist_integer) { type = Token.Type.OPERATOR; if (buf.Peek() == '.') { ReadInValue(); if (buf.Peek() != '.') { throw_exception("необходима \".\""); } ReadInValue(); } return 0; } } else if (!is_exist_integer) { throw_exception("необходима \".\"");//необходима . } if (IsEnter("EepP", buf.Peek())) { if ((platform == 8 || platform == 16) && (!IsEnter("Pp", buf.Peek()))) { throw_exception("необходимо \"P\" для основания \"" + platform + "\"");// ожидалось p } else if (platform == 10 && (!IsEnter("Ee", buf.Peek()))) { throw_exception("необходимо \"E\" для основания \"10\"");// ожидалось E } val += 'E'; buf.Read(); if (IsEnter("+-", buf.Peek())) { ReadInValue(); } if (!IsDecimal(buf.Peek())) { throw_exception("требуется экспоненциальное значение");//полсе Е должна быть цифра } GetInteger(platform, true); } if (!is_exist_integer) val = StringToDouble(val).ToString(); if (IsAlpha(buf.Peek())) throw_exception("неверная запись числа"); return platform; }
public Variable(string value) { this.tokenType = Token.Type.Identifier; this.value = value; }
public TokenExpectation(int level, Token.Type tokenType) { Level = level; TokenType = tokenType; }
public LinkedList<object> Tokenize() { while (start < end) { char cur = input[start]; //number literals if (cur == '0' || cur == '1' || cur == '2' || cur == '3' || cur == '4' || cur == '5' || cur == '6' || cur == '7' || cur == '8' || cur == '9') { //if there is no carry - that means that this is an integer if (carryType == Token.Type.EOS) carryType = Token.Type.Integer; } switch (cur) { //saperators case ' ': BreakCarry(); break; case ',': BreakCarry(); break; case '\t': BreakCarry(); break; case '\n': BreakCarry(); break; //full stop saperator case '.': //if the point is comming after idHead that is mean that you got idtail after this token if (carryType == Token.Type.IdHead) BreakCarry(Token.Type.IdTail); //if you are now carrying tail - that means that there is more tail after him else if (carryType == Token.Type.IdTail) BreakCarry(Token.Type.IdTail); //it may be inside of an integer-then it converts to doublee. else if (carryType == Token.Type.Integer) { carryType = Token.Type.Double; carry += '.'; } else throw new Exception("You cannot place '.' after " + carryType.ToString()); break; //operators case ';': BreakCarry(); output.AddLast(new Token(";",Token.Type.Operator)); break; case '+': BreakCarry(); output.AddLast(new Token("+", Token.Type.Operator)); break; case '-': BreakCarry(); output.AddLast(new Token("-", Token.Type.Operator)); break; case '*': BreakCarry(); output.AddLast(new Token("*", Token.Type.Operator)); break; case '/': BreakCarry(); output.AddLast(new Token("/", Token.Type.Operator)); break; case '{': BreakCarry(); output.AddLast(new Token("{", Token.Type.Operator)); break; case '}': BreakCarry(); output.AddLast(new Token("}", Token.Type.Operator)); break; //indexer case '[': Next(); BreakCarry(Token.Type.Indexer); while (input[start] != ']') { carry += input[start]; Next(); } BreakCarry(); break; //string literals case '\"': BreakCarry(Token.Type.String); Next(); while (input[start] != '\"') { //if it is an escape charecter. if (input[start] == '\\') { //TODO: escape charecter extantion for hex dec and other stuff. switch (LookAhead()) { case 'n': carry += '\n'; break; case 't': carry += '\t'; break; case '\"': carry += '\"'; break; case '\'': carry += '\''; break; default: throw new Exception("unrecognized charecter after escape charecter"); } Next(); } else { carry += input[start]; } Next(); } BreakCarry(); break; //escape charecters //function call case '<': BreakCarry(); if(LookAhead()=='-') Next(); output.AddLast(new Token("<-", Token.Type.Operator)); BreakCarry(); break; //default default: if(carryType==Token.Type.EOS) carryType=Token.Type.IdHead; carry+=cur; break; } Next(); } BreakCarry(); return output; }
public static LinkedList<Token> TokenizeOld(string input,int start, int end) { LinkedList<Token> output=new LinkedList<Token>(); string carry=""; //end of stream carry is an emty carry. Token.Type carryType= Token.Type.EOS; while (start < end) { switch(input[start]){ //escape charecter case '[': if (carryType == Token.Type.String) carry += '['; else if (carryType != Token.Type.EOS) { output.AddLast(new Token(carry, carryType)); carry = ""; carryType = Token.Type.EOS; } string carry2 = ""; while (start < end) { start++; if (input[start] != ']') carry2 += input[start]; else { output.AddLast(new Token(carry2, Token.Type.Indexer)); break; } } break; case '\\': if (carryType != Token.Type.String) throw new Exception("escape char outside of a string defention"); start++; if (start >= end) throw new Exception("end of string toosoon"); switch(input[start]){ case 'n': carry += '\n'; break; case 't': carryType += '\t'; break; case '\"': carryType += '\"'; break; case '\'': carryType += '\''; break; default: throw new Exception("unrecognized charecter after escape charecter"); } break; case '\n': if (carryType != Token.Type.String) { if (carryType == Token.Type.FunctionCall && ((carry == "") || (carry == "<"))) { //ignore uneeded spaces } else { if (carryType != Token.Type.EOS) { output.AddLast(new Token(carry, carryType)); carry = ""; carryType = Token.Type.EOS; } } } break; case ' ': if (carryType != Token.Type.String) { if (carryType == Token.Type.FunctionCall && ((carry == "") || (carry == "<"))) { //ignore uneeded spaces } else { if (carryType != Token.Type.EOS) { output.AddLast(new Token(carry, carryType)); carry = ""; carryType = Token.Type.EOS; } } } else carry += ' '; break; case '\"': if (carryType == Token.Type.String) { output.AddLast(new Token(carry, carryType)); carryType = Token.Type.EOS; } else if (carryType == Token.Type.EOS) { carryType = Token.Type.String; } else throw new Exception("invalid operator placement"); break; case '.': switch(carryType){ case Token.Type.String: carry += '.'; break; case Token.Type.Integer: carryType=Token.Type.Double; carry+='.'; break; case Token.Type.IdHead: output.AddLast(new Token(carry, carryType)); carryType=Token.Type.IdTail; carry=""; break; case Token.Type.IdTail: output.AddLast(new Token(carry, carryType)); carryType=Token.Type.IdTail; carry=""; break; default: throw new Exception("invalid dot position"); } break; case '0': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='0'; break; case '1': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='1'; break; case '2': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='2'; break; case '3': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='3'; break; case '4': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='4'; break; case '5': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='5'; break; case '6': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='6'; break; case '7': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='7'; break; case '8': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='8'; break; case '9': if(carryType==Token.Type.EOS) carryType=Token.Type.Integer; carry+='9'; break; case '<': if (carryType != Token.Type.EOS) { if (carryType == Token.Type.String) carry += "<"; else throw new Exception("invalid placement"); } else { carryType = Token.Type.FunctionCall; carry = "<"; } break; case '-': if(carryType==Token.Type.EOS) output.AddLast(new Token("-", Token.Type.Operator)); else if(carryType == Token.Type.FunctionCall){ if(carry!="<") throw new Exception("invalid placement"); carry=""; } else throw new Exception("invalid placement"); break; case '{': if (carryType == Token.Type.String) carry += '{'; else if (carryType != Token.Type.EOS) { output.AddLast(new Token(carry, carryType)); carry = ""; carryType = Token.Type.EOS; } output.AddLast(new Token("{", Token.Type.Operator)); break; case '}': if (carryType == Token.Type.String) carry += '}'; else if (carryType != Token.Type.EOS) { output.AddLast(new Token(carry, carryType)); carry = ""; carryType = Token.Type.EOS; } output.AddLast(new Token("}", Token.Type.Operator)); break; case '+': if(carryType==Token.Type.EOS) output.AddLast(new Token("+", Token.Type.Operator)); else throw new Exception("invalid placement"); break; case '*': if(carryType==Token.Type.EOS) output.AddLast(new Token("*", Token.Type.Operator)); else throw new Exception("invalid placement"); break; case ';': if (carryType == Token.Type.String) carry += ';'; else if (carryType != Token.Type.EOS) { output.AddLast(new Token(carry, carryType)); carry = ""; carryType = Token.Type.EOS; } output.AddLast(new Token(";", Token.Type.Operator)); break; default: if (carryType == Token.Type.EOS) carryType = Token.Type.IdHead; if (carryType == Token.Type.FunctionCall && carry == "<") { carry = ""; } carry+=input[start]; break; } start++; } if (carry != "") { output.AddLast(new Token(carry, carryType)); } output.AddLast(new Token("$", Token.Type.EOS)); return output; }
void BreakCarry() { if (carryType == Token.Type.EOS) return; if (carryType == Token.Type.IdTail) carryType = Token.Type.IdEnd; if (carryType == Token.Type.IdHead) carryType = Token.Type.IdSingle; output.AddLast(new Token(carry, carryType)); carry = ""; carryType = Token.Type.EOS; }
void BreakCarry(Token.Type with) { if (carryType == Token.Type.EOS) { carryType = with; return; } if (carryType == Token.Type.IdTail && with!=Token.Type.IdTail) carryType = Token.Type.IdEnd; if (carryType == Token.Type.IdHead && with != Token.Type.IdTail) carryType = Token.Type.IdSingle; output.AddLast(new Token(carry, carryType)); carry = ""; carryType = with; }
public Variable(Token t) { tokenType = t.tokentype; value = t.getValue(); }