private void AddToken(Token.Type type)
{
idToken++;
ListToken.Add(new Token(idToken, row, column - auxiliary.Length, type, auxiliary));
auxiliary = "";
state = 0;
}
public Token(string value, Token.Type type, int startPosition, int line)
{
Value = value;
TokenType = type;
StartPosition = startPosition;
Line = line;
}
public void parea(Token.Type type)
{
if (syntacticError)
{
if (index < ListToken.Count - 1)
{
index++;
preAnalysis = ListToken[index];
if (preAnalysis.TypeToken == Token.Type.SIMBOLO_PUNTO_Y_COMA)
{
syntacticError = false;
}
}
}
else
{
if (index < ListToken.Count - 1)
{
if (preAnalysis.TypeToken == type)
{
index++;
preAnalysis = ListToken[index];
}
else
{
addError(preAnalysis.Row, preAnalysis.Column, preAnalysis.toStringTypeToken, "Was expected '" + type + "'");
syntacticError = true;
}
}
}
}
void Require(out Token t, Token.Type tt)
{
if (!Consume(out t, tt))
{
throw Expected(tt);
}
}
/// <summary>
/// Verifies that the current token is of the expected type, and generates
/// an error if not.
/// </summary>
/// <param name="type"></param>
private void Expect(Token.Type type)
{
if (CrntCode != type)
{
SyntaxError($"Unexpected {lexer.Crnt}; expected {type}");
}
}
public static void CheckToken(Token token, Token.Type type, string text)
{
if (token.type != type)
{
throw new Syntax.Exception(text, token.GetIndex(), token.GetLine());
}
}
public Token(Token.Type type, string raw, int row, int col)
{
this.type = type;
this.raw = raw;
this.row = row;
this.col = col;
}
/// <summary>
/// Parse binary operator expression
/// </summary>
/// <param name="type">Type of binary operation</param>
/// <returns>Expression with binary operator</returns>
private Func <Expression, Expression> ParseBinaryOperatorExpression(Token.Type type)
{
return((Expression left) =>
{
if (PeekedToken == type)
{
Next();
}
else
{
return null;
}
var expression = new BinaryOperatorExpression(CurrentToken)
{
LeftExpression = left
};
Next();
expression.RightExpression = ParseExpression();
return expression;
});
}
void Require(Token.Type tt)
{
if (!Consume(tt))
{
throw Expected(tt);
}
}
private void AddToken(Token.Type type)
{
int end = current - start;
if (end < 0)
{
end = 0;
}
if (current + end < 1)
{
return;
}
// delimiter redeclaration
string value = script.Substring(start, end);
if (value.ToLowerInvariant().Equals(DELIMITER_DECLARE))
{
DelimiterDeclaration();
return;
}
// typical token addition
tokens.Add(new Token(type, value, line, start));
}
private void Parser(Token.Type type)
{
if (syntacticError)
{
if (index < ListToken.Count - 1)
{
index++;
preAnalysis = ListToken[index];
if (preAnalysis.TypeToken == Token.Type.SYMBOL_SEMICOLON)
{
syntacticError = false;
}
}
}
else
{
if (preAnalysis.TypeToken != Token.Type.END)
{
if (preAnalysis.TypeToken == type)
{
index++;
preAnalysis = ListToken[index];
}
else
{
AddError(preAnalysis.Row, preAnalysis.Column, preAnalysis.toStringTypeToken, "Was expected '" + type + "'");
syntacticError = true;
}
}
}
}
// Consume the expected token, panic if we dont get it
private Token consume(Token.Type type, string error_msg)
{
if (expect(type, error_msg))
{
return(advance());
}
throw new ParseExceptionUnexpectedToken(error_msg);
}
// Check to see if what we are expecting is correct
private bool expect(Token.Type type)
{
if (peek().type == type)
{
return(true);
}
return(false);
}
private Token CreateToken(Token.Type type, string text)
{
Token token = new Token(text, type);
token._meta_lineNumber = _lineNumber;
token._meta_index = _index;
return(token);
}
// Check to see if what we are expecting is correct, if not throw an error
private bool expect(Token.Type type, string error_msg)
{
if (peek().type == type)
{
return(true);
}
return(false);
}
public Token(Token.Type Kind, int StartPosition, int StartLine, int StartColumn, bool FirstOnLine)
{
this.Kind = Kind;
this.StartPosition = StartPosition;
this.StartLine = StartLine;
this.StartColumn = StartColumn;
this.firstOnLine = FirstOnLine;
}
private Token CreateToken(Token.Type type, params char[] chars)
{
Token token = new Token(new string(chars), type);
token._meta_lineNumber = _lineNumber;
token._meta_index = _index;
return(token);
}
/// <summary>
/// Checks to see if the current token type is the spefified type
/// </summary>
private bool CheckType(Token.Type type)
{
if (isAtEnd)
{
return(false);
}
return(Peek().type == type);
}
// Consume the expected token, don't panic if we dont get it
private Token consume(Token.Type type)
{
if (expect(type))
{
return(advance());
}
return(null);
}
public TokenExpression(string pattern, Token.Type tokenType)
{
Contract.NotNull(pattern, nameof(pattern));
_tokenType = tokenType;
_compiledRegEx = new Lazy <Regex>(
() => new Regex("^" + pattern, RegexOptions.Compiled));
}
public Tokenizer(string input)
{
start = 0;
end = input.Length;
this.output = new LinkedList <object>();
this.input = input;
this.carry = "";
carryType = Token.Type.EOS;
}
public Tokenizer(string input)
{
start = 0;
end = input.Length;
this.output = new LinkedList<object>();
this.input = input;
this.carry = "";
carryType = Token.Type.EOS;
}
/// <summary>
/// Performs actions on a new token creating.
/// </summary>
/// <returns>A new created token.</returns>
private Token OnNewToken(Token.Type tokenType)
{
state = State.None;
var value = tokenValue.ToString();
tokenValue.Clear();
return(new Token(value, tokenType, tokenStartPosition, currentLine));
}
private void addOperatorToken(char ch, Token.Type type)
{
if (_curToken.Length > 0)
{
_tokens.Add(new Token(_curToken, Token.Type.Data));
_curToken = "";
}
_tokens.Add(new Token(ch.ToString(), type));
}
/// <summary>
/// Checks whether the current token has the given type or not.
/// </summary>
/// <param name="value">The comparing type.</param>
public bool CurrentTokenTypeIs(Token.Type type)
{
if (IsFinished)
{
return(false);
}
return(CurrentOrLast.TokenType == type);
}
/// <summary>
/// 태그를 더함
/// </summary>
private static void AddTag(List <Token> tokens, string value)
{
if (value.Length < 3 || value[0] != '<' || value[value.Length - 1] != '>')
{
return;
}
Token.Type type = Token.Type.text;
if (value.Contains("<b>"))
{
type = Token.Type.boldStart;
}
else if (value.Contains("</b>"))
{
type = Token.Type.boldEnd;
}
else if (value.Contains("<i>"))
{
type = Token.Type.italicStart;
}
else if (value.Contains("</i>"))
{
type = Token.Type.italicEnd;
}
else if (value.Contains("<size="))
{
type = Token.Type.sizeStart;
}
else if (value.Contains("</size>"))
{
type = Token.Type.sizeEnd;
}
else if (value.Contains("<color="))
{
type = Token.Type.colorStart;
value = value.Replace("'", string.Empty);
if (!value.Contains("#"))
{
value = StringToHexValue(value);
}
else if ((value.Substring(8, value.Length - 9)).Length == 6)
{
value = SetAlphaValue(value);
}
}
else if (value.Contains("</color>"))
{
type = Token.Type.colorEnd;
}
tokens.Add(new Token(type, value));
}
public MatchableElement(
Token.Type type,
ElementFactory factory,
string name
)
{
TokenType = type;
this.factory = factory;
this.name = name;
}
bool Pat(string s, Token.Type tt)
{
if (str.StartsWith(s))
{
result.Add(new Token(tt, s));
str = str.Substring(s.Length);
return(true);
}
return(false);
}
private void SetState(State state, Token.Type type, bool updateLocation)
{
activeState = state;
token.TypeId = type;
token.RawValue += ((char)stream.Symbol).ToString();
if (updateLocation)
{
UpdateTokenLocation();
}
}
bool Consume(Token.Type tt)
{
var res = Is(tt);
if (res)
{
++pos;
}
return(res);
}
public ExpressionParser(SharedParseData parseData, Token.Type terminator, SymbolTable scope, DefineMode defineMode, int end)
{
contextStack.Push(new Context(parseData.ast.Count, Context.Kind.STATEMENT));
this.terminator = terminator;
this.parseData = parseData;
this.defineMode = defineMode;
this.canDefine = true;
this.scope = scope;
this.end = end;
}
public Token(string lexema, Token.Type type)
{
this.type = type;
this.lexema = lexema;
}
private Token Next()
{
type = Token.Type.NONE;
val = "";
while (IsWhite(buf.Peek())) { buf.Read(); }
line = buf.GetLine();
pos = buf.getPos();
int ch = buf.Peek();
if (ch == Buffer.EOF)
{
type = Token.Type.EOF;
}
else if (IsDecimal(ch))
{
GetScalar();
}
else if (IsAlpha(ch))
{
GetIdentificator();
Object keyword = Token.Terms[val];
type = keyword == null? Token.Type.IDENTIFICATOR: (Token.Type)keyword;
}
else if (ch == '.')
{
type = Token.Type.CONST_DOUBLE;
GetFloat();
}
else if (IsSeparator(ch))
{
type = Token.Type.SEPARATOR;
ReadInValue();
}
else if (IsOperator(ch))
{
type = Token.Type.OPERATOR;
GetOperator();
}
else if (ch == '\'')
{
type = Token.Type.CONST_CHAR;
GetChar();
}
else if (ch == '\"')
{
type = Token.Type.CONST_STRING;
GetString();
}
else
{
throw_exception("недопустимый символ \"" + (char)ch + "\"");
}
return new Token(pos, line, type, val);;
}
private void GetScalar()
{
int platform = GetInteger();
if (IsEnter(".eEpP", buf.Peek()))
{
GetFloat(true, platform);
val = StringToDouble(val).ToString();
type = Token.Type.CONST_DOUBLE;
}
else
{
type = Token.Type.CONST_INT;
}
if (IsAlpha(buf.Peek()))
{
throw_exception("недопустимый символ \"" + (char)buf.Peek() + "\" в записи числа.");
}
else if ((platform == 8 && IsDecimal(buf.Peek())))
{
throw_exception("недопустимая цифра \"" + (char)buf.Peek() + "\" для основания \"8\"");
}
}
private void GetIdentificator()
{
while (IsAlpha(buf.Peek()) || IsDecimal(buf.Peek())) { ReadInValue(); }
if (!IsPermissed(buf.Peek()))
{
throw_exception("недопустимый символ \"" + (char)buf.Peek() + "\"");
}
type = Token.Type.IDENTIFICATOR;
}
private int GetFloat(bool is_exist_integer = false, int pl = 0)
{
int platform = 0;
if (buf.Peek() == '.')
{
ReadInValue();
if (IsDecimal(buf.Peek()))
platform = GetInteger(pl, true);
else if(!is_exist_integer)
{
type = Token.Type.OPERATOR;
if (buf.Peek() == '.')
{
ReadInValue();
if (buf.Peek() != '.')
{
throw_exception("необходима \".\"");
}
ReadInValue();
}
return 0;
}
}
else if (!is_exist_integer)
{
throw_exception("необходима \".\"");//необходима .
}
if (IsEnter("EepP", buf.Peek()))
{
if ((platform == 8 || platform == 16) && (!IsEnter("Pp", buf.Peek())))
{
throw_exception("необходимо \"P\" для основания \"" + platform + "\"");// ожидалось p
}
else if (platform == 10 && (!IsEnter("Ee", buf.Peek())))
{
throw_exception("необходимо \"E\" для основания \"10\"");// ожидалось E
}
val += 'E';
buf.Read();
if (IsEnter("+-", buf.Peek()))
{
ReadInValue();
}
if (!IsDecimal(buf.Peek()))
{
throw_exception("требуется экспоненциальное значение");//полсе Е должна быть цифра
}
GetInteger(platform, true);
}
if (!is_exist_integer)
val = StringToDouble(val).ToString();
if (IsAlpha(buf.Peek()))
throw_exception("неверная запись числа");
return platform;
}
public Variable(string value)
{
this.tokenType = Token.Type.Identifier;
this.value = value;
}
public TokenExpectation(int level, Token.Type tokenType)
{
Level = level;
TokenType = tokenType;
}
public LinkedList<object> Tokenize()
{
while (start < end) {
char cur = input[start];
//number literals
if (cur == '0' ||
cur == '1' ||
cur == '2' ||
cur == '3' ||
cur == '4' ||
cur == '5' ||
cur == '6' ||
cur == '7' ||
cur == '8' ||
cur == '9') {
//if there is no carry - that means that this is an integer
if (carryType == Token.Type.EOS)
carryType = Token.Type.Integer;
}
switch (cur) {
//saperators
case ' ':
BreakCarry();
break;
case ',':
BreakCarry();
break;
case '\t':
BreakCarry();
break;
case '\n':
BreakCarry();
break;
//full stop saperator
case '.':
//if the point is comming after idHead that is mean that you got idtail after this token
if (carryType == Token.Type.IdHead)
BreakCarry(Token.Type.IdTail);
//if you are now carrying tail - that means that there is more tail after him
else if (carryType == Token.Type.IdTail)
BreakCarry(Token.Type.IdTail);
//it may be inside of an integer-then it converts to doublee.
else if (carryType == Token.Type.Integer) {
carryType = Token.Type.Double;
carry += '.';
}
else
throw new Exception("You cannot place '.' after " + carryType.ToString());
break;
//operators
case ';':
BreakCarry();
output.AddLast(new Token(";",Token.Type.Operator));
break;
case '+':
BreakCarry();
output.AddLast(new Token("+", Token.Type.Operator));
break;
case '-':
BreakCarry();
output.AddLast(new Token("-", Token.Type.Operator));
break;
case '*':
BreakCarry();
output.AddLast(new Token("*", Token.Type.Operator));
break;
case '/':
BreakCarry();
output.AddLast(new Token("/", Token.Type.Operator));
break;
case '{':
BreakCarry();
output.AddLast(new Token("{", Token.Type.Operator));
break;
case '}':
BreakCarry();
output.AddLast(new Token("}", Token.Type.Operator));
break;
//indexer
case '[':
Next();
BreakCarry(Token.Type.Indexer);
while (input[start] != ']') {
carry += input[start];
Next();
}
BreakCarry();
break;
//string literals
case '\"':
BreakCarry(Token.Type.String);
Next();
while (input[start] != '\"') {
//if it is an escape charecter.
if (input[start] == '\\') {
//TODO: escape charecter extantion for hex dec and other stuff.
switch (LookAhead()) {
case 'n':
carry += '\n';
break;
case 't':
carry += '\t';
break;
case '\"':
carry += '\"';
break;
case '\'':
carry += '\'';
break;
default:
throw new Exception("unrecognized charecter after escape charecter");
}
Next();
}
else {
carry += input[start];
}
Next();
}
BreakCarry();
break;
//escape charecters
//function call
case '<':
BreakCarry();
if(LookAhead()=='-')
Next();
output.AddLast(new Token("<-", Token.Type.Operator));
BreakCarry();
break;
//default
default:
if(carryType==Token.Type.EOS)
carryType=Token.Type.IdHead;
carry+=cur;
break;
}
Next();
}
BreakCarry();
return output;
}
public static LinkedList<Token> TokenizeOld(string input,int start, int end)
{
LinkedList<Token> output=new LinkedList<Token>();
string carry="";
//end of stream carry is an emty carry.
Token.Type carryType= Token.Type.EOS;
while (start < end) {
switch(input[start]){
//escape charecter
case '[':
if (carryType == Token.Type.String)
carry += '[';
else if (carryType != Token.Type.EOS) {
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = Token.Type.EOS;
}
string carry2 = "";
while (start < end) {
start++;
if (input[start] != ']')
carry2 += input[start];
else {
output.AddLast(new Token(carry2, Token.Type.Indexer));
break;
}
}
break;
case '\\':
if (carryType != Token.Type.String)
throw new Exception("escape char outside of a string defention");
start++;
if (start >= end)
throw new Exception("end of string toosoon");
switch(input[start]){
case 'n':
carry += '\n';
break;
case 't':
carryType += '\t';
break;
case '\"':
carryType += '\"';
break;
case '\'':
carryType += '\'';
break;
default:
throw new Exception("unrecognized charecter after escape charecter");
}
break;
case '\n':
if (carryType != Token.Type.String) {
if (carryType == Token.Type.FunctionCall && ((carry == "") || (carry == "<"))) {
//ignore uneeded spaces
}
else {
if (carryType != Token.Type.EOS) {
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = Token.Type.EOS;
}
}
}
break;
case ' ':
if (carryType != Token.Type.String) {
if (carryType == Token.Type.FunctionCall && ((carry == "") || (carry == "<"))) {
//ignore uneeded spaces
}
else {
if (carryType != Token.Type.EOS) {
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = Token.Type.EOS;
}
}
}
else
carry += ' ';
break;
case '\"':
if (carryType == Token.Type.String) {
output.AddLast(new Token(carry, carryType));
carryType = Token.Type.EOS;
}
else if (carryType == Token.Type.EOS) {
carryType = Token.Type.String;
}
else
throw new Exception("invalid operator placement");
break;
case '.':
switch(carryType){
case Token.Type.String:
carry += '.';
break;
case Token.Type.Integer:
carryType=Token.Type.Double;
carry+='.';
break;
case Token.Type.IdHead:
output.AddLast(new Token(carry, carryType));
carryType=Token.Type.IdTail;
carry="";
break;
case Token.Type.IdTail:
output.AddLast(new Token(carry, carryType));
carryType=Token.Type.IdTail;
carry="";
break;
default:
throw new Exception("invalid dot position");
}
break;
case '0':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='0';
break;
case '1':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='1';
break;
case '2':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='2';
break;
case '3':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='3';
break;
case '4':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='4';
break;
case '5':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='5';
break;
case '6':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='6';
break;
case '7':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='7';
break;
case '8':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='8';
break;
case '9':
if(carryType==Token.Type.EOS)
carryType=Token.Type.Integer;
carry+='9';
break;
case '<':
if (carryType != Token.Type.EOS) {
if (carryType == Token.Type.String)
carry += "<";
else
throw new Exception("invalid placement");
}
else {
carryType = Token.Type.FunctionCall;
carry = "<";
}
break;
case '-':
if(carryType==Token.Type.EOS)
output.AddLast(new Token("-", Token.Type.Operator));
else if(carryType == Token.Type.FunctionCall){
if(carry!="<")
throw new Exception("invalid placement");
carry="";
}
else
throw new Exception("invalid placement");
break;
case '{':
if (carryType == Token.Type.String)
carry += '{';
else if (carryType != Token.Type.EOS) {
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = Token.Type.EOS;
}
output.AddLast(new Token("{", Token.Type.Operator));
break;
case '}':
if (carryType == Token.Type.String)
carry += '}';
else if (carryType != Token.Type.EOS) {
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = Token.Type.EOS;
}
output.AddLast(new Token("}", Token.Type.Operator));
break;
case '+':
if(carryType==Token.Type.EOS)
output.AddLast(new Token("+", Token.Type.Operator));
else
throw new Exception("invalid placement");
break;
case '*':
if(carryType==Token.Type.EOS)
output.AddLast(new Token("*", Token.Type.Operator));
else
throw new Exception("invalid placement");
break;
case ';':
if (carryType == Token.Type.String)
carry += ';';
else if (carryType != Token.Type.EOS) {
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = Token.Type.EOS;
}
output.AddLast(new Token(";", Token.Type.Operator));
break;
default:
if (carryType == Token.Type.EOS)
carryType = Token.Type.IdHead;
if (carryType == Token.Type.FunctionCall && carry == "<") {
carry = "";
}
carry+=input[start];
break;
}
start++;
}
if (carry != "") {
output.AddLast(new Token(carry, carryType));
}
output.AddLast(new Token("$", Token.Type.EOS));
return output;
}
void BreakCarry()
{
if (carryType == Token.Type.EOS)
return;
if (carryType == Token.Type.IdTail)
carryType = Token.Type.IdEnd;
if (carryType == Token.Type.IdHead)
carryType = Token.Type.IdSingle;
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = Token.Type.EOS;
}
void BreakCarry(Token.Type with)
{
if (carryType == Token.Type.EOS) {
carryType = with;
return;
}
if (carryType == Token.Type.IdTail && with!=Token.Type.IdTail)
carryType = Token.Type.IdEnd;
if (carryType == Token.Type.IdHead && with != Token.Type.IdTail)
carryType = Token.Type.IdSingle;
output.AddLast(new Token(carry, carryType));
carry = "";
carryType = with;
}
public Variable(Token t)
{
tokenType = t.tokentype;
value = t.getValue();
}
