public TokenWrapper(IToken token, string tokenName) { TokenName = tokenName; Token = token; Type = new TokenType {Type = token.Type}; tokenHashCode = Type.GetHashCode(); }
public Token(TokenType t, string v) { tokenType = t; value = v; }
// Parse instructions bool Pass2() { Instruction currentInstruction; Tokenizer.Token currentToken = tokenizer.GetNextToken(); if (currentToken.Type == Tokenizer.TokenType.Empty) { return(false); } while (currentToken.Type != Tokenizer.TokenType.EOF && currentToken.Type != Tokenizer.TokenType.Unknown) { // =================================================================== // Skip end of lines if (currentToken.Type == Tokenizer.TokenType.EOL) { currentToken = tokenizer.GetNextToken(); } // =================================================================== // Skip variables declaration else if (currentToken.Type == Tokenizer.TokenType.Rsvd_Var) { currentToken = tokenizer.GetNextToken(); // Skip the VAR reserved word currentToken = tokenizer.GetNextToken(); // Skip VAR's identifier } // =================================================================== // Parse instructions and labels else if (currentToken.Type == Tokenizer.TokenType.Ident) { string ident = currentToken.Lexeme; currentToken = tokenizer.GetNextToken(); // =================================================================== // Is it a label? Skip it if (currentToken.Type == Tokenizer.TokenType.Colon) { currentToken = tokenizer.GetNextToken(); } // =================================================================== // It's an instruction else { InstrDecl instr; if (!tables.GetInstrLookUp(ident, out instr)) { logger.Log("Parser Error: syntax error"); return(false); } currentInstruction = new Instruction(); currentInstruction.OpCode = instr.OpCode; if (instr.ParamsCount > 0) { currentInstruction.Values = new Value[instr.ParamsCount]; } // =================================================================== // Parse params for (int i = 0; i < instr.ParamsCount; i++) { // We have to skip the ',' if (i > 0) { currentToken = tokenizer.GetNextToken(); if (currentToken.Type != Tokenizer.TokenType.Comma) { logger.Log("Parser Error: Comma expected"); return(false); } currentToken = tokenizer.GetNextToken(); } Tokenizer.TokenType t = currentToken.Type; int flags = instr.ParamsFlags[i]; // =================================================================== // Is it a variable or label? if (t == Tokenizer.TokenType.Ident) { if ((flags & OpFlags.MemIdx) != 0) { VarDecl varDecl; if (!tables.GetVarByIdent(currentToken.Lexeme, out varDecl)) { logger.Log("Parser Error: Variable doesn't exist"); return(false); } currentInstruction.Values[i].Type = OpType.MemIdx; currentInstruction.Values[i].StackIndex = varDecl.Idx; } else if ((flags & OpFlags.InstrIdx) != 0) { LabelDecl label; if (!tables.GetLabelByName(currentToken.Lexeme, out label)) { logger.Log("Parser Error: Label doesn't exist"); return(false); } currentInstruction.Values[i].Type = OpType.InstrIdx; currentInstruction.Values[i].InstrIndex = label.Idx; } else if ((flags & OpFlags.HostAPICallIdx) != 0) { // TODO: host api calls } } // =================================================================== // Is it a literal value? else if (t == Tokenizer.TokenType.Number || t == Tokenizer.TokenType.String) { if ((flags & OpFlags.Literal) == 0) { // TODO: Log error: doesn´t allow literals return(false); } if (t == Tokenizer.TokenType.Number) { if (StringUtil.IsStringFloat(currentToken.Lexeme)) { float val = 0; currentInstruction.Values[i].Type = OpType.Float; if (float.TryParse(currentToken.Lexeme, out val)) { currentInstruction.Values[i].FloatLiteral = val; } else { // TODO: log error: error parsing float value return(false); } } else if (StringUtil.IsStringInt(currentToken.Lexeme)) { int val = 0; currentInstruction.Values[i].Type = OpType.Int; if (int.TryParse(currentToken.Lexeme, out val)) { currentInstruction.Values[i].IntLiteral = val; } else { // TODO: log error: error parsing int value return(false); } } else if (StringUtil.IsStringHex(currentToken.Lexeme)) { currentInstruction.Values[i].Type = OpType.Int; currentInstruction.Values[i].IntLiteral = StringUtil.StrHexToInt(currentToken.Lexeme); } else { // TODO: log error: error parsing literal value return(false); } } else { currentInstruction.Values[i].Type = OpType.String; currentInstruction.Values[i].StringLiteral = currentToken.Lexeme; } } else { // TODO: log error: unexpected token return(false); } } // Add the instruction to the stream tables.AddInstrToStream(currentInstruction); // Skip to next token currentToken = tokenizer.GetNextToken(); } } else { // TODO: log error: unexpected token return(false); } } return(true); }
// Parse instructions bool Pass2() { scope = -1; Instruction currentInstruction; Tokenizer.Token currentToken = tokenizer.GetNextToken(); if (currentToken.Type == Tokenizer.TokenType.Empty) { return(false); } while (currentToken.Type != Tokenizer.TokenType.EOF && currentToken.Type != Tokenizer.TokenType.Unknown) { // =================================================================== // Skip end of lines if (currentToken.Type == Tokenizer.TokenType.EOL) { currentToken = tokenizer.GetNextToken(); } // =================================================================== // Skip variables declaration else if (currentToken.Type == Tokenizer.TokenType.Rsvd_Var) { currentToken = tokenizer.GetNextToken(); // Skip the VAR reserved word currentToken = tokenizer.GetNextToken(); // Skip VAR's identifier } else if (currentToken.Type == Tokenizer.TokenType.Rsvd_Arg) { currentToken = tokenizer.GetNextToken(); // Skip the ARG reserved word currentToken = tokenizer.GetNextToken(); // Skip ARG's identifier } else if (currentToken.Type == Tokenizer.TokenType.Rsvd_Func) { currentToken = tokenizer.GetNextToken(); // Skip the FUNC reserved word FuncDecl func; if (!tables.GetFuncByIdent(currentToken.Lexeme, out func)) { // Error imposible return(false); } scope = func.scope; currentToken = tokenizer.GetNextToken(); // Skip FUNC`s identifier } else if (currentToken.Type == Tokenizer.TokenType.Rsvd_EndFunc) { scope = -1; currentToken = tokenizer.GetNextToken(); // Skip the ENDFUNC reserved word } // =================================================================== // Parse instructions and labels else if (currentToken.Type == Tokenizer.TokenType.Ident) { string ident = currentToken.Lexeme; currentToken = tokenizer.GetNextToken(); // =================================================================== // Is it a label? Skip it if (currentToken.Type == Tokenizer.TokenType.Colon) { currentToken = tokenizer.GetNextToken(); } // =================================================================== // It's an instruction else { InstrDecl instr; if (!tables.GetInstrLookUp(ident, out instr)) { errorHandler.ParserLogError("Syntax Error"); return(false); } currentInstruction = new Instruction(); currentInstruction.OpCode = instr.OpCode; if (instr.ParamsCount > 0) { currentInstruction.Values = new Value[instr.ParamsCount]; } // =================================================================== // Parse params for (int i = 0; i < instr.ParamsCount; i++) { // We have to skip the ',' if (i > 0) { currentToken = tokenizer.GetNextToken(); if (currentToken.Type != Tokenizer.TokenType.Comma) { errorHandler.ParserLogError("Comma Expected"); return(false); } currentToken = tokenizer.GetNextToken(); } Tokenizer.TokenType t = currentToken.Type; int flags = instr.ParamsFlags[i]; // =================================================================== // Is it a variable or label? if (t == Tokenizer.TokenType.Ident) { if ((flags & OpFlags.MemIdx) != 0) { VarDecl varDecl; if (!tables.GetVarByIdent(currentToken.Lexeme, out varDecl, scope)) { errorHandler.ParserLogError("Variable Doesn´t Exist"); return(false); } if (varDecl.scope == -1) { currentInstruction.Values[i].Type = OpType.AbsMemIdx; } else if (varDecl.isArg) { currentInstruction.Values[i].Type = OpType.ArgMemIdx; } else { currentInstruction.Values[i].Type = OpType.RelMemIdx; } currentInstruction.Values[i].StackIndex = varDecl.Idx; } else if ((flags & OpFlags.InstrIdx) != 0) { LabelDecl label; if (!tables.GetLabelByName(currentToken.Lexeme, out label, scope)) { errorHandler.ParserLogError("Label Doesn´t Exist"); return(false); } currentInstruction.Values[i].Type = OpType.InstrIdx; currentInstruction.Values[i].InstrIndex = label.Idx; } else if ((flags & OpFlags.FuncIdx) != 0) { FuncDecl func; if (!tables.GetFuncByIdent(currentToken.Lexeme, out func)) { errorHandler.ParserLogError("Function Doesn´t Exist"); return(false); } currentInstruction.Values[i].Type = OpType.FuncIdx; currentInstruction.Values[i].FunctionIndex = func.scope; } else if ((flags & OpFlags.HostAPICallIdx) != 0) { currentInstruction.Values[i].Type = OpType.HostAPICallString; currentInstruction.Values[i].StringLiteral = currentToken.Lexeme; // TODO: host api calls } } // =================================================================== // Is it a literal value? else if (t == Tokenizer.TokenType.Number || t == Tokenizer.TokenType.String) { if ((flags & OpFlags.Literal) == 0) { errorHandler.ParserLogError("Doesn´t Allow Literals"); return(false); } if (t == Tokenizer.TokenType.Number) { if (StringUtil.IsStringFloat(currentToken.Lexeme)) { float val = 0; currentInstruction.Values[i].Type = OpType.Float; if (float.TryParse(currentToken.Lexeme, out val)) { currentInstruction.Values[i].FloatLiteral = val; } else { errorHandler.ParserLogError("Error Parsing Float Value"); return(false); } } else if (StringUtil.IsStringInt(currentToken.Lexeme)) { int val = 0; currentInstruction.Values[i].Type = OpType.Int; if (int.TryParse(currentToken.Lexeme, out val)) { currentInstruction.Values[i].IntLiteral = val; } else { errorHandler.ParserLogError("Error Parsing Int Value"); return(false); } } else if (StringUtil.IsStringHex(currentToken.Lexeme)) { currentInstruction.Values[i].Type = OpType.Int; currentInstruction.Values[i].IntLiteral = StringUtil.StrHexToInt(currentToken.Lexeme); } else { errorHandler.ParserLogError("Error Parsing Literal Value"); return(false); } } else { currentInstruction.Values[i].Type = OpType.String; currentInstruction.Values[i].StringLiteral = currentToken.Lexeme; } } else { errorHandler.ParserLogError("Unexpected Token"); return(false); } } // Add the instruction to the stream tables.AddInstrToStream(currentInstruction); // Skip to next token currentToken = tokenizer.GetNextToken(); } } else { errorHandler.ParserLogError("Unexpected Token"); return(false); } } return(true); }
static string TokenTypesToString(TokenType[] token_types) { string s = ""; foreach (TokenType token_type in token_types) { if (s.Length > 0) s += ", "; s += Token.TokenTypeToLiteral(token_type); } return s; }