public static List <Line> Parse(List <string> codeLines, int currentLineOffset = 0, bool isNestedInFunction = false, bool isNestedInConditional = false) { Stopwatch sw = new Stopwatch(); sw.Start(); List <Line> lines = new List <Line>(); for (int i = 0; i < codeLines.Count; i++) { Interpreter.CurrentLine = i + 1 + currentLineOffset; while (codeLines[i].IndexOf(" ") == 0 || codeLines[i].IndexOf(" ") == 0) { codeLines[i] = codeLines[i].Remove(0, 1); } string safeLine = codeLines[i]; if (!isNestedInConditional && !isNestedInFunction) // literals are already encoded { safeLine = EncodeLiterals(safeLine); } while (safeLine.IndexOf(Syntax.Identifiers.Comment) != -1) { int pos = safeLine.IndexOf(Syntax.Identifiers.Comment); safeLine = safeLine.Remove(pos, safeLine.Length - pos); } // get rid of all comments and anything after it (but it cannot be enclosed in quotes or apostrophes) if (string.IsNullOrWhiteSpace(safeLine)) { lines.Add(new Line(new List <Token>())); continue; } // no need to parse or evaluate empty line lines.Add(new Line(ParseLine(safeLine))); Line currentLine = lines[lines.Count - 1]; if (!isNestedInFunction && Interpreter.Debug && currentLine.Count > 0) { Console.WriteLine("\n(" + Interpreter.CurrentFile + ") Line " + (i + 1) + ": "); foreach (var token in currentLine) { Console.WriteLine(token.type.ToString() + " - " + token.identifier.Replace("\n", @"\n")); } } // We must handle Define function usages and FastStruct definitions before execution to avoid bad lexing errors if (Interpreter.Functions.FunctionExists("Define(") && currentLine.Count > 0 && currentLine[0].type == TokenType.FunctionCall && currentLine[0].identifier.IndexOf("Define(") == 0) { Interpreter.Functions["Define("].Invoke(currentLine[0].identifier); lines[lines.Count - 1] = new Line(new List <Token>()); // line was already handled, clear line } else if (currentLine.Count == 1 && currentLine[0].type == TokenType.FastStruct) { var @struct = lines[i][0].identifier; string name = @struct.Substring(0, @struct.IndexOf("<")); @struct = @struct.Substring(@struct.IndexOf("<")); Interpreter.Structs.Create(new FastStruct(name, FastStruct.ParseDefinitionParams(@struct))); lines[lines.Count - 1] = new Line(new List <Token>()); // line was already handled, clear line } } sw.Stop(); if (Interpreter.Debug) { Console.WriteLine("Parsing " + (Interpreter.CurrentFile != "" ? "'" + Interpreter.CurrentFile + "'" : codeLines.Count + " lines") + " took " + sw.ElapsedMilliseconds + " ms"); } return(lines); }
public static Token ParseToken(string token, bool @throw = true, int index = -1) { while (token.Length > 0 && token[0] == ' ') { token = token.Remove(0); } if (token.Length > 0) { if (Interpreter.Definitions.ContainsKey(token)) { token = Interpreter.Definitions[token].To; } if (Utils.IsHexadecimal(token)) { token = long.Parse(token.Substring(2), NumberStyles.HexNumber).ToString(); } Type type = Type.GetType(token, false); if (type != null) { return(new Token(TokenType.TypeIdentifier, token, index)); } switch (token) { case "true": case "false": return(new Token(TokenType.BooleanLiteral, token, index)); case Syntax.Statements.Return: return(new Token(TokenType.ReturnStatement, token, index)); case Syntax.Operators.Or: return(new Token(TokenType.OrOperator, token, index)); case Syntax.Operators.BitwiseOR: return(new Token(TokenType.BitwiseOROperator, token, index)); case Syntax.Operators.BitwiseAND: return(new Token(TokenType.BitwiseANDOperator, token, index)); case Syntax.Comparators.IsEqual: return(new Token(TokenType.IsEqualOperator, token, index)); case Syntax.Comparators.IsNotEqual: return(new Token(TokenType.IsNotEqualOperator, token, index)); case Syntax.Comparators.IsGreaterThan: return(new Token(TokenType.IsGreaterThanOperator, token, index)); case Syntax.Comparators.IsLessThan: return(new Token(TokenType.IsLessThanOperator, token, index)); case Syntax.Comparators.IsGreaterThanOrEqual: return(new Token(TokenType.IsGreaterThanOrEqualOperator, token, index)); case Syntax.Comparators.IsLessThanOrEqual: return(new Token(TokenType.IsLessThanOrEqualOperator, token, index)); case Syntax.Operators.Add: return(new Token(TokenType.AddOperator, token, index)); case Syntax.Operators.And: return(new Token(TokenType.AndOperator, token, index)); case Syntax.Operators.Subtract: return(new Token(TokenType.SubtractOperator, token, index)); case Syntax.Operators.Multiply: return(new Token(TokenType.MultiplyOperator, token, index)); case Syntax.Operators.Divide: return(new Token(TokenType.DivideOperator, token, index)); case Syntax.Operators.Modulo: return(new Token(TokenType.ModuloOperator, token, index)); case Syntax.Operators.Equal: return(new Token(TokenType.EqualOperator, token, index)); case "''": return(new Token(TokenType.CharacterLiteral, "", index)); // support empty Characters ('') case "{": return(new Token(TokenType.StartBracket, token, index)); case "}": return(new Token(TokenType.EndBracket, token, index)); case "(": case ")": return(new Token(TokenType.Parenthesis, token, index)); case Syntax.Conditionals.Else: return(new Token(TokenType.ElseConditional, token, index)); } if (token[0] == '\"' && token[token.Length - 1] == '\"' && token.OccurrencesOf("\"") == 2) { return(new Token(TokenType.StringLiteral, token.Substring(1, token.Length - 2).FromBase64().Replace("\\\"", "\""), index)); } else if (token[0] == '\'' && token[token.Length - 1] == '\'') // we can't ensure the length here because the contents are encoded { return(new Token(TokenType.CharacterLiteral, token.Substring(1, token.Length - 2).FromBase64(), index)); } else if (token.IndexOf(Syntax.Conditionals.If + "(") == 0 && token[token.Length - 1] == ')') { return(new Token(TokenType.IfConditional, token, index)); } else if (token.IndexOf(Syntax.Conditionals.While + "(") == 0 && token[token.Length - 1] == ')') { return(new Token(TokenType.WhileConditional, token, index)); } else if (token.IndexOf(Syntax.Conditionals.Loop + "(") == 0 && token[token.Length - 1] == ')') { return(new Token(TokenType.LoopConditional, token, index)); } else if (Utils.IsNumber(token)) { try { if (token.OccurrencesOf(".") == 1) { Convert.ToDecimal(token); return(new Token(TokenType.DecimalLiteral, token, index)); } else { long number = Convert.ToInt64(token); if (number <= int.MaxValue && number >= int.MinValue) { return(new Token(TokenType.IntegerLiteral, token, index)); } else { return(new Token(TokenType.Integer64Literal, token, index)); } } } catch (FormatException) { throw new BaseException("Number literal is in a invalid format"); } catch (OverflowException) { throw new BaseException("Number literal is either too large or small to be parsed"); } } else if (FunctionChain.IsChain(token)) { return(new Token(TokenType.FunctionChain, token, index)); } else if (BaseFunction.IsFunctionCall(token)) { return(new Token(TokenType.FunctionCall, token, index)); } else if (FastStruct.IsFastStruct(token)) { return(new Token(TokenType.FastStruct, token, index)); } else if (Utils.IsValidObjectName(token) || FastStruct.IsValidMemberAccessor(token)) { return(new Token(TokenType.VariableIdentifier, token, index)); } } if (@throw) { throw new BaseException("Unknown token: " + token); } return(null); }