private static int FindBlockLength(TokenArray tokens) { var iterator = new TokenIterator(tokens); List <TokenMode> blocks = new List <TokenMode>(); { var token = iterator.GetNext(); var end = GetBlockEndMode(token.Mode); Debug.Assert(end != null, "Block must begin with a block start"); blocks.Add(end); } while (iterator.HasNext && blocks.Count > 0) { var token = iterator.GetNext(); if (token.Mode == blocks[blocks.Count - 1]) { blocks.RemoveAt(blocks.Count - 1); } else { var end = GetBlockEndMode(token.Mode); if (end != null) { blocks.Add(end); } } } return(iterator.Iterator - tokens.Start + 1); // +1 to include the last bracket }
private static BlockLexeme GetStructureLexeme(List <Token> tokens, out CompileError error) { error = null; var iterator = new TokenIterator <Token>(tokens); var possibleLexeme = new List <Token>(); var currentParent = new BlockLexeme(null, -1); int level = 0; Token lastClosed = null; while (iterator.GetNext() != null) { if (iterator.Current.Type == TokenType.Semicolon || iterator.Current.Type == TokenType.BraceOpened || iterator.Current.Type == TokenType.BraceClosed) { currentParent.ChildLexemes.Add(new UnknownLexeme(possibleLexeme)); possibleLexeme.Clear(); switch (iterator.Current.Type) { case TokenType.BraceOpened: var oldParent = currentParent; currentParent = new BlockLexeme(oldParent, iterator.Index); oldParent.ChildLexemes.Add(currentParent); level++; break; case TokenType.BraceClosed: currentParent = (BlockLexeme)currentParent.Parent; lastClosed = iterator.Current; level--; break; } } else { possibleLexeme.Add(iterator.Current); } } if (possibleLexeme.Count != 0) { currentParent.ChildLexemes.Add(new UnknownLexeme(possibleLexeme)); } if (level != 0) { error = new CompileError(CompileErrorType.WrongCodeStructure, lastClosed); } return(currentParent); }
public static CompileError CreateAttribute(List <Token> tokens, out Attribute attribute) { attribute = null; if (tokens.Count == 0) { return(null); } var parameters = new List <Token>(); var name = tokens[1]; var info = Types.Find(p => p.Regex.IsMatch(tokens[1].StringValue)); if (info == null) { return(new CompileError(CompileErrorType.UnknownAttributeType, name)); } var iterator = new TokenIterator <Token>(tokens.Skip(3).Take(tokens.Count - 4)); while (iterator.GetNext() != null) { if (iterator.Current.Type != TokenType.ComaSign) { parameters.Add(iterator.Current); } } if (parameters.Count != info.ParametersCount && info.ParametersCount != -1) { return(new CompileError(CompileErrorType.WrongAttributeParameterCount, name)); } attribute = CreateAttributeMember(info.Type, parameters); if (attribute == null) { return(new CompileError(CompileErrorType.UnknownAttributeType, name)); } return(attribute.VerifyParameters()); }
public static Instruction Parse(TokenArray tokens) { var iterator = new TokenIterator(tokens); /* VDeclr: type, named, (equals, Statement){0..1}, semicolon * Declr: [FDeclr, VDeclr] * PDeclr: (VarDeclr, comma){1..n} * Params: (Statement, comma){1..n} * Block: [Function, Declr, Call]{1..n} * Statement: [Call, Equation], semicolon * Call: named, paren, Params, paren * FDeclr: type, named, paren, PDeclr, paren, brace, Block, brace */ var iter = Parse(ref iterator, blockRule, 0); if (iter == null) { iterator.Iterator = farthestIterator; string str = ""; for (int t = 0; t < 10 && iterator.HasNext; ++t) { str += (t > 0 ? " " : "") + iterator.GetNext().String; } Debug.WriteLine("Unable to complete parse, error at " + str); return(null); } var context = new ParseContext(); iter(context); var store = context.GetStoreAt(0); for (int o = 0; o < store.Count; ++o) { Console.WriteLine(store[o]); } return(store[0] as Instruction); while (iterator.HasNext) { var token = iterator.GetNext(); if (token.Mode == commentToken) { continue; } else if (token.Mode == typeToken) // Variable / function declaration { string type = token.String; token = iterator.GetNext(); Debug.Assert(token.Mode == nameToken, "A name must come after the type in a variable declaration, " + type); string name = token.String; Debug.Assert(iterator.HasNext, "Missing semicolon, assignment operator or open parenthesis after type declaration, " + type + " " + name); token = iterator.GetNext(); if (token.Mode == endStatementToken) { Console.WriteLine("Variable found, " + type + " " + name); } else if (token.Mode == operatorToken) { Debug.Assert(token.String == "=", "A type declaration can only have an assignment operator, " + type + " " + name); string equalTo = ""; token = iterator.GetNext(); int equalI = iterator.Iterator; while (token.Mode != endStatementToken) { equalTo += token.String; token = iterator.GetNext(); } Console.WriteLine("Variable found, " + type + " " + name + " = " + equalTo); Parse(new TokenArray(tokens.Tokens, equalI, iterator.Iterator)); } else if (token.Mode == parmStartToken) { int len = FindBlockLength(new TokenArray(tokens.Tokens, iterator.Iterator, tokens.End)); iterator.Iterator += len - 1; token = iterator.GetNext(); if (token.Mode == blockStartToken) { int bodyLen = FindBlockLength(new TokenArray(tokens.Tokens, iterator.Iterator, tokens.End)); int bodyStart = iterator.Iterator + 1; iterator.Iterator += bodyLen - 1; Console.WriteLine("Function found, " + type + " " + name + "(" + len + ") " + bodyLen); Parse(new TokenArray(tokens.Tokens, bodyStart, iterator.Iterator)); } else if (token.Mode == endStatementToken) { Console.WriteLine("Function call, " + type + " " + name + "(" + len + ") "); } } else { Debug.Assert(false, "Unknown or incomplete variable declaration, " + type + " " + name); } } else if (token.Mode == nameToken) { } else if (token.Mode == flowControlToken) { switch (token.String) { case "if": { //Console.WriteLine("If"); } break; } } } return(null); }
public static Action <ParseContext> Parse(ref TokenIterator iterator, GrammarRule condition, int level) { int c; TokenIterator startIter = iterator; string tabs = ""; for (int l = 0; l < level; ++l) { tabs += " "; } List <Action <ParseContext> > children = new List <Action <ParseContext> >(); for (c = 0; c < condition.Cardinality.To; ++c) { TokenIterator titerator = iterator; for (int g = 0; g < condition.Grammar.Length; ++g) { var grammar = condition.Grammar[g]; if (grammar is GrammarElementT) { var el = (grammar as GrammarElementT); Token token; do { if (!titerator.HasNext) { goto fail; } token = titerator.GetNext(); } while (token.Mode == commentToken); if (token.Mode != el.Mode || (el.ValidityCheck != null && !el.ValidityCheck(token))) { goto fail; } } else if (grammar is GrammarElementC) { var success = Parse(ref titerator, (grammar as GrammarElementC).Condition, level + 1); if (success == null) { goto fail; } children.Add(success); } else if (grammar is GrammarElementO) { var options = (grammar as GrammarElementO).Conditions; int goodOpt = -1; for (int o = 0; o < options.Length; ++o) { TokenIterator subIter = titerator; var success = Parse(ref subIter, options[o], level + 1); if (success != null) { titerator = subIter; children.Add(success); goodOpt = o; break; } } if (goodOpt == -1) { goto fail; } } if (iterator.Iterator > farthestIterator) { farthestIterator = iterator.Iterator; } } iterator = titerator; Console.WriteLine(tabs + condition.Name); } fail: if (c >= condition.Cardinality.From) { return delegate(ParseContext context) { using (var contextLevel = new ParseContext.Level(context, level)) { contextLevel.ClearChildren(); for (int cn = 0; cn < children.Count; ++cn) { children[cn](context); } if (condition.Build != null) { condition.Build(contextLevel, startIter); } else { contextLevel.PassThrough(); } } } } ; return(null); }
public FunctionLexeme(List <Token> tokens) : base(tokens, LexemeType.Function, true) { switch (tokens[0].Type) { case TokenType.PublicKeyword: Modifier = FunctionModifier.Public; break; case TokenType.PrivateKeyword: Modifier = FunctionModifier.Private; break; case TokenType.EntrypointKeyword: Modifier = FunctionModifier.Entrypoint; break; case TokenType.FinalizationKeyword: Modifier = FunctionModifier.Finalization; break; case TokenType.InitializationKeyword: Modifier = FunctionModifier.Initialization; break; default: Modifier = FunctionModifier.None; break; } var index = 0; if (Modifier != FunctionModifier.None) { index = 1; } index++; //Function keyword; var bracketIndex = tokens.FindIndex(p => p.Type == TokenType.BracketOpen); if (bracketIndex != 2 + (Modifier != FunctionModifier.None ? 1 : 0)) { //Has type and Name Name = tokens[bracketIndex - 1]; ReturnType = tokens[index].StringValue == "void" ? null : new List <Token> { tokens[index] }; index += 1; while (tokens[index].Type == TokenType.SquareBracketClosed || tokens[index].Type == TokenType.SquareBracketOpen) { if (ReturnType == null) { throw new CompileException(CompileErrorType.UnexpectedToken, tokens[index]); } ReturnType.Add(tokens[index]); index++; } index++; } else { Name = tokens[index]; index++; } index++; //bracket Token parameterName = null; var parameterType = new List <Token>(); Parameters = new List <LexemeFunctionParameter>(); var state = 0; var tokenIterator = new TokenIterator <Token>(tokens.Skip(index).Take(tokens.Count - 1 - index)); while (tokenIterator.GetNext() != null) { switch (state) { case 0: { if (tokenIterator.Current.Type != TokenType.Identifier) { throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current); } parameterName = tokenIterator.Current; state = 1; break; } case 1: { if (tokenIterator.Current.Type != TokenType.Colon) { throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current); } state = 2; break; } case 2: { if (tokenIterator.Current.Type != TokenType.Identifier) { throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current); } parameterType.Add(tokenIterator.Current); tokenIterator.GetNext(); while (tokenIterator.Current.Type == TokenType.SquareBracketOpen || tokenIterator.Current.Type == TokenType.SquareBracketClosed) { parameterType.Add(tokenIterator.Current); if (tokenIterator.GetNext() == null) { break; } } Parameters.Add(new LexemeFunctionParameter(parameterType, parameterName)); state = 3; if (parameterType.Count == 1 && !tokenIterator.IsLast) { tokenIterator.RollBack(); } break; } case 3: { if (tokenIterator.Current.Type != TokenType.ComaSign) { throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current); } state = 0; break; } } } }
public ExpressionLexeme(List <Token> tokens) : base(tokens, LexemeType.Expression, false) { var lastParent = new ExpressionToken(null); var iterator = new TokenIterator <Token>(tokens); var level = 0; bool first = true; Operator lastOperator = null; Operator lastUnaryOperator = null; Token lastBracketClosed = null; Token lastFunctionCallToken = null; Token lastIndexerToken = null; while (iterator.GetNext() != null) { if (iterator.Current.Type == TokenType.BracketOpen || iterator.Current.Type == TokenType.SquareBracketOpen) { var oldParent = lastParent; lastParent = new ExpressionToken(null); if (lastFunctionCallToken != null) { lastParent.UnaryFunction = lastFunctionCallToken; } if (lastIndexerToken != null) { lastParent.Indexer = lastIndexerToken; } lastParent.LOperator = lastOperator; if (lastUnaryOperator != null) { lastParent.UnaryOperators.Add(lastUnaryOperator); lastUnaryOperator = null; } lastOperator = null; lastParent.Parent = oldParent; oldParent.SubTokens.Add(lastParent); first = true; level++; } else if (iterator.Current.Type == TokenType.BracketClosed || iterator.Current.Type == TokenType.SquareBracketClosed) { if (lastParent.SubTokens.Count == 1 && lastParent.SubTokens[0].UnaryFunction == null) { lastParent.Parent.SubTokens.Remove(lastParent); lastParent.Parent.SubTokens.Add(lastParent.SubTokens[0]); lastParent.SubTokens[0].UnaryFunction = lastParent.UnaryFunction; lastParent.SubTokens[0].Indexer = lastParent.Indexer; lastParent.SubTokens[0].UnaryOperators.AddRange(lastParent.UnaryOperators); lastParent.SubTokens[0].LOperator = lastParent.LOperator; lastParent.SubTokens[0].ROperator = lastParent.ROperator; } if (lastOperator != null) { throw new CompileException(CompileErrorType.OperatorWithoutOperand, iterator.Current); } lastBracketClosed = iterator.Current; lastParent = lastParent.Parent; level--; } else { if (Token.MathOperatorTokenType.HasFlag(iterator.Current.Type)) { var possibleOperator = new List <TokenType>(); var matchedOperators = new List <Operator>(); possibleOperator.Add(iterator.Current.Type); while (iterator.Index != tokens.Count - 1 && Token.MathOperatorTokenType.HasFlag(tokens[iterator.Index + 1].Type)) { possibleOperator.Add(iterator.GetNext().Type); } foreach (var op in Operator.Operators.OrderBy(p => p.IsUnary)) { var c1 = 0; while (c1 < op.OperatorTypes.Count && c1 < possibleOperator.Count) { if (op.OperatorTypes[c1].HasFlag(possibleOperator[c1])) { c1++; } else { break; } } if (c1 == op.OperatorTypes.Count && c1 == possibleOperator.Count) { matchedOperators.Add(op); } } if (matchedOperators.Count == 0) { //Cant find full matches, try to find partial ones foreach (var op in Operator.Operators.FindAll(p => p.IsUnary)) { var c1 = 0; while (c1 < op.OperatorTypes.Count) { if (op.OperatorTypes[c1] .HasFlag(possibleOperator[possibleOperator.Count - c1 - 1])) { c1++; } else { break; } } if (c1 == op.OperatorTypes.Count) { matchedOperators.Add(op); } } if (matchedOperators.Count == 0) { throw new CompileException(CompileErrorType.UnknownOperator, iterator.Current); } if (possibleOperator.Count - matchedOperators[0].OperatorTypes.Count == 0) { throw new CompileException(CompileErrorType.UnknownOperator, iterator.Current); } var binaryOp = possibleOperator.Take(possibleOperator.Count - matchedOperators[0].OperatorTypes.Count).ToList(); Operator binary = null; foreach (var op in Operator.Operators.FindAll(p => !p.IsUnary)) { var c1 = 0; while (c1 < op.OperatorTypes.Count && c1 < binaryOp.Count) { if (op.OperatorTypes[c1].HasFlag(binaryOp[c1])) { c1++; } else { break; } } if (c1 == op.OperatorTypes.Count && c1 == binaryOp.Count) { binary = op; break; } } if (binary == null) { throw new CompileException(CompileErrorType.UnknownOperator, iterator.Current); } //Console.Write(binary + " "); //Console.WriteLine(matchedOperators[0]); if (lastParent.SubTokens.Last().ROperator != null) { throw new CompileException(CompileErrorType.MultipleOperators, iterator.Current); } lastParent.SubTokens.Last().ROperator = binary; lastOperator = binary; lastUnaryOperator = matchedOperators[0]; } else { //Console.WriteLine(matchedOperators[0]); Operator unary = matchedOperators.Find(p => p.IsUnary); if (lastParent.SubTokens.Count == 0 && unary == null) { throw new CompileException(CompileErrorType.OperatorWithoutOperand, iterator.Current); } if (unary != null && (lastParent.SubTokens.Count == 0 || lastParent.SubTokens.Last().ROperator != null)) { lastUnaryOperator = unary; } else { if (lastParent.SubTokens.Last().ROperator != null) { throw new CompileException(CompileErrorType.MultipleOperators, iterator.Current); } lastParent.SubTokens.Last().ROperator = matchedOperators[0]; lastOperator = matchedOperators[0]; } } } else { if (iterator.Current.Type == TokenType.Number || iterator.Current.Type == TokenType.FloatNumber || iterator.Current.Type == TokenType.StringToken || iterator.Current.Type == TokenType.Identifier) { if (iterator.Current.Type == TokenType.Number || iterator.Current.Type == TokenType.FloatNumber || iterator.Current.Type == TokenType.StringToken || (iterator.Current.Type == TokenType.Identifier && iterator.Index != tokens.Count - 1 && tokens[iterator.Index + 1].Type != TokenType.BracketOpen && tokens[iterator.Index + 1].Type != TokenType.SquareBracketOpen) || (iterator.Current.Type == TokenType.Identifier && iterator.Index == tokens.Count - 1)) { if (!first && lastOperator == null && lastUnaryOperator == null) { throw new CompileException(CompileErrorType.UnexpectedToken, iterator.Current); } first = false; lastParent.SubTokens.Add(new ExpressionToken(iterator.Current)); lastParent.SubTokens.Last().LOperator = lastOperator; if (lastUnaryOperator != null) { lastParent.SubTokens.Last().UnaryOperators.Add(lastUnaryOperator); } lastUnaryOperator = null; lastOperator = null; } if (iterator.Current.Type == TokenType.Identifier && iterator.Index != tokens.Count - 1 && tokens[iterator.Index + 1].Type == TokenType.BracketOpen) { lastFunctionCallToken = iterator.Current; } else if (iterator.Current.Type == TokenType.Identifier && iterator.Index != tokens.Count - 1 && tokens[iterator.Index + 1].Type == TokenType.SquareBracketOpen) { lastIndexerToken = iterator.Current; } else { lastFunctionCallToken = null; lastIndexerToken = null; } } else { throw new CompileException(CompileErrorType.WrongTokenInExpression, iterator.Current); } } } } if (level != 0) { throw new CompileException(CompileErrorType.WrongExpresionStructure, lastBracketClosed); } if (lastOperator != null) { throw new CompileException(CompileErrorType.OperatorWithoutOperand, iterator.Current); } Root = lastParent; }