Example #1
0
        /// <summary>
        /// Splits tokenchain into separated tokens with a given separator token type.
        /// Also ignores subcodeblocks delimited by (, [ or {.
        /// </summary>
        /// <param name="tokens"></param>
        /// <param name="index"></param>
        /// <param name="length"></param>
        /// <param name="splitter"></param>
        /// <returns></returns>
        public static List<List<Token>> SplitTokenChain(Token[] tokens, int index, int length, TokenType splitter)
        {
            var round = 0;
            var square = 0;
            var curly = 0;
            var listList = new List<List<Token>>();
            var list = new List<Token>();
            for (var i = index; i < index + length; i++)
            {
                var token = tokens[i];
                if (token.Type == TokenType.RoundOpen) round++;
                if (token.Type == TokenType.SquareOpen) square++;
                if (token.Type == TokenType.CurlyOpen) curly++;
                if (token.Type == TokenType.RoundClose) round--;
                if (token.Type == TokenType.SquareClose) square--;
                if (token.Type == TokenType.CurlyClose) curly--;

                if (round <= 0 && square <= 0 && curly <= 0 && token.Type == splitter)
                {
                    listList.Add(new List<Token>() { token}); // add splitters too
                    listList.Add(list);
                    list = new List<Token>();
                }
                else
                {
                    list.Add(token);
                }
            }
            listList.Add(list); // add last part, only if at least one separator is detected
            return listList;
        }
Example #2
0
        public override Expression Parse(Token[] tokens, int index)
        {
            if (tokens[index].Type == TokenType.CurlyOpen)
            {
                var codeBlock = new CodeBlock {OpenBracket = tokens[index]};
                codeBlock.StringIndex = codeBlock.OpenBracket.Index;
                codeBlock.Add(new SingleToken() { Token = codeBlock.OpenBracket});
                var closeIndex = ParseUtils.FindClosingBracket(tokens, index, TokenType.CurlyOpen, TokenType.CurlyClose);
                if (closeIndex.HasValue)
                {
                    codeBlock.CloseBracket = tokens[closeIndex.Value];
                    codeBlock.TokenLength = closeIndex.Value - index + 1;
                }
                else
                {
                    codeBlock.TokenLength = tokens.Length - index;
                }
                codeBlock.Body = ParseSubExpression(tokens, index + 1, codeBlock.TokenLength - (closeIndex.HasValue? 2 : 1), BodyParsers);
                if (codeBlock.Body != null) codeBlock.Add(codeBlock.Body);
                if (codeBlock.CloseBracket != null) codeBlock.Add(new SingleToken() { Token = codeBlock.CloseBracket});

                var lastToken = tokens[index + codeBlock.TokenLength - 1];
                codeBlock.StringLength = lastToken.Index + lastToken.Value.Length - codeBlock.StringIndex;

                return codeBlock;
            }
            return null;
        }
Example #3
0
        public override Expression Parse(Token[] tokens, int index)
        {
            if (tokens[index].Value == "<")
            {
                var annotation = new Annotation {OpenBracket = tokens[index]};

                var closeIndex = ParseUtils.FindClosingBracket(tokens, index, "<", ">");
                if (closeIndex.HasValue)
                {
                    annotation.CloseBracket = tokens[closeIndex.Value];
                    annotation.TokenLength = closeIndex.Value - index + 1;
                }
                else
                {
                    annotation.TokenLength = tokens.Length - index;
                }
                annotation.Body = ParseSubExpression(tokens, index + 1, annotation.TokenLength - (closeIndex.HasValue? 2 : 1), BodyParsers);

                annotation.Add(new SingleToken() { Token = tokens[index] });
                if (annotation.Body != null) annotation.Add(annotation.Body);
                if (annotation.CloseBracket != null) annotation.Add(new SingleToken() { Token = tokens[index] });
                annotation.StringIndex = annotation.OpenBracket.Index;
                var lastToken = tokens[index + annotation.TokenLength - 1];
                annotation.StringLength = lastToken.Index + lastToken.Value.Length - annotation.StringIndex;

                return annotation;
            }
            return null;
        }
Example #4
0
        public override Expression Parse(Token[] tokens, int index)
        {
            var firstToken = tokens[index];
            if (firstToken.Type == TokenType.Identifier && firstToken.Value == "pass")
            {
                var i = index + 1;
                var expression = new Pass {PassToken = firstToken, StringIndex = firstToken.Index};
                expression.Add(new SingleToken() { Token = firstToken});
                if (tokens[i].Type == TokenType.Identifier)
                {
                    expression.Identifier = tokens[i++];
                    expression.Add(new SingleToken() {Token = expression.Identifier});
                }
                expression.CodeBlock = CodeBlockParser.Parse(tokens, i);
                if (expression.CodeBlock != null)
                {
                    expression.Add(expression.CodeBlock);
                    expression.TokenLength = i + expression.CodeBlock.TokenLength - index;
                }
                else return null;

                var lastToken = tokens[index + expression.TokenLength - 1];
                expression.StringLength = lastToken.Index + lastToken.Value.Length - expression.StringIndex;

                return expression;
            }
            return null;
        }
Example #5
0
        public override Expression Parse(Token[] tokens, int index)
        {
            if (index < tokens.Length - 2 && tokens[index].Type == TokenType.Identifier && tokens[index + 1].Type == TokenType.RoundOpen)
            {
                var function = new FunctionCall
                                   {
                                       Identifier = tokens[index],
                                       ArgumentList = ParseSubExpression(tokens, index + 1, 1, ArgumentListParsers),
                                       StringIndex = tokens[index].Index
                                   };
                function.Add(new SingleToken() { Token = function.Identifier});
                if (function.ArgumentList != null) function.Add(function.ArgumentList);
                function.TokenLength = 1 + function.ArgumentList.TokenLength;
                var semiColonIndex = index + 1 + function.ArgumentList.TokenLength;
                if (semiColonIndex < tokens.Length)
                {
                    var semiColon = tokens[semiColonIndex];
                    if (semiColon.Type == TokenType.SemiColon)
                    {
                        function.SemiColon = semiColon;
                        function.TokenLength++;
                        function.Add(new SingleToken() { Token = semiColon});
                    }
                }

                var lastToken = tokens[index + function.TokenLength - 1];
                function.StringLength = lastToken.Index + lastToken.Value.Length - function.StringIndex;

                return function;
            }
            return null;
        }
        public override Expression Parse(Token[] tokens, int index)
        {
            if (index < tokens.Length - 5 && tokens[index].Type == TokenType.Identifier && tokens[index + 1].Type == TokenType.Identifier && tokens[index + 2].Type == TokenType.RoundOpen)
            {
                var codeBlockIndex = ParseUtils.FindTokenIndex(tokens, index, TokenType.CurlyOpen);
                if (codeBlockIndex.HasValue)
                {
                    var function = new FunctionDefinition
                                       {
                                           ReturnType = tokens[index],
                                           Identifier = tokens[index + 1],
                                           ArgumentList = ParseSubExpression(tokens, index + 2, 1, ArgumentListParsers),
                                           CodeBlock =
                                               ParseSubExpression(tokens, codeBlockIndex.Value, 1, CodeBlockParsers)
                                       };
                    function.StringIndex = function.ReturnType.Index;
                    function.Add(new SingleToken() { Token = function.ReturnType});
                    function.Add(new SingleToken() { Token = function.Identifier });
                    if (function.ArgumentList != null) function.Add(function.ArgumentList);
                    if (function.CodeBlock != null) function.Add(function.CodeBlock);
                    function.TokenLength = codeBlockIndex.Value - index + function.CodeBlock.TokenLength;

                    var lastToken = tokens[index + function.TokenLength - 1];
                    function.StringLength = lastToken.Index + lastToken.Value.Length - function.StringIndex;

                    return function;

                }
            }
            return null;
        }
Example #7
0
 public static Token[] GetCodeBlockTokens(Token[] tokens, int index, TokenType open, TokenType close)
 {
     if (tokens[index].Type == TokenType.CurlyOpen)
     {
         var closeIndex = FindClosingBracket(tokens, index, open, close);
         return closeIndex.HasValue ? tokens.SubSet(index, closeIndex.Value - index + 1)
                                    : tokens.SubSet(index);
     }
     return null;
 }
Example #8
0
 public static int? FindTokenIndex(Token[] tokens, int index, string tokenValue)
 {
     for (var i = index; i < tokens.Length; i++)
     {
         if (tokens[i].Value == tokenValue)
         {
             return i;
         }
     }
     return null;
 }
Example #9
0
 public static int? FindTokenIndex(Token[] tokens, int index, TokenType tokenType)
 {
     for (var i = index; i < tokens.Length; i++)
     {
         if (tokens[i].Type == tokenType)
         {
             return i;
         }
     }
     return null;
 }
Example #10
0
 public static int? FindClosingBracket(Token[] tokens, int index, string open, string close)
 {
     if (tokens[index].Value != open) throw new Exception("Index should point to a opening bracket for which the closing bracket must be found.");
     var count = 1;
     var tokenCount = tokens.Length;
     for (var i = index + 1; i < tokenCount; i++)
     {
         var currentToken = tokens[i];
         if (currentToken.Value == open) count++;
         if (currentToken.Value == close) count--;
         if (count == 0) return i;
     }
     return null;
 }
Example #11
0
        public override Expression Parse(Token[] tokens, int index)
        {
            var effect = new Effect
                             {
                                 TokenLength = tokens.Length,
                                 Content = ParseSubExpression(tokens, 0, null, Parsers),
                                 StringIndex = tokens[index].Index,
                                 StringLength = tokens[tokens.Length - 1].Index + tokens[tokens.Length - 1].Value.Length
                             };
            effect.Add(effect.Content);

            var lastToken = tokens[index + effect.TokenLength - 1];
            effect.StringLength = lastToken.Index + lastToken.Value.Length - effect.StringIndex;

            return effect;
        }
Example #12
0
        public override Expression Parse(Token[] tokens, int index)
        {
            var firstToken = tokens[index];
            if (firstToken.Type == TokenType.Identifier && firstToken.Value.StartsWith("technique"))
            {
                var expression = new Technique {TechniqueToken = firstToken, StringIndex = firstToken.Index};
                expression.Add(new SingleToken() { Token = firstToken});
                if (tokens[index + 1].Type == TokenType.Identifier)
                {
                    expression.Identifier = tokens[index + 1];
                    expression.Add(new SingleToken() { Token = expression.Identifier });
                    expression.TokenLength = 2;

                    if (tokens[index + 2].Value == "<")
                    {
                        expression.Annotation = AnnotationParser.Parse(tokens, index + 2);
                        if (expression.Annotation != null)
                        {
                            expression.TokenLength = 2 + expression.Annotation.TokenLength;
                            expression.Add(expression.Annotation);
                        }
                    }

                    var codeBlockIndex = ParseUtils.FindTokenIndex(tokens, index + 2, TokenType.CurlyOpen);
                    if (codeBlockIndex.HasValue)
                    {
                        expression.CodeBlock = CodeBlockParser.Parse(tokens, codeBlockIndex.Value);
                        if (expression.CodeBlock != null)
                        {
                            expression.TokenLength = codeBlockIndex.Value - index + expression.CodeBlock.TokenLength;
                            expression.Add(expression.CodeBlock);
                        }
                    }

                    var lastToken = tokens[index + expression.TokenLength - 1];
                    expression.StringLength = lastToken.Index + lastToken.Value.Length - expression.StringIndex;

                    return expression;
                }
            }
            return null;
        }
Example #13
0
        public override Expression Parse(Token[] tokens, int index)
        {
            if (tokens[index].Type == TokenType.RoundOpen)
            {
                var argumentList = new ArgumentList {OpenBracket = tokens[index]};
                argumentList.StringIndex = argumentList.OpenBracket.Index;
                argumentList.Add(new SingleToken() { Token = argumentList.OpenBracket});
                var closeIndex = ParseUtils.FindClosingBracket(tokens, index, TokenType.RoundOpen, TokenType.RoundClose);
                var length = closeIndex.HasValue ? closeIndex.Value - index + 1 : tokens.Length - index;
                if (closeIndex.HasValue)
                {
                    argumentList.CloseBracket = tokens[closeIndex.Value];
                }
                argumentList.TokenLength = length;
                var arguments = ParseUtils.SplitTokenChain(tokens, index + 1, length - (closeIndex.HasValue ? 2 : 1), TokenType.Comma);
                if (!(arguments.Count == 1 && arguments[0].Count == 0))
                {
                    foreach (var argument in arguments)
                    {
                        if (argument.Count > 0)
                        if (argument[0].Type != TokenType.Comma)
                        {
                            var argumentExpression = ParseSubExpression(argument.ToArray(), 0, null, ArgumentParsers);
                            argumentList.Arguments.Add(argumentExpression);
                            argumentList.Add(argumentExpression);
                        }
                        else
                        {
                            argumentList.Add(new SingleToken() { Token = argument[0]});
                        }
                    }
                }
                if (argumentList.CloseBracket != null) argumentList.Add(new SingleToken() { Token = argumentList.CloseBracket});

                var lastToken = tokens[index + argumentList.TokenLength - 1];
                argumentList.StringLength = lastToken.Index + lastToken.Value.Length - argumentList.StringIndex;

                return argumentList;
            }
            return null;
        }
Example #14
0
 public Effect Parse(Token[] tokens)
 {
     if (tokens == null) throw new ArgumentNullException("Tokens");
     var result = tokens.Length == 0 ? new Effect() : (Effect)Language.EffectParser.Parse(tokens, 0);
     return result;
 }
Example #15
0
        public Expression ParseSubExpression(Token[] tokens, int index, int? maxLength, IEnumerable<ExpressionParser> parsers)
        {
            if (parsers == null) parsers = new List<ExpressionParser>(); // make empty list
            var parserArray = parsers.ToArray();
            var sequence = new Sequence();
            TokenChain unsupported = null;
            var maxIndex = maxLength.HasValue ? index + maxLength - 1 : tokens.Length - 1;
            while (index <= maxIndex)
            {
                Expression expression = null;
                for (var parserIndex = 0; parserIndex < parserArray.Length; parserIndex++)
                {
                    expression = parserArray[parserIndex].Parse(tokens, index);
                    if (expression != null)
                    {
                        if (unsupported != null)
                        {
                            if (unsupported.Tokens.Count == 1)
                            {
                                sequence.Add(new SingleToken() { Token = unsupported.Tokens[0], TokenLength = 1 });
                            }
                            else
                            {
                                sequence.Add(unsupported);
                                unsupported.StringLength = unsupported.Tokens[unsupported.Tokens.Count - 1].Index +
                                                           unsupported.Tokens[unsupported.Tokens.Count - 1].Value.Length - unsupported.StringIndex;
                            }
                            sequence.TokenLength += unsupported.TokenLength;
                            unsupported = null;
                        }
                        sequence.Add(expression);
                        sequence.TokenLength += expression.TokenLength;
                        index += expression.TokenLength;
                        break;
                    }
                }
                if (expression == null)
                {
                    if (unsupported == null) unsupported = new TokenChain() { StringIndex = tokens[index].Index};
                    unsupported.Tokens.Add(tokens[index]);
                    unsupported.TokenLength++;
                    index++;
                }
            }
            if (unsupported != null)
            {
                if (unsupported.Tokens.Count == 1)
                {
                    sequence.Add(new SingleToken() {Token = unsupported.Tokens[0]});
                }
                else
                {
                    sequence.Add(unsupported);
                    unsupported.StringLength = unsupported.Tokens[unsupported.Tokens.Count - 1].Index +
                                                           unsupported.Tokens[unsupported.Tokens.Count - 1].Value.Length;
                }
                sequence.TokenLength += unsupported.TokenLength;
            }

            switch (sequence.Expressions.Count)
            {
                case 0:
                    return null;
                case 1:
                    return sequence.Expressions[0];
                default:
                    sequence.StringIndex = sequence.Expressions[0].StringIndex;
                    var last = sequence.Expressions[sequence.Expressions.Count - 1];
                    sequence.StringLength = last.StringIndex + last.StringLength - sequence.StringIndex;
                    return sequence;
            }
        }
Example #16
0
 public abstract Expression Parse(Token[] tokens, int index);