/// <summary>
/// Splits tokenchain into separated tokens with a given separator token type.
/// Also ignores subcodeblocks delimited by (, [ or {.
/// </summary>
/// <param name="tokens"></param>
/// <param name="index"></param>
/// <param name="length"></param>
/// <param name="splitter"></param>
/// <returns></returns>
public static List<List<Token>> SplitTokenChain(Token[] tokens, int index, int length, TokenType splitter)
{
var round = 0;
var square = 0;
var curly = 0;
var listList = new List<List<Token>>();
var list = new List<Token>();
for (var i = index; i < index + length; i++)
{
var token = tokens[i];
if (token.Type == TokenType.RoundOpen) round++;
if (token.Type == TokenType.SquareOpen) square++;
if (token.Type == TokenType.CurlyOpen) curly++;
if (token.Type == TokenType.RoundClose) round--;
if (token.Type == TokenType.SquareClose) square--;
if (token.Type == TokenType.CurlyClose) curly--;
if (round <= 0 && square <= 0 && curly <= 0 && token.Type == splitter)
{
listList.Add(new List<Token>() { token}); // add splitters too
listList.Add(list);
list = new List<Token>();
}
else
{
list.Add(token);
}
}
listList.Add(list); // add last part, only if at least one separator is detected
return listList;
}
public override Expression Parse(Token[] tokens, int index)
{
if (tokens[index].Type == TokenType.CurlyOpen)
{
var codeBlock = new CodeBlock {OpenBracket = tokens[index]};
codeBlock.StringIndex = codeBlock.OpenBracket.Index;
codeBlock.Add(new SingleToken() { Token = codeBlock.OpenBracket});
var closeIndex = ParseUtils.FindClosingBracket(tokens, index, TokenType.CurlyOpen, TokenType.CurlyClose);
if (closeIndex.HasValue)
{
codeBlock.CloseBracket = tokens[closeIndex.Value];
codeBlock.TokenLength = closeIndex.Value - index + 1;
}
else
{
codeBlock.TokenLength = tokens.Length - index;
}
codeBlock.Body = ParseSubExpression(tokens, index + 1, codeBlock.TokenLength - (closeIndex.HasValue? 2 : 1), BodyParsers);
if (codeBlock.Body != null) codeBlock.Add(codeBlock.Body);
if (codeBlock.CloseBracket != null) codeBlock.Add(new SingleToken() { Token = codeBlock.CloseBracket});
var lastToken = tokens[index + codeBlock.TokenLength - 1];
codeBlock.StringLength = lastToken.Index + lastToken.Value.Length - codeBlock.StringIndex;
return codeBlock;
}
return null;
}
public override Expression Parse(Token[] tokens, int index)
{
if (tokens[index].Value == "<")
{
var annotation = new Annotation {OpenBracket = tokens[index]};
var closeIndex = ParseUtils.FindClosingBracket(tokens, index, "<", ">");
if (closeIndex.HasValue)
{
annotation.CloseBracket = tokens[closeIndex.Value];
annotation.TokenLength = closeIndex.Value - index + 1;
}
else
{
annotation.TokenLength = tokens.Length - index;
}
annotation.Body = ParseSubExpression(tokens, index + 1, annotation.TokenLength - (closeIndex.HasValue? 2 : 1), BodyParsers);
annotation.Add(new SingleToken() { Token = tokens[index] });
if (annotation.Body != null) annotation.Add(annotation.Body);
if (annotation.CloseBracket != null) annotation.Add(new SingleToken() { Token = tokens[index] });
annotation.StringIndex = annotation.OpenBracket.Index;
var lastToken = tokens[index + annotation.TokenLength - 1];
annotation.StringLength = lastToken.Index + lastToken.Value.Length - annotation.StringIndex;
return annotation;
}
return null;
}
public override Expression Parse(Token[] tokens, int index)
{
var firstToken = tokens[index];
if (firstToken.Type == TokenType.Identifier && firstToken.Value == "pass")
{
var i = index + 1;
var expression = new Pass {PassToken = firstToken, StringIndex = firstToken.Index};
expression.Add(new SingleToken() { Token = firstToken});
if (tokens[i].Type == TokenType.Identifier)
{
expression.Identifier = tokens[i++];
expression.Add(new SingleToken() {Token = expression.Identifier});
}
expression.CodeBlock = CodeBlockParser.Parse(tokens, i);
if (expression.CodeBlock != null)
{
expression.Add(expression.CodeBlock);
expression.TokenLength = i + expression.CodeBlock.TokenLength - index;
}
else return null;
var lastToken = tokens[index + expression.TokenLength - 1];
expression.StringLength = lastToken.Index + lastToken.Value.Length - expression.StringIndex;
return expression;
}
return null;
}
public override Expression Parse(Token[] tokens, int index)
{
if (index < tokens.Length - 2 && tokens[index].Type == TokenType.Identifier && tokens[index + 1].Type == TokenType.RoundOpen)
{
var function = new FunctionCall
{
Identifier = tokens[index],
ArgumentList = ParseSubExpression(tokens, index + 1, 1, ArgumentListParsers),
StringIndex = tokens[index].Index
};
function.Add(new SingleToken() { Token = function.Identifier});
if (function.ArgumentList != null) function.Add(function.ArgumentList);
function.TokenLength = 1 + function.ArgumentList.TokenLength;
var semiColonIndex = index + 1 + function.ArgumentList.TokenLength;
if (semiColonIndex < tokens.Length)
{
var semiColon = tokens[semiColonIndex];
if (semiColon.Type == TokenType.SemiColon)
{
function.SemiColon = semiColon;
function.TokenLength++;
function.Add(new SingleToken() { Token = semiColon});
}
}
var lastToken = tokens[index + function.TokenLength - 1];
function.StringLength = lastToken.Index + lastToken.Value.Length - function.StringIndex;
return function;
}
return null;
}
public override Expression Parse(Token[] tokens, int index)
{
if (index < tokens.Length - 5 && tokens[index].Type == TokenType.Identifier && tokens[index + 1].Type == TokenType.Identifier && tokens[index + 2].Type == TokenType.RoundOpen)
{
var codeBlockIndex = ParseUtils.FindTokenIndex(tokens, index, TokenType.CurlyOpen);
if (codeBlockIndex.HasValue)
{
var function = new FunctionDefinition
{
ReturnType = tokens[index],
Identifier = tokens[index + 1],
ArgumentList = ParseSubExpression(tokens, index + 2, 1, ArgumentListParsers),
CodeBlock =
ParseSubExpression(tokens, codeBlockIndex.Value, 1, CodeBlockParsers)
};
function.StringIndex = function.ReturnType.Index;
function.Add(new SingleToken() { Token = function.ReturnType});
function.Add(new SingleToken() { Token = function.Identifier });
if (function.ArgumentList != null) function.Add(function.ArgumentList);
if (function.CodeBlock != null) function.Add(function.CodeBlock);
function.TokenLength = codeBlockIndex.Value - index + function.CodeBlock.TokenLength;
var lastToken = tokens[index + function.TokenLength - 1];
function.StringLength = lastToken.Index + lastToken.Value.Length - function.StringIndex;
return function;
}
}
return null;
}
public static Token[] GetCodeBlockTokens(Token[] tokens, int index, TokenType open, TokenType close)
{
if (tokens[index].Type == TokenType.CurlyOpen)
{
var closeIndex = FindClosingBracket(tokens, index, open, close);
return closeIndex.HasValue ? tokens.SubSet(index, closeIndex.Value - index + 1)
: tokens.SubSet(index);
}
return null;
}
public static int? FindTokenIndex(Token[] tokens, int index, string tokenValue)
{
for (var i = index; i < tokens.Length; i++)
{
if (tokens[i].Value == tokenValue)
{
return i;
}
}
return null;
}
public static int? FindTokenIndex(Token[] tokens, int index, TokenType tokenType)
{
for (var i = index; i < tokens.Length; i++)
{
if (tokens[i].Type == tokenType)
{
return i;
}
}
return null;
}
public static int? FindClosingBracket(Token[] tokens, int index, string open, string close)
{
if (tokens[index].Value != open) throw new Exception("Index should point to a opening bracket for which the closing bracket must be found.");
var count = 1;
var tokenCount = tokens.Length;
for (var i = index + 1; i < tokenCount; i++)
{
var currentToken = tokens[i];
if (currentToken.Value == open) count++;
if (currentToken.Value == close) count--;
if (count == 0) return i;
}
return null;
}
public override Expression Parse(Token[] tokens, int index)
{
var effect = new Effect
{
TokenLength = tokens.Length,
Content = ParseSubExpression(tokens, 0, null, Parsers),
StringIndex = tokens[index].Index,
StringLength = tokens[tokens.Length - 1].Index + tokens[tokens.Length - 1].Value.Length
};
effect.Add(effect.Content);
var lastToken = tokens[index + effect.TokenLength - 1];
effect.StringLength = lastToken.Index + lastToken.Value.Length - effect.StringIndex;
return effect;
}
public override Expression Parse(Token[] tokens, int index)
{
var firstToken = tokens[index];
if (firstToken.Type == TokenType.Identifier && firstToken.Value.StartsWith("technique"))
{
var expression = new Technique {TechniqueToken = firstToken, StringIndex = firstToken.Index};
expression.Add(new SingleToken() { Token = firstToken});
if (tokens[index + 1].Type == TokenType.Identifier)
{
expression.Identifier = tokens[index + 1];
expression.Add(new SingleToken() { Token = expression.Identifier });
expression.TokenLength = 2;
if (tokens[index + 2].Value == "<")
{
expression.Annotation = AnnotationParser.Parse(tokens, index + 2);
if (expression.Annotation != null)
{
expression.TokenLength = 2 + expression.Annotation.TokenLength;
expression.Add(expression.Annotation);
}
}
var codeBlockIndex = ParseUtils.FindTokenIndex(tokens, index + 2, TokenType.CurlyOpen);
if (codeBlockIndex.HasValue)
{
expression.CodeBlock = CodeBlockParser.Parse(tokens, codeBlockIndex.Value);
if (expression.CodeBlock != null)
{
expression.TokenLength = codeBlockIndex.Value - index + expression.CodeBlock.TokenLength;
expression.Add(expression.CodeBlock);
}
}
var lastToken = tokens[index + expression.TokenLength - 1];
expression.StringLength = lastToken.Index + lastToken.Value.Length - expression.StringIndex;
return expression;
}
}
return null;
}
public override Expression Parse(Token[] tokens, int index)
{
if (tokens[index].Type == TokenType.RoundOpen)
{
var argumentList = new ArgumentList {OpenBracket = tokens[index]};
argumentList.StringIndex = argumentList.OpenBracket.Index;
argumentList.Add(new SingleToken() { Token = argumentList.OpenBracket});
var closeIndex = ParseUtils.FindClosingBracket(tokens, index, TokenType.RoundOpen, TokenType.RoundClose);
var length = closeIndex.HasValue ? closeIndex.Value - index + 1 : tokens.Length - index;
if (closeIndex.HasValue)
{
argumentList.CloseBracket = tokens[closeIndex.Value];
}
argumentList.TokenLength = length;
var arguments = ParseUtils.SplitTokenChain(tokens, index + 1, length - (closeIndex.HasValue ? 2 : 1), TokenType.Comma);
if (!(arguments.Count == 1 && arguments[0].Count == 0))
{
foreach (var argument in arguments)
{
if (argument.Count > 0)
if (argument[0].Type != TokenType.Comma)
{
var argumentExpression = ParseSubExpression(argument.ToArray(), 0, null, ArgumentParsers);
argumentList.Arguments.Add(argumentExpression);
argumentList.Add(argumentExpression);
}
else
{
argumentList.Add(new SingleToken() { Token = argument[0]});
}
}
}
if (argumentList.CloseBracket != null) argumentList.Add(new SingleToken() { Token = argumentList.CloseBracket});
var lastToken = tokens[index + argumentList.TokenLength - 1];
argumentList.StringLength = lastToken.Index + lastToken.Value.Length - argumentList.StringIndex;
return argumentList;
}
return null;
}
public Effect Parse(Token[] tokens)
{
if (tokens == null) throw new ArgumentNullException("Tokens");
var result = tokens.Length == 0 ? new Effect() : (Effect)Language.EffectParser.Parse(tokens, 0);
return result;
}
public Expression ParseSubExpression(Token[] tokens, int index, int? maxLength, IEnumerable<ExpressionParser> parsers)
{
if (parsers == null) parsers = new List<ExpressionParser>(); // make empty list
var parserArray = parsers.ToArray();
var sequence = new Sequence();
TokenChain unsupported = null;
var maxIndex = maxLength.HasValue ? index + maxLength - 1 : tokens.Length - 1;
while (index <= maxIndex)
{
Expression expression = null;
for (var parserIndex = 0; parserIndex < parserArray.Length; parserIndex++)
{
expression = parserArray[parserIndex].Parse(tokens, index);
if (expression != null)
{
if (unsupported != null)
{
if (unsupported.Tokens.Count == 1)
{
sequence.Add(new SingleToken() { Token = unsupported.Tokens[0], TokenLength = 1 });
}
else
{
sequence.Add(unsupported);
unsupported.StringLength = unsupported.Tokens[unsupported.Tokens.Count - 1].Index +
unsupported.Tokens[unsupported.Tokens.Count - 1].Value.Length - unsupported.StringIndex;
}
sequence.TokenLength += unsupported.TokenLength;
unsupported = null;
}
sequence.Add(expression);
sequence.TokenLength += expression.TokenLength;
index += expression.TokenLength;
break;
}
}
if (expression == null)
{
if (unsupported == null) unsupported = new TokenChain() { StringIndex = tokens[index].Index};
unsupported.Tokens.Add(tokens[index]);
unsupported.TokenLength++;
index++;
}
}
if (unsupported != null)
{
if (unsupported.Tokens.Count == 1)
{
sequence.Add(new SingleToken() {Token = unsupported.Tokens[0]});
}
else
{
sequence.Add(unsupported);
unsupported.StringLength = unsupported.Tokens[unsupported.Tokens.Count - 1].Index +
unsupported.Tokens[unsupported.Tokens.Count - 1].Value.Length;
}
sequence.TokenLength += unsupported.TokenLength;
}
switch (sequence.Expressions.Count)
{
case 0:
return null;
case 1:
return sequence.Expressions[0];
default:
sequence.StringIndex = sequence.Expressions[0].StringIndex;
var last = sequence.Expressions[sequence.Expressions.Count - 1];
sequence.StringLength = last.StringIndex + last.StringLength - sequence.StringIndex;
return sequence;
}
}
public abstract Expression Parse(Token[] tokens, int index);