private static int FindBlockLength(TokenArray tokens)
{
var iterator = new TokenIterator(tokens);
List <TokenMode> blocks = new List <TokenMode>();
{
var token = iterator.GetNext();
var end = GetBlockEndMode(token.Mode);
Debug.Assert(end != null,
"Block must begin with a block start");
blocks.Add(end);
}
while (iterator.HasNext && blocks.Count > 0)
{
var token = iterator.GetNext();
if (token.Mode == blocks[blocks.Count - 1])
{
blocks.RemoveAt(blocks.Count - 1);
}
else
{
var end = GetBlockEndMode(token.Mode);
if (end != null)
{
blocks.Add(end);
}
}
}
return(iterator.Iterator - tokens.Start + 1); // +1 to include the last bracket
}
private static BlockLexeme GetStructureLexeme(List <Token> tokens, out CompileError error)
{
error = null;
var iterator = new TokenIterator <Token>(tokens);
var possibleLexeme = new List <Token>();
var currentParent = new BlockLexeme(null, -1);
int level = 0;
Token lastClosed = null;
while (iterator.GetNext() != null)
{
if (iterator.Current.Type == TokenType.Semicolon ||
iterator.Current.Type == TokenType.BraceOpened ||
iterator.Current.Type == TokenType.BraceClosed)
{
currentParent.ChildLexemes.Add(new UnknownLexeme(possibleLexeme));
possibleLexeme.Clear();
switch (iterator.Current.Type)
{
case TokenType.BraceOpened:
var oldParent = currentParent;
currentParent = new BlockLexeme(oldParent, iterator.Index);
oldParent.ChildLexemes.Add(currentParent);
level++;
break;
case TokenType.BraceClosed:
currentParent = (BlockLexeme)currentParent.Parent;
lastClosed = iterator.Current;
level--;
break;
}
}
else
{
possibleLexeme.Add(iterator.Current);
}
}
if (possibleLexeme.Count != 0)
{
currentParent.ChildLexemes.Add(new UnknownLexeme(possibleLexeme));
}
if (level != 0)
{
error = new CompileError(CompileErrorType.WrongCodeStructure, lastClosed);
}
return(currentParent);
}
public static CompileError CreateAttribute(List <Token> tokens, out Attribute attribute)
{
attribute = null;
if (tokens.Count == 0)
{
return(null);
}
var parameters = new List <Token>();
var name = tokens[1];
var info = Types.Find(p => p.Regex.IsMatch(tokens[1].StringValue));
if (info == null)
{
return(new CompileError(CompileErrorType.UnknownAttributeType, name));
}
var iterator = new TokenIterator <Token>(tokens.Skip(3).Take(tokens.Count - 4));
while (iterator.GetNext() != null)
{
if (iterator.Current.Type != TokenType.ComaSign)
{
parameters.Add(iterator.Current);
}
}
if (parameters.Count != info.ParametersCount && info.ParametersCount != -1)
{
return(new CompileError(CompileErrorType.WrongAttributeParameterCount, name));
}
attribute = CreateAttributeMember(info.Type, parameters);
if (attribute == null)
{
return(new CompileError(CompileErrorType.UnknownAttributeType, name));
}
return(attribute.VerifyParameters());
}
public static Instruction Parse(TokenArray tokens)
{
var iterator = new TokenIterator(tokens);
/* VDeclr: type, named, (equals, Statement){0..1}, semicolon
* Declr: [FDeclr, VDeclr]
* PDeclr: (VarDeclr, comma){1..n}
* Params: (Statement, comma){1..n}
* Block: [Function, Declr, Call]{1..n}
* Statement: [Call, Equation], semicolon
* Call: named, paren, Params, paren
* FDeclr: type, named, paren, PDeclr, paren, brace, Block, brace
*/
var iter = Parse(ref iterator, blockRule, 0);
if (iter == null)
{
iterator.Iterator = farthestIterator;
string str = "";
for (int t = 0; t < 10 && iterator.HasNext; ++t)
{
str += (t > 0 ? " " : "") + iterator.GetNext().String;
}
Debug.WriteLine("Unable to complete parse, error at " + str);
return(null);
}
var context = new ParseContext();
iter(context);
var store = context.GetStoreAt(0);
for (int o = 0; o < store.Count; ++o)
{
Console.WriteLine(store[o]);
}
return(store[0] as Instruction);
while (iterator.HasNext)
{
var token = iterator.GetNext();
if (token.Mode == commentToken)
{
continue;
}
else if (token.Mode == typeToken) // Variable / function declaration
{
string type = token.String;
token = iterator.GetNext();
Debug.Assert(token.Mode == nameToken,
"A name must come after the type in a variable declaration, " + type);
string name = token.String;
Debug.Assert(iterator.HasNext,
"Missing semicolon, assignment operator or open parenthesis after type declaration, " + type + " " + name);
token = iterator.GetNext();
if (token.Mode == endStatementToken)
{
Console.WriteLine("Variable found, " + type + " " + name);
}
else if (token.Mode == operatorToken)
{
Debug.Assert(token.String == "=",
"A type declaration can only have an assignment operator, " + type + " " + name);
string equalTo = "";
token = iterator.GetNext();
int equalI = iterator.Iterator;
while (token.Mode != endStatementToken)
{
equalTo += token.String; token = iterator.GetNext();
}
Console.WriteLine("Variable found, " + type + " " + name + " = " + equalTo);
Parse(new TokenArray(tokens.Tokens, equalI, iterator.Iterator));
}
else if (token.Mode == parmStartToken)
{
int len = FindBlockLength(new TokenArray(tokens.Tokens, iterator.Iterator, tokens.End));
iterator.Iterator += len - 1;
token = iterator.GetNext();
if (token.Mode == blockStartToken)
{
int bodyLen = FindBlockLength(new TokenArray(tokens.Tokens, iterator.Iterator, tokens.End));
int bodyStart = iterator.Iterator + 1;
iterator.Iterator += bodyLen - 1;
Console.WriteLine("Function found, " + type + " " + name + "(" + len + ") " + bodyLen);
Parse(new TokenArray(tokens.Tokens, bodyStart, iterator.Iterator));
}
else if (token.Mode == endStatementToken)
{
Console.WriteLine("Function call, " + type + " " + name + "(" + len + ") ");
}
}
else
{
Debug.Assert(false, "Unknown or incomplete variable declaration, " + type + " " + name);
}
}
else if (token.Mode == nameToken)
{
}
else if (token.Mode == flowControlToken)
{
switch (token.String)
{
case "if": {
//Console.WriteLine("If");
} break;
}
}
}
return(null);
}
public static Action <ParseContext> Parse(ref TokenIterator iterator, GrammarRule condition, int level)
{
int c;
TokenIterator startIter = iterator;
string tabs = "";
for (int l = 0; l < level; ++l)
{
tabs += " ";
}
List <Action <ParseContext> > children = new List <Action <ParseContext> >();
for (c = 0; c < condition.Cardinality.To; ++c)
{
TokenIterator titerator = iterator;
for (int g = 0; g < condition.Grammar.Length; ++g)
{
var grammar = condition.Grammar[g];
if (grammar is GrammarElementT)
{
var el = (grammar as GrammarElementT);
Token token;
do
{
if (!titerator.HasNext)
{
goto fail;
}
token = titerator.GetNext();
} while (token.Mode == commentToken);
if (token.Mode != el.Mode || (el.ValidityCheck != null && !el.ValidityCheck(token)))
{
goto fail;
}
}
else if (grammar is GrammarElementC)
{
var success = Parse(ref titerator, (grammar as GrammarElementC).Condition, level + 1);
if (success == null)
{
goto fail;
}
children.Add(success);
}
else if (grammar is GrammarElementO)
{
var options = (grammar as GrammarElementO).Conditions;
int goodOpt = -1;
for (int o = 0; o < options.Length; ++o)
{
TokenIterator subIter = titerator;
var success = Parse(ref subIter, options[o], level + 1);
if (success != null)
{
titerator = subIter;
children.Add(success);
goodOpt = o;
break;
}
}
if (goodOpt == -1)
{
goto fail;
}
}
if (iterator.Iterator > farthestIterator)
{
farthestIterator = iterator.Iterator;
}
}
iterator = titerator;
Console.WriteLine(tabs + condition.Name);
}
fail:
if (c >= condition.Cardinality.From)
{
return delegate(ParseContext context) {
using (var contextLevel = new ParseContext.Level(context, level)) {
contextLevel.ClearChildren();
for (int cn = 0; cn < children.Count; ++cn)
{
children[cn](context);
}
if (condition.Build != null)
{
condition.Build(contextLevel, startIter);
}
else
{
contextLevel.PassThrough();
}
}
}
}
;
return(null);
}
public FunctionLexeme(List <Token> tokens) : base(tokens, LexemeType.Function, true)
{
switch (tokens[0].Type)
{
case TokenType.PublicKeyword:
Modifier = FunctionModifier.Public;
break;
case TokenType.PrivateKeyword:
Modifier = FunctionModifier.Private;
break;
case TokenType.EntrypointKeyword:
Modifier = FunctionModifier.Entrypoint;
break;
case TokenType.FinalizationKeyword:
Modifier = FunctionModifier.Finalization;
break;
case TokenType.InitializationKeyword:
Modifier = FunctionModifier.Initialization;
break;
default:
Modifier = FunctionModifier.None;
break;
}
var index = 0;
if (Modifier != FunctionModifier.None)
{
index = 1;
}
index++; //Function keyword;
var bracketIndex = tokens.FindIndex(p => p.Type == TokenType.BracketOpen);
if (bracketIndex != 2 + (Modifier != FunctionModifier.None ? 1 : 0))
{
//Has type and Name
Name = tokens[bracketIndex - 1];
ReturnType =
tokens[index].StringValue == "void" ? null :
new List <Token>
{
tokens[index]
};
index += 1;
while (tokens[index].Type == TokenType.SquareBracketClosed ||
tokens[index].Type == TokenType.SquareBracketOpen)
{
if (ReturnType == null)
{
throw new CompileException(CompileErrorType.UnexpectedToken, tokens[index]);
}
ReturnType.Add(tokens[index]);
index++;
}
index++;
}
else
{
Name = tokens[index];
index++;
}
index++; //bracket
Token parameterName = null;
var parameterType = new List <Token>();
Parameters = new List <LexemeFunctionParameter>();
var state = 0;
var tokenIterator = new TokenIterator <Token>(tokens.Skip(index).Take(tokens.Count - 1 - index));
while (tokenIterator.GetNext() != null)
{
switch (state)
{
case 0:
{
if (tokenIterator.Current.Type != TokenType.Identifier)
{
throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current);
}
parameterName = tokenIterator.Current;
state = 1;
break;
}
case 1:
{
if (tokenIterator.Current.Type != TokenType.Colon)
{
throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current);
}
state = 2;
break;
}
case 2:
{
if (tokenIterator.Current.Type != TokenType.Identifier)
{
throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current);
}
parameterType.Add(tokenIterator.Current);
tokenIterator.GetNext();
while (tokenIterator.Current.Type == TokenType.SquareBracketOpen ||
tokenIterator.Current.Type == TokenType.SquareBracketClosed)
{
parameterType.Add(tokenIterator.Current);
if (tokenIterator.GetNext() == null)
{
break;
}
}
Parameters.Add(new LexemeFunctionParameter(parameterType, parameterName));
state = 3;
if (parameterType.Count == 1 && !tokenIterator.IsLast)
{
tokenIterator.RollBack();
}
break;
}
case 3:
{
if (tokenIterator.Current.Type != TokenType.ComaSign)
{
throw new CompileException(CompileErrorType.UnexpectedToken, tokenIterator.Current);
}
state = 0;
break;
}
}
}
}
public ExpressionLexeme(List <Token> tokens) : base(tokens, LexemeType.Expression, false)
{
var lastParent = new ExpressionToken(null);
var iterator = new TokenIterator <Token>(tokens);
var level = 0;
bool first = true;
Operator lastOperator = null;
Operator lastUnaryOperator = null;
Token lastBracketClosed = null;
Token lastFunctionCallToken = null;
Token lastIndexerToken = null;
while (iterator.GetNext() != null)
{
if (iterator.Current.Type == TokenType.BracketOpen || iterator.Current.Type == TokenType.SquareBracketOpen)
{
var oldParent = lastParent;
lastParent = new ExpressionToken(null);
if (lastFunctionCallToken != null)
{
lastParent.UnaryFunction = lastFunctionCallToken;
}
if (lastIndexerToken != null)
{
lastParent.Indexer = lastIndexerToken;
}
lastParent.LOperator = lastOperator;
if (lastUnaryOperator != null)
{
lastParent.UnaryOperators.Add(lastUnaryOperator);
lastUnaryOperator = null;
}
lastOperator = null;
lastParent.Parent = oldParent;
oldParent.SubTokens.Add(lastParent);
first = true;
level++;
}
else if (iterator.Current.Type == TokenType.BracketClosed || iterator.Current.Type == TokenType.SquareBracketClosed)
{
if (lastParent.SubTokens.Count == 1 && lastParent.SubTokens[0].UnaryFunction == null)
{
lastParent.Parent.SubTokens.Remove(lastParent);
lastParent.Parent.SubTokens.Add(lastParent.SubTokens[0]);
lastParent.SubTokens[0].UnaryFunction = lastParent.UnaryFunction;
lastParent.SubTokens[0].Indexer = lastParent.Indexer;
lastParent.SubTokens[0].UnaryOperators.AddRange(lastParent.UnaryOperators);
lastParent.SubTokens[0].LOperator = lastParent.LOperator;
lastParent.SubTokens[0].ROperator = lastParent.ROperator;
}
if (lastOperator != null)
{
throw new CompileException(CompileErrorType.OperatorWithoutOperand, iterator.Current);
}
lastBracketClosed = iterator.Current;
lastParent = lastParent.Parent;
level--;
}
else
{
if (Token.MathOperatorTokenType.HasFlag(iterator.Current.Type))
{
var possibleOperator = new List <TokenType>();
var matchedOperators = new List <Operator>();
possibleOperator.Add(iterator.Current.Type);
while (iterator.Index != tokens.Count - 1 && Token.MathOperatorTokenType.HasFlag(tokens[iterator.Index + 1].Type))
{
possibleOperator.Add(iterator.GetNext().Type);
}
foreach (var op in Operator.Operators.OrderBy(p => p.IsUnary))
{
var c1 = 0;
while (c1 < op.OperatorTypes.Count && c1 < possibleOperator.Count)
{
if (op.OperatorTypes[c1].HasFlag(possibleOperator[c1]))
{
c1++;
}
else
{
break;
}
}
if (c1 == op.OperatorTypes.Count && c1 == possibleOperator.Count)
{
matchedOperators.Add(op);
}
}
if (matchedOperators.Count == 0)
{
//Cant find full matches, try to find partial ones
foreach (var op in Operator.Operators.FindAll(p => p.IsUnary))
{
var c1 = 0;
while (c1 < op.OperatorTypes.Count)
{
if (op.OperatorTypes[c1]
.HasFlag(possibleOperator[possibleOperator.Count - c1 - 1]))
{
c1++;
}
else
{
break;
}
}
if (c1 == op.OperatorTypes.Count)
{
matchedOperators.Add(op);
}
}
if (matchedOperators.Count == 0)
{
throw new CompileException(CompileErrorType.UnknownOperator, iterator.Current);
}
if (possibleOperator.Count - matchedOperators[0].OperatorTypes.Count == 0)
{
throw new CompileException(CompileErrorType.UnknownOperator, iterator.Current);
}
var binaryOp = possibleOperator.Take(possibleOperator.Count - matchedOperators[0].OperatorTypes.Count).ToList();
Operator binary = null;
foreach (var op in Operator.Operators.FindAll(p => !p.IsUnary))
{
var c1 = 0;
while (c1 < op.OperatorTypes.Count && c1 < binaryOp.Count)
{
if (op.OperatorTypes[c1].HasFlag(binaryOp[c1]))
{
c1++;
}
else
{
break;
}
}
if (c1 == op.OperatorTypes.Count && c1 == binaryOp.Count)
{
binary = op;
break;
}
}
if (binary == null)
{
throw new CompileException(CompileErrorType.UnknownOperator, iterator.Current);
}
//Console.Write(binary + " ");
//Console.WriteLine(matchedOperators[0]);
if (lastParent.SubTokens.Last().ROperator != null)
{
throw new CompileException(CompileErrorType.MultipleOperators, iterator.Current);
}
lastParent.SubTokens.Last().ROperator = binary;
lastOperator = binary;
lastUnaryOperator = matchedOperators[0];
}
else
{
//Console.WriteLine(matchedOperators[0]);
Operator unary = matchedOperators.Find(p => p.IsUnary);
if (lastParent.SubTokens.Count == 0 && unary == null)
{
throw new CompileException(CompileErrorType.OperatorWithoutOperand, iterator.Current);
}
if (unary != null && (lastParent.SubTokens.Count == 0 || lastParent.SubTokens.Last().ROperator != null))
{
lastUnaryOperator = unary;
}
else
{
if (lastParent.SubTokens.Last().ROperator != null)
{
throw new CompileException(CompileErrorType.MultipleOperators, iterator.Current);
}
lastParent.SubTokens.Last().ROperator = matchedOperators[0];
lastOperator = matchedOperators[0];
}
}
}
else
{
if (iterator.Current.Type == TokenType.Number ||
iterator.Current.Type == TokenType.FloatNumber ||
iterator.Current.Type == TokenType.StringToken ||
iterator.Current.Type == TokenType.Identifier)
{
if (iterator.Current.Type == TokenType.Number || iterator.Current.Type == TokenType.FloatNumber || iterator.Current.Type == TokenType.StringToken ||
(iterator.Current.Type == TokenType.Identifier && iterator.Index != tokens.Count - 1 &&
tokens[iterator.Index + 1].Type != TokenType.BracketOpen && tokens[iterator.Index + 1].Type != TokenType.SquareBracketOpen) ||
(iterator.Current.Type == TokenType.Identifier && iterator.Index == tokens.Count - 1))
{
if (!first && lastOperator == null && lastUnaryOperator == null)
{
throw new CompileException(CompileErrorType.UnexpectedToken, iterator.Current);
}
first = false;
lastParent.SubTokens.Add(new ExpressionToken(iterator.Current));
lastParent.SubTokens.Last().LOperator = lastOperator;
if (lastUnaryOperator != null)
{
lastParent.SubTokens.Last().UnaryOperators.Add(lastUnaryOperator);
}
lastUnaryOperator = null;
lastOperator = null;
}
if (iterator.Current.Type == TokenType.Identifier && iterator.Index != tokens.Count - 1 &&
tokens[iterator.Index + 1].Type == TokenType.BracketOpen)
{
lastFunctionCallToken = iterator.Current;
}
else if (iterator.Current.Type == TokenType.Identifier && iterator.Index != tokens.Count - 1 &&
tokens[iterator.Index + 1].Type == TokenType.SquareBracketOpen)
{
lastIndexerToken = iterator.Current;
}
else
{
lastFunctionCallToken = null;
lastIndexerToken = null;
}
}
else
{
throw new CompileException(CompileErrorType.WrongTokenInExpression, iterator.Current);
}
}
}
}
if (level != 0)
{
throw new CompileException(CompileErrorType.WrongExpresionStructure, lastBracketClosed);
}
if (lastOperator != null)
{
throw new CompileException(CompileErrorType.OperatorWithoutOperand, iterator.Current);
}
Root = lastParent;
}
