public void AllowsRepeatableTraversalWhileTraversingUnderlyingEnumeratorItemsAtMostOnce()
{
var tokens = new TokenStream(Tokens());
tokens.Current.ShouldBe(upper, "ABC", 1, 1);
tokens.Advance().Current.ShouldBe(lower, "def", 1, 4);
tokens.Advance().Advance().Current.ShouldBe(upper, "GHI", 1, 7);
tokens.Advance().Advance().Advance().Current.ShouldBe(TokenKind.EndOfInput, "", 1, 10);
tokens.Advance().ShouldBeSameAs(tokens.Advance());
}
public void AllowsRepeatableTraversalWhileTraversingUnderlyingEnumeratorItemsAtMostOnce()
{
var tokens = new TokenStream(Tokens());
tokens.Current.ShouldBe(upper, "ABC", 1, 1);
tokens.Advance().Current.ShouldBe(lower, "def", 1, 4);
tokens.Advance().Advance().Current.ShouldBe(upper, "GHI", 1, 7);
tokens.Advance().Advance().Advance().Current.ShouldBe(TokenKind.EndOfInput, "", 1, 10);
tokens.Advance().ShouldBeSameAs(tokens.Advance());
}
public void TokenStream_Whitespace()
{
TokenStream iter = Helpers.MakeTokenStream("* .foo");
Assert.IsFalse(iter.IsWhiteSpaceBeforeCurrentToken());
Assert.IsTrue(iter.IsWhiteSpaceAfterCurrentToken());
iter.Advance(1);
Assert.IsTrue(iter.IsWhiteSpaceBeforeCurrentToken());
Assert.IsFalse(iter.IsWhiteSpaceAfterCurrentToken());
iter.Advance(1);
Assert.IsFalse(iter.IsWhiteSpaceBeforeCurrentToken());
Assert.IsFalse(iter.IsWhiteSpaceAfterCurrentToken());
}
public void Test_Stream_Advance_type()
{
List <ScannerToken> tokens = new List <ScannerToken>();
tokens.Add(new ScannerToken(TokenType.PROG, "", 1, 1));
tokens.Add(new ScannerToken(TokenType.VAR, "a", 1, 1));
tokens.Add(new ScannerToken(TokenType.ASSIGN, 1, 2));
tokens.Add(new ScannerToken(TokenType.NUMERIC_INT, "5", 1, 2));
TokenStream streamToken = new TokenStream(tokens);
streamToken.Advance();
streamToken.Advance();
Assert.AreEqual(streamToken.Peek().Type, TokenType.NUMERIC_INT);
}
public void ProvidesEndOfInputTokenWhenGivenEmptyEnumerator()
{
var tokens = new TokenStream(Empty());
tokens.Current.ShouldBe(TokenKind.EndOfInput, "", 1, 1);
tokens.Advance().ShouldBeSameAs(tokens);
}
private ASTNode ParseInternal(string input)
{
tokenStream = new TokenStream(ExpressionTokenizer.Tokenize(input, StructList <ExpressionToken> .Get()));
expressionStack = expressionStack ?? StackPool <ASTNode> .Get();
operatorStack = operatorStack ?? StackPool <OperatorNode> .Get();
if (tokenStream.Current == ExpressionTokenType.ExpressionOpen)
{
tokenStream.Advance();
}
if (!tokenStream.HasMoreTokens)
{
throw new ParseException("Failed trying to parse empty expression");
}
if (tokenStream.Last == ExpressionTokenType.ExpressionClose)
{
tokenStream.Chop();
}
ASTNode retn = ParseLoop();
Release();
return(retn);
}
private static AstExpression ParseBinaryOpHelper(TokenStream tks, AstExpression lhs, int minPrec)
{
var si = tks.SourceInfo;
var binOps = BinaryOpPrec.Select(kvp => kvp.Key).ToArray();
while (true)
{
var op = tks.Current;
var thisPrec = GetBinOpPrec(op);
if (thisPrec < minPrec)
{
return(lhs);
}
tks.ThrowIfTokenIsNotIn(binOps);
tks.Advance();
var rhs = ParseUnaryOp(tks);
var next = tks.Current;
var nextPrec = GetBinOpPrec(next);
if (thisPrec < nextPrec ||
(thisPrec == nextPrec && IsBinOpRightAssoc(op)))
{
rhs = ParseBinaryOpHelper(tks, rhs, thisPrec + 1);
}
lhs = new AstBinaryOp(si, lhs, op.Type, rhs);
}
}
public void ProvidesEndOfInputTokenAfterEnumeratorIsExhausted()
{
var tokens = new TokenStream(OneToken());
var end = tokens.Advance();
end.Current.ShouldBe(TokenKind.EndOfInput, "", 1, 4);
end.Advance().ShouldBeSameAs(end);
}
/// <inheritdoc/>
public override IReply <TValue> Parse(TokenStream tokens)
{
if (tokens.Current.Kind == _kind)
{
return(new Success <TValue>(_value, tokens.Advance()));
}
return(new Failure <TValue>(tokens, FailureMessage.Expected(_kind.Name)));
}
/// <inheritdoc/>
public override IGeneralReply ParseGenerally(TokenStream tokens)
{
if (tokens.Current.Kind == _kind)
{
return(new GeneralSuccess(tokens.Advance()));
}
return(new GeneralFailure(tokens, FailureMessage.Expected(_kind.Name)));
}
public Reply <Token> Parse(TokenStream tokens)
{
if (tokens.Current.Literal == expectation)
{
return(new Parsed <Token>(tokens.Current, tokens.Advance()));
}
return(new Error <Token>(tokens, ErrorMessage.Expected(expectation)));
}
public Reply <Token> Parse(TokenStream tokens)
{
if (tokens.Current.Kind == kind)
{
return(new Parsed <Token>(tokens.Current, tokens.Advance()));
}
return(new Error <Token>(tokens, ErrorMessage.Expected(kind.Name)));
}
public void ParseTest()
{
string[] samples1 = new string[]
{
"100px",
"200%",
"-3grad",
"0.8em",
"-0.5deg",
"22s",
"345.7Hz",
"144dpi",
"12dppx",
"12st",
"0.7vmin",
"0vmax",
"1.117gr",
"-6.0dB",
"+12dB",
};
UnitType[] types1 = new UnitType[]
{
UnitType.Length,
UnitType.Percentage,
UnitType.Angle,
UnitType.Length,
UnitType.Angle,
UnitType.Time,
UnitType.Frequency,
UnitType.Resolution,
UnitType.Resolution,
UnitType.Semitones,
UnitType.Viewport,
UnitType.Viewport,
UnitType.Grid,
UnitType.Volume,
UnitType.Volume,
};
int i = 0;
foreach (string text in samples1)
{
ITextProvider tp = new StringTextProvider(text);
TokenStream tokens = Helpers.MakeTokenStream(tp);
tokens.Advance(1);
UnitValue uv = new UnitValue();
uv.Parse(new ItemFactory(tp, null), tp, tokens);
Assert.AreEqual(types1[i++], uv.UnitType);
Assert.AreEqual(CssTokenType.Units, uv.UnitToken.TokenType);
}
}
/// <inheritdoc/>
public override IReply <string> Parse(TokenStream tokens)
{
var currentToken = tokens.Current;
if (currentToken.Kind != _kind)
{
return(new Failure <string>(tokens, FailureMessage.Expected(_kind.Name)));
}
return(new Success <string>(currentToken.Lexeme, tokens.Advance()));
}
/// <inheritdoc/>
public override IReply <TValue> Parse(TokenStream tokens)
{
if (tokens.Current.Kind != _kind)
{
return(new Failure <TValue>(tokens, FailureMessage.Expected(_kind.Name)));
}
var parsedValue = _lexemeMapping(tokens.Current.Lexeme);
return(new Success <TValue>(parsedValue, tokens.Advance()));
}
public void TokenStream_Simple()
{
TokenStream iter = Helpers.MakeTokenStream(".foo");
Assert.AreEqual(CssTokenType.Dot, iter.CurrentToken.TokenType);
Assert.AreEqual(CssTokenType.Identifier, iter.Peek(1).TokenType);
Assert.AreEqual(CssTokenType.EndOfFile, iter.Peek(2).TokenType);
Assert.AreEqual(0, iter.Position);
Assert.AreEqual(3, iter.Tokens.Count);
Assert.AreEqual(iter.Tokens[0], iter.Advance(1));
Assert.AreEqual(1, iter.Position);
Assert.AreEqual(CssTokenType.Identifier, iter.CurrentToken.TokenType);
iter.Advance(-1);
Assert.AreEqual(0, iter.Position);
iter.Advance(2);
Assert.AreEqual(CssTokenType.EndOfFile, iter.CurrentToken.TokenType);
}
public void HtmlComment_ParseTest()
{
string text1 = "<!-- foo#bar -->";
ITextProvider tp = new StringTextProvider(text1);
TokenStream tokens = Helpers.MakeTokenStream(tp);
// Parse "<!--"
{
HtmlComment hc = new HtmlComment();
Assert.IsTrue(hc.Parse(new ItemFactory(tp, null), tp, tokens));
Assert.AreEqual(1, hc.Children.Count);
Assert.AreEqual(CssTokenType.OpenHtmlComment, ((TokenItem)hc.Children[0]).TokenType);
}
// Parse "foo#bar"
{
Assert.AreEqual(CssTokenType.Identifier, tokens.Advance(1).TokenType);
Assert.AreEqual(CssTokenType.HashName, tokens.Advance(1).TokenType);
}
// Parse "-->"
{
HtmlComment hc = new HtmlComment();
Assert.IsTrue(hc.Parse(new ItemFactory(tp, null), tp, tokens));
Assert.AreEqual(1, hc.Children.Count);
Assert.AreEqual(CssTokenType.CloseHtmlComment, ((TokenItem)hc.Children[0]).TokenType);
}
string text2 = "<!-- @namespace foo \"www.foo.com\" -->";
CssParser parser = new CssParser();
StyleSheet sheet = parser.Parse(text2, true);
Assert.IsTrue(sheet.ComplexItemFromRange(14, 0) is NamespaceDirective);
Assert.IsTrue(sheet.ComplexItemFromRange(2, 0) is HtmlComment);
Assert.IsTrue(sheet.ComplexItemFromRange(35, 0) is HtmlComment);
}
private static IEnumerable <IArgumentInfo> ParseArgumentItem(RdParseContext context)
{
List <IArgumentInfo> arguments = null;
TokenStream <RdToken> tokens = context.Tokens;
tokens.Advance(1);
// Past '\item'. Inside { } we can find any number of '\dots' which are keywords.
Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace)
{
int startTokenIndex, endTokenIndex;
if (RdParseUtility.GetKeywordArgumentBounds(tokens, out startTokenIndex, out endTokenIndex))
{
TextRange range = TextRange.FromBounds(tokens[startTokenIndex].End, tokens[endTokenIndex].Start);
string argumentsText = context.TextProvider.GetText(range);
string[] argumentNames = argumentsText.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
arguments = new List <IArgumentInfo>();
// Move past \item{}
tokens.Position = endTokenIndex + 1;
Debug.Assert(tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace);
if (tokens.CurrentToken.TokenType == RdTokenType.OpenCurlyBrace)
{
string description = RdText.GetText(context);
foreach (string n in argumentNames)
{
string name = n.Trim();
if (name == @"\dots")
{
name = "...";
}
ArgumentInfo info = new ArgumentInfo(name, description.Trim());
arguments.Add(info);
}
}
}
}
return(arguments);
}
// <package-ident> ::= <ident> ( '.' <ident> )*
private static AstPackageIdent ParsePackageIdent(TokenStream tks)
{
var si = tks.SourceInfo;
var idents = new List <string>();
while (true)
{
var ident = tks.Expect(TokenType.Ident).StringValue;
idents.Add(ident);
if (tks.Peek().Type == TokenType.Dot && tks.Peek(1).Type == TokenType.Ident)
{
tks.Advance();
continue;
}
break;
}
return(new AstPackageIdent(si, idents));
}
public void ProvidesPositionOfCurrentToken()
{
var tokens = new TokenStream(Tokens());
tokens.Position.Line.ShouldBe(1);
tokens.Position.Column.ShouldBe(1);
tokens.Advance().Position.Line.ShouldBe(1);
tokens.Advance().Position.Column.ShouldBe(4);
tokens.Advance().Advance().Position.Line.ShouldBe(1);
tokens.Advance().Advance().Position.Column.ShouldBe(7);
tokens.Advance().Advance().Advance().Position.Line.ShouldBe(1);
tokens.Advance().Advance().Advance().Position.Column.ShouldBe(10);
}
public static Ast.ComplexString ParseComplexString(TokenStream Stream, ParseContext Context)
{
System.Diagnostics.Debug.Assert(Stream.Next().Type == TokenType.Dollar);
var start = Stream.Next();
Stream.PushState(TokenStreamState.ComplexString); //Enter complex string parsing mode
Stream.Advance();
if (Stream.Next().Type != TokenType.ComplexStringQuote)
throw new CompileError("[015] Expected \"", Stream);
Stream.Advance();
var stringPieces = new List<Ast.Node>();
while (Stream.Next().Type != TokenType.ComplexStringQuote)
{
if (Stream.Next().Type == TokenType.ComplexStringPart)
{
stringPieces.Add(new Ast.StringLiteral(Stream.Next(), Stream.Next().Value));
Stream.Advance();
}
else if (Stream.Next().Type == TokenType.OpenBracket)
{
Stream.PushState(TokenStreamState.Normal); //Make sure we are parsing normally for the
Stream.Advance(); //Skip the [ //embedded expression
stringPieces.Add(ParseExpression(Stream, Context, TokenType.CloseBracket));
if (Stream.Next().Type != TokenType.CloseBracket) //Shouldn't be possible
throw new CompileError("[016] Expected ]", Stream);
Stream.PopState(); //Return to complex string parsing mode
Stream.Advance();
}
else
throw new InvalidProgramException();
}
Stream.PopState(); //Return to normal parsing mode
Stream.Advance();
return new Ast.ComplexString(start, stringPieces);
}
public static List<Ast.Node> ParseStaticInvokation(
TokenStream Stream,
ParseContext Context)
{
if (Stream.Next().Type != TokenType.OpenBracket) throw new CompileError("[017] Expected [", Stream);
Stream.Advance();
var parameters = new List<Ast.Node>();
while (true)
{
if (IsEndOfStatement(Stream)) throw new CompileError("[018] Expected ]", Stream);
if (Stream.Next().Type == TokenType.CloseBracket)
{
Stream.Advance();
return parameters;
}
parameters.Add(ParseTerm(Stream, Context));
}
}
public static Declaration Parse(String Header)
{
var tokenIterator = new TokenStream(new StringIterator(Header), new ParseContext());
var headerTerms = EtcScriptLib.Parse.ParseMacroDeclarationHeader(tokenIterator,
EtcScriptLib.Parse.DeclarationHeaderTerminatorType.StreamEnd);
var r = new Declaration();
r.ReturnTypeName = "VOID";
if (!tokenIterator.AtEnd() && tokenIterator.Next().Type == TokenType.Colon)
{
tokenIterator.Advance();
if (tokenIterator.Next().Type != TokenType.Identifier) throw new CompileError("Expected identifier", tokenIterator);
r.ReturnTypeName = tokenIterator.Next().Value.ToUpper();
tokenIterator.Advance();
}
if (!tokenIterator.AtEnd()) throw new CompileError("Header did not end when expected");
r.Terms = headerTerms;
r.Body = new LambdaBlock(null);
return r;
}
private static Ast.Statement ParseStatement(
TokenStream Stream,
ParseContext Context)
{
Ast.Statement r = null;
var firstToken = Stream.Next().Value.ToUpper();
if (firstToken == "LET")
{
r = ParseLetStatement(Stream, Context);
}
else if (firstToken == "IF")
{
r = ParseIfStatement(Stream, Context);
}
else if (firstToken == "RETURN")
{
r = ParseReturnStatement(Stream, Context);
}
else if (firstToken == "VAR" || firstToken == "VARIABLE")
{
r = ParseLocalDeclaration(Stream, Context);
}
else
{
//If it's not any special statement, it must be a function call.
var parameters = ParseStaticInvokationStatement(Stream, Context);
var control = Context.FindControl(parameters);
if (control != null)
{
Ast.Node childBlock = null;
if (control.BlockType == ControlBlockType.RequiredBlock)
{
if (Stream.Next().Type == TokenType.OpenBrace)
childBlock = ParseBlock(Stream, Context);
else
throw new CompileError("[00C] Expected {", Stream);
}
else
{
if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[00D] Expected ;", Stream);
Stream.Advance();
}
r = new Ast.ControlInvokation(parameters[0].Source, control, parameters, childBlock);
}
else
{
r = new Ast.StaticInvokation(parameters[0].Source, parameters);
if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[00E] Expected ;", Stream);
Stream.Advance();
}
}
return r;
}
internal static Variable ParseMemberDeclaration(TokenStream Stream, ParseContext Context)
{
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next());
Stream.Advance();
var r = new Variable();
r.StorageMethod = VariableStorageMethod.Member;
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next());
r.Name = Stream.Next().Value.ToUpper();
Stream.Advance();
if (Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next());
r.DeclaredTypeName = Stream.Next().Value.ToUpper();
Stream.Advance();
}
if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
r.Documentation = Stream.Next().Value;
Stream.Advance();
}
if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next());
Stream.Advance();
return r;
}
internal static Declaration ParseRuleDeclaration(TokenStream Stream, ParseContext Context)
{
if (Stream.AtEnd())
throw new CompileError("[02F] Impossible error: ParseRuleDeclaration entered at end of stream.", Stream);
try
{
var r = new Declaration();
r.ReturnTypeName = "RULE-RESULT";
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[030] Expected identifier", Stream.Next());
r.Type = DeclarationType.Rule;
Stream.Advance();
r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBraceOrWhen);
foreach (var t in r.Terms)
if (t.Type == DeclarationTermType.Operator)
throw new CompileError("Rule headers cannot contain operators", Stream);
if (Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream);
r.ReturnTypeName = Stream.Next().Value.ToUpper();
Stream.Advance();
}
if (Stream.Next().Value.ToUpper() == "WHEN")
{
Stream.Advance();
r.WhenClause = new WhenClause(ParseExpression(Stream, Context, (stream) =>
{
if (stream.Next().Type == TokenType.OpenBrace) return true;
if (stream.Next().Type == TokenType.Identifier &&
(stream.Next().Value.ToUpper() == "WITH" ||
stream.Next().Value.ToUpper() == "ORDER")) return true;
return false;
}));
}
r.OrderOperator = OrderOperator.NONE;
if (Stream.Next().Value.ToUpper() == "ORDER")
{
Stream.Advance();
var temp = OrderOperator.NONE;
if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp))
throw new CompileError("Expected FIRST or LAST", Stream);
if (temp != OrderOperator.FIRST && temp != OrderOperator.LAST)
throw new CompileError("Expected FIRST or LAST", Stream);
Stream.Advance();
r.OrderOperator |= temp;
}
if (Stream.Next().Value.ToUpper() == "WITH")
{
Stream.Advance();
var temp = OrderOperator.NONE;
if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp))
throw new CompileError("Expected HIGH or LOW", Stream);
if (temp != OrderOperator.HIGH && temp != OrderOperator.LOW)
throw new CompileError("Expected HIGH or LOW", Stream);
Stream.Advance();
if (Stream.Next().Value.ToUpper() != "PRIORITY")
throw new CompileError("Expected PRIORITY", Stream);
Stream.Advance();
r.OrderOperator |= temp;
}
if (Stream.Next().Type == TokenType.OpenBrace)
{
r.Body = new LambdaBlock(ParseBlock(Stream, Context));
}
else
{ throw new CompileError("[031] Expected block", Stream);
}
if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
r.Documentation = Stream.Next().Value;
Stream.Advance();
}
return r;
}
catch (CompileError ce)
{
throw ce;
}
catch (Exception e)
{
throw new CompileError(e.Message + e.StackTrace, Stream);
}
}
public void ProvidesEndOfInputTokenAfterEnumeratorIsExhausted()
{
var tokens = new TokenStream(OneToken());
var end = tokens.Advance();
end.Current.ShouldBe(TokenKind.EndOfInput, "", 1, 4);
end.Advance().ShouldBeSameAs(end);
}
private static Ast.If ParseIfStatement(
TokenStream Stream,
ParseContext Context)
{
if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "IF")
throw new CompileError("[009] Impossible error: If parse entered, no if found.", Stream);
var r = new Ast.If(Stream.Next());
Stream.Advance(); //Skip 'if'
r.Header = ParseExpression(Stream, Context, TokenType.OpenBrace);
if (Stream.Next().Type != TokenType.OpenBrace) throw new CompileError("[00A] Expected {", Stream);
r.ThenBlock = ParseBlock(Stream, Context);
if (Stream.Next().Type == TokenType.Identifier && Stream.Next().Value.ToUpper() == "ELSE")
{
Stream.Advance();
if (Stream.Next().Type == TokenType.OpenBrace)
r.ElseBlock = ParseBlock(Stream, Context);
else if (Stream.Next().Type == TokenType.Identifier && Stream.Next().Value.ToUpper() == "IF")
r.ElseBlock = ParseIfStatement(Stream, Context);
}
return r;
}
private static List<Ast.Initializer> ParseInitializers(
TokenStream Stream,
ParseContext Context)
{
Stream.Advance();
var r = new List<Ast.Initializer>();
while (Stream.Next().Type != TokenType.CloseBrace)
{
var initializerStart = Stream.Next();
if (Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "LET")
throw new CompileError("[01E] Expected let", Stream);
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[01E] Expected identifier", Stream);
var memberName = Stream.Next().Value.ToUpper();
Stream.Advance();
if (Stream.Next().Type != TokenType.Operator && Stream.Next().Value != "=")
throw new CompileError("[01F] Expected =", Stream);
Stream.Advance();
var value = ParseExpression(Stream, Context, TokenType.Semicolon);
if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[020] Expected ;", Stream);
Stream.Advance();
r.Add(new Ast.Initializer(initializerStart, memberName, value));
}
Stream.Advance();
return r;
}
private static Ast.Let ParseLetStatement(
TokenStream Stream,
ParseContext Context)
{
var start = Stream.Next();
if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "LET")
throw new CompileError("[000] Impossible error: Let parse entered, no let found.", Stream);
Stream.Advance();
var LHS = ParseTerm(Stream, Context);
if (Stream.Next().Type != TokenType.Operator || Stream.Next().Value != "=")
throw new CompileError("[001] Expected '='", Stream);
Stream.Advance();
var RHS = ParseExpression(Stream, Context, TokenType.Semicolon);
if (!IsEndOfStatement(Stream)) throw new CompileError("[002] Expected ;", Stream);
Stream.Advance();
return new Ast.Let(start, LHS, RHS);
}
private static Ast.LocalDeclaration ParseLocalDeclaration(
TokenStream Stream,
ParseContext Context)
{
var start = Stream.Next();
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[003] Expected identifier", Stream);
var r = new Ast.LocalDeclaration(start);
r.Name = Stream.Next().Value;
//r.Typename = "GENERIC";
Stream.Advance();
if (Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[004] Expected identifier", Stream);
r.Typename = Stream.Next().Value.ToUpper();
Stream.Advance();
}
if (Stream.Next().Type == TokenType.Operator && Stream.Next().Value == "=")
{
Stream.Advance();
r.Value = ParseExpression(Stream, Context, TokenType.Semicolon);
}
if (!IsEndOfStatement(Stream)) throw new CompileError("[005] Expected ;", Stream);
Stream.Advance();
return r;
}
private static Ast.Node ParseTerm(
TokenStream Stream,
ParseContext Context)
{
if (IsEndOfStatement(Stream)) throw new CompileError("[012] Expected argument", Stream);
Ast.Node r = null;
if (Stream.Next().Type == TokenType.OpenParen)
{
Stream.Advance();
r = ParseExpression(Stream, Context, TokenType.CloseParen);
Stream.Advance(); //Skip close paren
}
else if (Stream.Next().Type == TokenType.OpenBracket)
{
var parameters = ParseStaticInvokation(Stream, Context);
r = new Ast.StaticInvokation(parameters[0].Source, parameters);
}
else if (Stream.Next().Type == TokenType.Identifier ||
Stream.Next().Type == TokenType.Number ||
Stream.Next().Type == TokenType.String)
{
r = new Ast.Identifier(Stream.Next());
Stream.Advance();
}
else if (Stream.Next().Type == TokenType.Operator)
{
r = new Ast.Identifier(Stream.Next());
Stream.Advance();
}
else if (Stream.Next().Type == TokenType.Dollar)
{
return ParseComplexString(Stream, Context);
}
else if (Stream.Next().Type == TokenType.OpenBrace)
{
var start = Stream.Next();
Stream.Advance();
var list = new List<Ast.Node>();
while (Stream.Next().Type != TokenType.CloseBrace)
list.Add(ParseTerm(Stream, Context));
Stream.Advance();
return new Ast.AssembleList(start, list);
}
else
throw new CompileError("[014] Illegal token in argument list", Stream.Next());
r = ParseOptionalDot(Stream, r, Context);
return r;
}
private static Ast.Node ParseOptionalDot(
TokenStream Stream,
Ast.Node LHS,
ParseContext Context)
{
if (Stream.AtEnd()) return LHS;
if (Stream.Next().Type == TokenType.Dot)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[00F] Dot operator must be followed by identifier", Stream);
var RHS = Stream.Next().Value;
var MA = new Ast.MemberAccess(LHS.Source);
MA.Object = LHS;
MA.Name = RHS.ToUpper();
Stream.Advance();
return ParseOptionalDot(Stream, MA, Context);
}
else if (Stream.Next().Type == TokenType.At)
{
Stream.Advance();
var RHS = ParseTerm(Stream, Context);
var I = new Ast.Indexer(LHS.Source);
I.Object = LHS;
I.Index = RHS;
return ParseOptionalDot(Stream, I, Context);
}
else if (Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[013] Expected identifier", Stream);
var C = new Ast.Cast(Stream.Next(), LHS, Stream.Next().Value.ToUpper());
Stream.Advance();
return ParseOptionalDot(Stream, C, Context);
}
else
return LHS;
}
//Implements http://en.wikipedia.org/wiki/Operator-precedence_parser
private static Ast.Node ParseExpression(
Ast.Node lhs,
TokenStream state,
ParseContext operators,
int minimum_precedence,
Predicate<TokenStream> IsTerminal)
{
while (true)
{
if (IsTerminal(state)) return lhs;
if (state.Next().Type != TokenType.Operator) throw new CompileError("[01C] Expected operator", state.Next());
var precedence = operators.FindPrecedence(state.Next().Value);
if (precedence < minimum_precedence) return lhs;
var op = state.Next();
state.Advance();
var rhs = ParseTerm(state, operators);
while (true)
{
if (state.AtEnd()) break;
if (state.Next().Type == TokenType.Operator)
{
var next_precedence = operators.FindPrecedence(state.Next().Value);
if (next_precedence > precedence)
rhs = ParseExpression(rhs, state, operators, next_precedence, IsTerminal);
else
break;
}
else
break;
}
lhs = new Ast.BinaryOperator(lhs.Source, operators.FindOperator(op.Value).Value, lhs, rhs);
}
}
internal static Type ParseTypeDeclaration(TokenStream Stream, ParseContext Context)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02C] Expected identifier", Stream);
var typename = Stream.Next().Value.ToUpper();
String superTypename;
Stream.Advance();
if (Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("Expected identifier", Stream);
superTypename = Stream.Next().Value.ToUpper();
Stream.Advance();
}
else
superTypename = "GENERIC";
var r = new Type(superTypename) { Name = typename };
if (Stream.Next().Type != TokenType.OpenBrace) throw new CompileError("[02D] Expected {", Stream);
Stream.Advance();
while (Stream.Next().Type != TokenType.CloseBrace)
{
if (Stream.Next().Value.ToUpper() == "VAR" || Stream.Next().Value.ToUpper() == "VARIABLE")
r.Members.Add(ParseMemberDeclaration(Stream, Context));
else
throw new CompileError("[02E] Expected var", Stream);
}
Stream.Advance();
if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
r.Documentation = Stream.Next().Value;
Stream.Advance();
}
return r;
}
public void TryingToAdvanceBeyondEndOfInputResultsInNoMovement()
{
var tokens = new TokenStream(Empty());
tokens.ShouldBeSameAs(tokens.Advance());
}
public void ProvidesEndOfInputTokenWhenGivenEmptyEnumerator()
{
var tokens = new TokenStream(Empty());
tokens.Current.ShouldBe(TokenKind.EndOfInput, "", 1, 1);
tokens.Advance().ShouldBeSameAs(tokens);
}
internal static Declaration ParseMacroDeclaration(TokenStream Stream, ParseContext Context)
{
if (Stream.AtEnd())
throw new CompileError("[028] Impossible error: ParseDeclaration entered at end of stream.", Stream);
try
{
var r = new Declaration();
r.ReturnTypeName = "VOID";
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[029] Expected identifier", Stream.Next());
r.Type = DeclarationType.Macro;
Stream.Advance();
r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBrace);
foreach (var t in r.Terms)
if (t.Type == DeclarationTermType.Operator)
r.DefinesOperator = true;
if (r.DefinesOperator)
{
if (r.Terms.Count != 3 ||
r.Terms[0].Type != DeclarationTermType.Term ||
r.Terms[1].Type != DeclarationTermType.Operator ||
r.Terms[2].Type != DeclarationTermType.Term)
throw new CompileError("Operator macros must be of the form 'term op term'", Stream);
r.Terms[1].Type = DeclarationTermType.Keyword;
}
if (!Stream.AtEnd() && Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream);
r.ReturnTypeName = Stream.Next().Value.ToUpper();
Stream.Advance();
}
if (r.DefinesOperator && r.ReturnTypeName == "VOID")
throw new CompileError("Operator macros must return a value.", Stream);
if (!Stream.AtEnd() && Stream.Next().Type == TokenType.OpenBrace)
{
r.Body = new LambdaBlock(ParseBlock(Stream, Context));
}
else
throw new CompileError("[02B] Expected block", Stream);
if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
r.Documentation = Stream.Next().Value;
Stream.Advance();
}
return r;
}
catch (CompileError ce)
{
throw ce;
}
catch (Exception e)
{
throw new CompileError(e.Message + e.StackTrace, Stream);
}
}
// consider replacing with access expression since this will always be a root property access
private bool ParseIdentifier(ref ASTNode node)
{
if (tokenStream.Current != ExpressionTokenType.Identifier)
{
return(false);
}
node = ASTNode.IdentifierNode(tokenStream.Current);
tokenStream.Advance();
return(true);
}
internal static Variable ParseGlobalDeclaration(TokenStream Stream, ParseContext Context)
{
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next());
Stream.Advance();
var r = new Variable();
r.StorageMethod = VariableStorageMethod.Member;
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next());
var start = Stream.Next();
r.Name = Stream.Next().Value.ToUpper();
Stream.Advance();
if (Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next());
r.DeclaredTypeName = Stream.Next().Value.ToUpper();
Stream.Advance();
}
if (Stream.Next().Type == TokenType.Operator && Stream.Next().Value == "=")
{
Stream.Advance();
var initialValue = ParseExpression(Stream, Context, TokenType.Semicolon);
var initializer = new Ast.Let(start, new Ast.Identifier(start), initialValue);
Context.Initialization.Add(initializer);
}
if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
r.Documentation = Stream.Next().Value;
Stream.Advance();
}
if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next());
Stream.Advance();
return r;
}
private static Ast.BlockStatement ParseBlock(
TokenStream Stream,
ParseContext Context)
{
var r = new Ast.BlockStatement(Stream.Next());
Stream.Advance(); //Skip the opening brace.
while (Stream.Next().Type != TokenType.CloseBrace)
r.Statements.Add(ParseStatement(Stream, Context));
Stream.Advance(); //Skip the closing brace.
return r;
}
private static Ast.Return ParseReturnStatement(
TokenStream Stream,
ParseContext Context)
{
var start = Stream.Next();
if (Stream.AtEnd() || Stream.Next().Type != TokenType.Identifier || Stream.Next().Value.ToUpper() != "RETURN")
throw new CompileError("[007] Impossible error: Return parse entered, no return found.", Stream);
Stream.Advance();
var r = new Ast.Return(start);
if (Stream.Next().Type != TokenType.Semicolon)
r.Value = ParseExpression(Stream, Context, TokenType.Semicolon);
if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[008] Expected ;", Stream);
Stream.Advance();
return r;
}
public void Style(TokenStream Stream)
{
this.Stream = Stream;
this.FoldLevel = 1;
while (!Stream.AtEnd())
{
if (Stream.Next().Value.ToUpper() == "MACRO"
|| Stream.Next().Value.ToUpper() == "FUNCTION"
|| Stream.Next().Value.ToUpper() == "TEST")
ParseMacroDeclaration();
else if (Stream.Next().Value.ToUpper() == "RULE")
ParseRuleDeclaration();
else if (Stream.Next().Value.ToUpper() == "TYPE")
ParseTypeDeclaration();
else if (Stream.Next().Value.ToUpper() == "GLOBAL")
ParseGlobalDeclaration();
else if (Stream.Next().Value.ToUpper() == "INCLUDE")
{
Style(TokenStyle.Keyword);
Expect(TokenType.String);
}
else if (Stream.Next().Value.ToUpper() == "DEFAULT")
{
Style(TokenStyle.Keyword);
Expect(TokenType.Identifier, t => t.Value.ToUpper() == "OF");
ParseRuleDeclaration();
}
else
Stream.Advance();
}
}
private bool ParseDeclaration(ref ASTNode node)
{
AttributeNode attrNode = null;
LightList <AttributeNode> attributes = LightList <AttributeNode> .Get();
while (ParseAttribute(ref attrNode))
{
attributes.Add(attrNode);
if (tokenStream.Current != ExpressionTokenType.ArrayAccessOpen)
{
break;
}
}
if (attributes.size == 0)
{
LightList <AttributeNode> .Release(ref attributes);
}
if (tokenStream.Current != ExpressionTokenType.Identifier)
{
return(false);
}
// modifiers? -> returnType -> name -> signature -> openBrace * closeBrace
tokenStream.Save();
bool isStatic = false;
if (tokenStream.Current == "static")
{
isStatic = true;
tokenStream.Advance();
}
ExpressionParser parser = new ExpressionParser(tokenStream);
StructList <LambdaArgument> signature = null;
TypeLookup typeLookup = default;
if (!parser.ParseTypePath(ref typeLookup))
{
goto fail;
}
tokenStream.Set(parser.GetTokenPosition());
parser.Release(false);
if (tokenStream.Current != ExpressionTokenType.Identifier)
{
goto fail;
}
string name = tokenStream.Current.value;
tokenStream.Advance();
// if semi colon then we have a field!
if (tokenStream.Current == ExpressionTokenType.SemiColon)
{
tokenStream.Advance();
node = new FieldNode()
{
name = name,
isStatic = isStatic,
attributes = attributes,
typeLookup = typeLookup
};
return(true);
}
if (tokenStream.Current != ExpressionTokenType.ParenOpen)
{
goto fail;
}
signature = StructList <LambdaArgument> .Get();
if (tokenStream.NextTokenIs(ExpressionTokenType.ParenClose))
{
tokenStream.Advance(2);
}
else
{
int matchingIndex = tokenStream.FindMatchingIndex(ExpressionTokenType.ParenOpen, ExpressionTokenType.ParenClose);
if (matchingIndex == -1)
{
goto fail;
}
TokenStream subStream = tokenStream.AdvanceAndReturnSubStream(matchingIndex);
subStream.Advance();
tokenStream.Advance();
if (!ExpressionParser.ParseSignature(subStream, signature))
{
goto fail;
}
for (int i = 0; i < signature.size; i++)
{
if (signature.array[i].type == null)
{
throw new ParseException($"When defining a method you must specify a type for all arguments. Found identifier {signature.array[i].identifier} but no type was given.");
}
}
}
if (tokenStream.Current != ExpressionTokenType.ExpressionOpen)
{
goto fail;
}
BlockNode block = ParseBlock();
node = new MethodNode()
{
body = block,
returnTypeLookup = typeLookup,
attributes = attributes,
name = name,
isStatic = isStatic,
signatureList = signature != null?signature.ToArray() : s_EmptySignature
};
StructList <LambdaArgument> .Release(ref signature);
parser.Release(false);
return(true);
fail:
{
tokenStream.Restore();
parser.Release(false);
typeLookup.Release();
signature?.Release();
return(false);
}
}
public static List<Declaration> Build(
TokenStream Stream,
ParseContext Context,
Func<String,ErrorStrategy> OnError,
bool PrepareInitializer = true)
{
var r = new List<Declaration>();
while (!Stream.AtEnd())
{
try
{
if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[036] Expected identifier", Stream);
if (Stream.Next().Value.ToUpper() == "MACRO" || Stream.Next().Value.ToUpper() == "FUNCTION")
{
var declaration = ParseMacroDeclaration(Stream, Context);
declaration.OwnerContextID = Context.ID;
Context.PendingEmission.Add(declaration);
}
else if (Stream.Next().Value.ToUpper() == "TEST")
{
var declaration = ParseMacroDeclaration(Stream, Context);
declaration.Type = DeclarationType.Test;
declaration.OwnerContextID = Context.ID;
r.Add(declaration);
Context.PendingEmission.Add(declaration);
}
else if (Stream.Next().Value.ToUpper() == "RULE")
{
var declaration = ParseRuleDeclaration(Stream, Context);
declaration.OwnerContextID = Context.ID;
var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms);
if (rulebook == null)
{
rulebook = new Rulebook { DeclarationTerms = new List<DeclarationTerm>(
declaration.Terms.Select(t => t.GenericClone())) };
rulebook.ResultTypeName = declaration.ReturnTypeName;
Context.Rules.Rulebooks.Add(rulebook);
}
//if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false)
// throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream);
if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper())
throw new CompileError("Rule return type not compatible with existing rulebook", Stream);
rulebook.Rules.Add(declaration);
Context.PendingEmission.Add(declaration);
}
else if (Stream.Next().Value.ToUpper() == "TYPE")
{
var type = ParseTypeDeclaration(Stream, Context);
if (Context.ActiveScope.FindType(type.Name) != null) throw new CompileError("[038] Type already defined", Stream);
Context.ActiveScope.Types.Add(type);
}
else if (Stream.Next().Value.ToUpper() == "GLOBAL")
{
var variable = ParseGlobalDeclaration(Stream, Context);
variable.StorageMethod = VariableStorageMethod.Static;
Context.ActiveScope.Variables.Add(variable);
}
else if (Stream.Next().Value.ToUpper() == "INCLUDE")
{
Stream.Advance();
if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected string", Stream);
var filename = Stream.Next().Value;
Stream.Advance();
if (Context.FileLoader == null) throw new CompileError("Inclusion not enabled", Stream);
var file = Context.FileLoader.LoadFile(filename, Stream.CurrentFile);
var newStream = new TokenStream(new StringIterator(file.Data), Context);
newStream.CurrentFile = file;
r.AddRange(Build(newStream, Context, OnError, false));
}
else if (Stream.Next().Value.ToUpper() == "DEFAULT")
{
Stream.Advance();
if (Stream.Next().Value.ToUpper() != "OF") throw new CompileError("Expected 'OF'", Stream);
Stream.Advance();
if (Stream.Next().Value.ToUpper() != "RULE") throw new CompileError("Expected 'RULE'", Stream);
var declaration = ParseRuleDeclaration(Stream, Context);
foreach (var term in declaration.Terms)
if (!String.IsNullOrEmpty(term.DeclaredTypeName) && term.DeclaredTypeName != "GENERIC")
throw new CompileError("Don't declare types for default rules.");
declaration.OwnerContextID = Context.ID;
var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms);
if (rulebook == null)
{
rulebook = new Rulebook { DeclarationTerms = declaration.Terms };
rulebook.ResultTypeName = declaration.ReturnTypeName;
Context.Rules.Rulebooks.Add(rulebook);
}
//if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false)
// throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream);
if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper())
throw new CompileError("Rule return type not compatible with existing rulebook", Stream);
if (rulebook.DefaultValue != null)
throw new CompileError("Rulebook already has a default value", Stream);
rulebook.DefaultValue = declaration;
Context.PendingEmission.Add(declaration);
}
else
throw new CompileError("[039] Unknown declaration type", Stream);
}
catch (Exception e)
{
if (OnError(e.Message) == ErrorStrategy.Abort) return r;
Stream.Advance(); //Prevent an error from causing an infinite loop
}
}
if (PrepareInitializer)
{
Context.InitializationFunction = Declaration.Parse("test initialize-globals : void");
var body = new Ast.BlockStatement(new Token());
body.Statements = Context.Initialization;
Context.InitializationFunction.Body.Body = body;
Context.PendingEmission.Add(Context.InitializationFunction);
}
return r;
}
private bool ParseAttribute(ref AttributeNode node)
{
if (tokenStream.Current != ExpressionTokenType.ArrayAccessOpen)
{
return(false);
}
tokenStream.Save();
tokenStream.Advance();
TypeLookup typeLookup = default;
ExpressionParser parser = new ExpressionParser(tokenStream);
if (!parser.ParseTypePath(ref typeLookup))
{
goto fail;
}
tokenStream.Set(parser.GetTokenPosition());
parser.Release(false);
if (tokenStream.Current == ExpressionTokenType.ArrayAccessClose)
{
tokenStream.Advance();
node = new AttributeNode()
{
typeLookup = typeLookup
};
return(true);
}
fail:
{
typeLookup.Release();
parser.Release(false);
return(false);
}
}
internal static DeclarationTerm ParseDeclarationTerm(TokenStream Stream)
{
DeclarationTerm r = null;
var start = Stream.Next();
if (Stream.Next().Type == TokenType.Identifier)
{
r = new DeclarationTerm
{
Name = Stream.Next().Value.ToUpper(),
Type = DeclarationTermType.Keyword,
RepetitionType = DeclarationTermRepetitionType.Once
};
Stream.Advance();
}
else if (Stream.Next().Type == TokenType.Operator)
{
r = new DeclarationTerm
{
Name = Stream.Next().Value.ToUpper(),
Type = DeclarationTermType.Operator,
RepetitionType = DeclarationTermRepetitionType.Once
};
Stream.Advance();
}
else if (Stream.Next().Type == TokenType.OpenParen)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[021] Expected identifier", start);
r = new DeclarationTerm
{
Name = Stream.Next().Value.ToUpper(),
Type = DeclarationTermType.Term,
RepetitionType = DeclarationTermRepetitionType.Once
};
Stream.Advance();
if (Stream.Next().Type == TokenType.Colon)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[022] Expected identifier", start);
var declaredType = Stream.Next().Value;
r.DeclaredTypeName = declaredType.ToUpper();
Stream.Advance();
}
if (Stream.Next().Type != TokenType.CloseParen)
throw new CompileError("[023] Expected )", start);
Stream.Advance();
}
else
throw new CompileError("[025] Illegal token in declaration header", start);
if (!Stream.AtEnd())
{
if (Stream.Next().Type == TokenType.QuestionMark)
{
var marker = Stream.Next();
var repetitionMarker = Stream.Next().Value;
Stream.Advance();
if (repetitionMarker == "?")
{
if (r.Type != DeclarationTermType.Keyword)
throw new CompileError("[026] Only keywords can be optional in a declaration header", Stream);
r.RepetitionType = DeclarationTermRepetitionType.Optional;
}
//Left over from when terms could be repeated.
//else if (repetitionMarker == "+")
// r.RepetitionType = DeclarationTermRepetitionType.OneOrMany;
//else if (repetitionMarker == "*")
// r.RepetitionType = DeclarationTermRepetitionType.NoneOrMany;
else
throw new CompileError("[027] Unrecognized repetition marker on declaration term", marker);
}
}
return r;
}
public void ProvidesPositionOfCurrentToken()
{
var tokens = new TokenStream(Tokens());
tokens.Position.Line.ShouldBe(1);
tokens.Position.Column.ShouldBe(1);
tokens.Advance().Position.Line.ShouldBe(1);
tokens.Advance().Position.Column.ShouldBe(4);
tokens.Advance().Advance().Position.Line.ShouldBe(1);
tokens.Advance().Advance().Position.Column.ShouldBe(7);
tokens.Advance().Advance().Advance().Position.Line.ShouldBe(1);
tokens.Advance().Advance().Advance().Position.Column.ShouldBe(10);
}
public void TryingToAdvanceBeyondEndOfInputResultsInNoMovement()
{
var tokens = new TokenStream(Empty());
tokens.ShouldBeSameAs(tokens.Advance());
}
public void AdvancesToTheNextToken()
{
var tokens = new TokenStream(Tokens());
tokens.Advance().Current.ShouldBe(lower, "def", 1, 4);
}
public void AdvancesToTheNextToken()
{
var tokens = new TokenStream(Tokens());
tokens.Advance().Current.ShouldBe(lower, "def", 1, 4);
}
private static Ast.New ParseNew(
TokenStream Stream,
ParseContext Context)
{
Stream.Advance();
if (Stream.Next().Type != TokenType.Identifier)
throw new CompileError("[01D] Expected identifier", Stream);
var r = new Ast.New(Stream.Next());
r.Typename = Stream.Next().Value.ToUpper();
Stream.Advance();
if (Stream.Next().Type == TokenType.OpenBrace)
r.Initializers = ParseInitializers(Stream, Context);
return r;
}