/// <summary>
/// Scans the input stream and returns scanned <see cref="Token"/> objects.
/// </summary>
/// <returns></returns>
public TokenSequence Scan()
{
TokenSequence tokens = new TokenSequence();
//
// Scan indents in the beginning
if(!sourceReader.Empty)
ScanIndent(tokens);
//
// Main loop
while(!sourceReader.Empty)
{
ScanInternal(tokens);
} // while
//
// Terminate statements, if any
if(tokens.Count > 0 && tokens.Last.Type != TokenType.EndStatement)
tokens.Add(new Token(TokenType.EndStatement, null));
//
// Now postprocess what we have scanned so far and
// remove extraneous Indents & stuff
tokens = PostProcessTokens(new TokenSequence(tokens));
return tokens;
}
public void Last()
{
TokenSequence sequence = new TokenSequence();
sequence.Add(new Token(TokenType.BeginBlock, null));
sequence.Add(new Token(TokenType.Colon, null));
sequence.Add(new Token(TokenType.Comma, null));
Assert.AreEqual(new Token(TokenType.Comma, null), sequence.Last);
}
public void AddAndCount()
{
TokenSequence sequence = new TokenSequence();
sequence.Add(new Token(TokenType.BeginBlock, null));
sequence.Add(new Token(TokenType.Colon, null));
sequence.Add(new Token(TokenType.Comma, null));
Assert.AreEqual(3, sequence.Count);
}
/// <summary>
/// Initializes a new instance of the <see cref="TokenSequence"/> class.
/// </summary>
/// <param name="tokens"></param>
public TokenSequence(TokenSequence tokens)
{
foreach(Token token in tokens.tokens)
Add(token);
}
private void ScanInternal(TokenSequence tokens)
{
char lookahead = sourceReader.Next;
if(char.IsDigit(lookahead) || lookahead == '-')
tokens.Add(ScanIntegerConstant());
else if(lookahead == '"')
tokens.Add(ScanStringConstant());
else if(char.IsLetter(lookahead))
tokens.Add(ScanSymbolOrKeyword());
else if(lookahead == ':')
tokens.Add(ScanColon());
else if(lookahead == '=')
tokens.Add(ScanPropertyAssignment());
else if(lookahead == '[')
tokens.Add(ScanLeftSquareBracket());
else if(lookahead == ']')
tokens.Add(ScanRightSquareBracket());
else if(lookahead == '{')
tokens.Add(ScanLeftBrace());
else if(lookahead == '}')
tokens.Add(ScanRightBrace());
else if(lookahead == '(')
tokens.Add(ScanLeftBracket());
else if(lookahead == ')')
tokens.Add(ScanRightBracket());
else if(lookahead == ',')
tokens.Add(ScanComma());
else if(lookahead == '/')
ScanComment();
else if(lookahead == '\r')
{
tokens.Add(new Token(TokenType.EndStatement, new Location(sourceReader.Line, sourceReader.Column + 1)));
sourceReader.ReadNext();
sourceReader.ReadNext();
ScanIndent(tokens);
} // else if
else if(!sourceReader.Empty)
sourceReader.ReadNext();
}
private void ScanIndent(TokenSequence tokens)
{
int indent = 0;
while(!sourceReader.Empty && (sourceReader.Next == ' ' || sourceReader.Next == '\t' ))
{
if(sourceReader.Next == ' ')
indent += 1;
else if(sourceReader.Next == '\t')
indent += 4;
sourceReader.ReadNext();
} // while
if(indent == 0)
return;
tokens.Add(new Token(TokenType.Indent, new StringBuilder().Insert(0, " ", indent).ToString(),
new Location(sourceReader.Line, sourceReader.Column + 1)));
//
// Return if we reached the end of the line - nothing to do here
/*if(!sourceReader.Empty && sourceReader.Next == '\r')
return;
if(indents.Count == 0 || indents.Peek() < indent)
{
if(indents.Count == 0 || tokens.Count == 0)
{
indents.Push(indent);
tokens.Add(new Token(TokenType.BeginBlock, null));
} // if
else
{
if(tokens.Last.Type == TokenType.BeginBlock)
{
if(indents.Peek() != indent)
{
indents.Pop();
indents.Push(indent);
} // if
} // if
else
{
indents.Push(indent);
tokens.Add(new Token(TokenType.BeginBlock, null));
} // else
} // else
} // if
else if(indents.Peek() > indent)
{
while(indents.Count != 0 && indents.Peek() != indent)
{
indents.Pop();
tokens.Add(new Token(TokenType.EndBlock, null));
} // while
} // else if
* */
}
private static TokenSequence OptimizeTokens(TokenSequence tokens)
{
TokenSequence optimizedTokens = new TokenSequence();
//
// Removing sequences of Indent(EndStatement)+ tokens
// and compacting EndStatement+ tokens to just one
while(!tokens.Empty)
{
Token token = tokens.RemoveFirst();
if(token.Type == TokenType.Indent && !tokens.Empty && tokens.First.Type == TokenType.EndStatement)
while(!tokens.Empty && tokens.First.Type == TokenType.EndStatement)
tokens.RemoveFirst();
else if(token.Type == TokenType.EndStatement && !tokens.Empty && tokens.First.Type == TokenType.EndStatement)
{
while(!tokens.Empty && tokens.First.Type == TokenType.EndStatement)
tokens.RemoveFirst();
optimizedTokens.Add(new Token(TokenType.EndStatement, null));
} // else if
else
optimizedTokens.Add(token);
} // while
return optimizedTokens;
}
private static TokenSequence LayoutTokens(TokenSequence tokens)
{
TokenSequence laidOutTokens = new TokenSequence();
Stack<int> indents = new Stack<int>();
indents.Push(0);
while(!tokens.Empty)
{
Token token = tokens.RemoveFirst();
if(token.Type == TokenType.Indent)
{
if(token.Lexeme.Length == indents.Peek())
continue;
if(token.Lexeme.Length > indents.Peek())
{
indents.Push(token.Lexeme.Length);
laidOutTokens.Add(new Token(TokenType.BeginBlock, token.Location));
} // if
else
{
while(indents.Peek() != 0 && indents.Peek() != token.Lexeme.Length)
{
indents.Pop();
laidOutTokens.Add(new Token(TokenType.EndBlock, token.Location));
} // while
} // else
} // if
else
laidOutTokens.Add(token);
} // while
//
// Close remaining blocks
while(indents.Peek() != 0)
{
indents.Pop();
laidOutTokens.Add(new Token(TokenType.EndBlock, null));
} // while
return laidOutTokens;
}
private static TokenSequence PostProcessTokens(TokenSequence tokens)
{
return LayoutTokens(OptimizeTokens(tokens));
}
public void RemoveFirst()
{
TokenSequence sequence = new TokenSequence();
sequence.Add(new Token(TokenType.BeginBlock, null));
sequence.Add(new Token(TokenType.Colon, null));
sequence.Add(new Token(TokenType.Comma, null));
Assert.AreEqual(new Token(TokenType.BeginBlock, null), sequence.RemoveFirst());
Assert.AreEqual(2, sequence.Count);
}
public void LastThrowsInvalidOperationExceptionOnEmptySequence()
{
TokenSequence sequence = new TokenSequence();
Token last = sequence.Last;
}
