public bool ReadToken(TokenList parentList, List <TokenKind> terminators)
{
Advance();
if (terminators.Contains(CurrentToken.Kind))
{
return(false);
}
//else
switch (CurrentToken.Kind)
{
case TokenKind.Name:
parentList.Add(ReadComplexName(TokenKind.Name));
break;
case TokenKind.Dollar:
Advance();
parentList.Add(ReadComplexName(TokenKind.Variable));
break;
case TokenKind.LeftRound:
ReadTokenList(parentList, TokenKind.RoundList, RoundListTerminators);
break;
case TokenKind.LeftSquare:
ReadTokenList(parentList, TokenKind.SquareList, SquareListTerminators);
break;
case TokenKind.LeftAngle:
ReadTokenList(parentList, TokenKind.SquareList, AngleListTerminators);
break;
case TokenKind.LineEnd:
if (NextToken.Kind == TokenKind.Indent)
{
Advance();
ReadTokenLists(parentList);
return(false);
}
return(false);
//break;
case TokenKind.Dedent:
throw new Escape();
default:
parentList.Add(CurrentToken);
break;
}
return(true);
}
public TokenList Tokenize(Stream str)
{
StreamReader sr = new StreamReader(str);
String line;
TokenList tokenList = new TokenList();
try
{
TokenList tokList = null;
while ((line = sr.ReadLine()) != null)
{
tokList = TokenizeLine(line);
if (tokList != null)
{
tokenList.AddRange(tokList);
}
}
}
finally
{
str.Close();
}
tokenList.Add(new Token(TokenKind.FileEnd, 0));
return(tokenList);
}
void Dedent(int indentLevel, TokenList tokenList)
{
while (IndentStack.Peek() != indentLevel)
{
tokenList.Add(new Token(TokenKind.Dedent, IndentStack.Pop()));
}
}
public bool ReadTokenList(TokenList parentList, TokenKind listKind, List <TokenKind> terminators)
{
TokenList outList = new TokenList();
bool more = true;
while (more)
{
more = ReadToken(outList, terminators);
}
if (listKind != TokenKind.RoundList && outList.Count == 0)
{
return(true);
}
//else
parentList.Add(new Token(listKind, outList));
return(true);
}
Token ReadComplexName(TokenKind targetKind)
{
if (NextToken.Kind != TokenKind.Dot)
{
if (targetKind == TokenKind.Name)
{
return(CurrentToken);
}
else
{
return(new Token(CurrentToken, targetKind));
}
}
//else
//TODO:Need to test this more
TokenList nameList = new TokenList();
do
{
TokenKind kind = CurrentToken.Kind;
if (kind == TokenKind.Name)
{
nameList.Add(CurrentToken);
if (NextToken.Kind != TokenKind.Dot)
{
break;
}
}
else if (kind == TokenKind.Dot)
{
continue;
}
else
{
break;
}
} while (Advance());
return(new Token(targetKind, new Token(TokenKind.DottedName, nameList)));
}
private TokenList TokenizeLine(string line)
{
int oldLength = line.Length;
line = line.TrimStart(' ');
if (line == "")
{
return(null);
}
//else
TokenList tokenList = new TokenList();
//
int indentLevel = oldLength - line.Length;
indentLevel += ExtraIndent;
if (indentLevel > IndentStack.Peek())
{
Indent(indentLevel, tokenList);
}
else if (indentLevel < IndentStack.Peek())
{
Dedent(indentLevel, tokenList);
}
//
Regex regexPattern = new Regex(TokenInfo.PatternString);
MatchCollection matches = regexPattern.Matches(line);
foreach (Match match in matches)
{
TokenKind i = 0;
TokenInfo reserved;
TokenKind code = 0;
foreach (Group group in match.Groups)
{
string matchValue = group.Value;
bool success = group.Success;
// ignore capture index 0 and 1 (general and WhiteSpace)
if (success && i > 0)
{
code = i;
switch (i)
{
case TokenKind.Name:
matchValue = matchValue.Replace('-', '_');
if (TokenInfo.Keywords.TryGetValue(matchValue, out reserved))
{
code = reserved.Kind;
}
tokenList.Add(CreateToken(code, matchValue, line));
break;
case TokenKind.Directive:
matchValue = matchValue.Replace('-', '_');
matchValue = matchValue.Replace("#", "pound");
if (TokenInfo.Directives.TryGetValue(matchValue, out reserved))
{
code = reserved.Kind;
}
switch (code)
{
case TokenKind.PoundIndent:
ExtraIndent += 4;
return(null);
case TokenKind.PoundDedent:
ExtraIndent -= 4;
return(null);
}
break;
case TokenKind.Type:
matchValue = matchValue.Replace('-', '_');
matchValue = matchValue.TrimEnd(':');
tokenList.Add(CreateToken(code, matchValue, line));
break;
case TokenKind.Property:
matchValue = matchValue.Replace('-', '_');
matchValue = matchValue.TrimStart(':');
tokenList.Add(CreateToken(code, matchValue, line));
break;
default:
tokenList.Add(CreateToken(code, matchValue, line));
break;
}
}
i++;
}
}
tokenList.Add(new Token(TokenKind.LineEnd, indentLevel));
return(tokenList);
}
void Indent(int indentLevel, TokenList tokenList)
{
IndentStack.Push(indentLevel);
tokenList.Add(new Token(TokenKind.Indent, indentLevel));
}