//
public TokenList Read(Stream str)
{
Tokenizer tokenizer = new Tokenizer();
TokenList inList = tokenizer.Tokenize(str);
//
TokenList outList = new TokenList();
CurrentTokenIter = inList.GetEnumerator();
NextTokenIter = inList.GetEnumerator();
NextTokenIter.MoveNext();
NextToken = NextTokenIter.Current;
//
ReadTokenLists(outList);
//
//DebugTextWriter writer = new DebugTextWriter();
StringWriter writer = new StringWriter();
inList.RawPrint(writer);
#if DEBUG_PREPROCESSOR
Debug.Print(writer.ToString());
#endif
return(outList);
}
private TokenList TokenizeLine(string line)
{
int oldLength = line.Length;
line = line.TrimStart(' ');
if (line == "")
{
return(null);
}
//else
TokenList tokenList = new TokenList();
//
int indentLevel = oldLength - line.Length;
indentLevel += ExtraIndent;
if (indentLevel > IndentStack.Peek())
{
Indent(indentLevel, tokenList);
}
else if (indentLevel < IndentStack.Peek())
{
Dedent(indentLevel, tokenList);
}
//
Regex regexPattern = new Regex(TokenInfo.PatternString);
MatchCollection matches = regexPattern.Matches(line);
foreach (Match match in matches)
{
TokenKind i = 0;
TokenInfo reserved;
TokenKind code = 0;
foreach (Group group in match.Groups)
{
string matchValue = group.Value;
bool success = group.Success;
// ignore capture index 0 and 1 (general and WhiteSpace)
if (success && i > 0)
{
code = i;
switch (i)
{
case TokenKind.Name:
matchValue = matchValue.Replace('-', '_');
if (TokenInfo.Keywords.TryGetValue(matchValue, out reserved))
{
code = reserved.Kind;
}
tokenList.Add(CreateToken(code, matchValue, line));
break;
case TokenKind.Directive:
matchValue = matchValue.Replace('-', '_');
matchValue = matchValue.Replace("#", "pound");
if (TokenInfo.Directives.TryGetValue(matchValue, out reserved))
{
code = reserved.Kind;
}
switch (code)
{
case TokenKind.PoundIndent:
ExtraIndent += 4;
return(null);
case TokenKind.PoundDedent:
ExtraIndent -= 4;
return(null);
}
break;
case TokenKind.Type:
matchValue = matchValue.Replace('-', '_');
matchValue = matchValue.TrimEnd(':');
tokenList.Add(CreateToken(code, matchValue, line));
break;
case TokenKind.Property:
matchValue = matchValue.Replace('-', '_');
matchValue = matchValue.TrimStart(':');
tokenList.Add(CreateToken(code, matchValue, line));
break;
default:
tokenList.Add(CreateToken(code, matchValue, line));
break;
}
}
i++;
}
}
tokenList.Add(new Token(TokenKind.LineEnd, indentLevel));
return(tokenList);
}
void Indent(int indentLevel, TokenList tokenList)
{
IndentStack.Push(indentLevel);
tokenList.Add(new Token(TokenKind.Indent, indentLevel));
}