IReadOnlyList<IToken> ITokenizer.GetTokens(string text)
{
List<IToken> tokens = new List<IToken>();
StringTokenizer tokenizer = new StringTokenizer(text);
tokenizer.IgnoreWhiteSpace = true;
Token current = null;
do
{
current = tokenizer.Next();
if (current.Kind == TokenKind.EOL)
{
}
IToken token = new TokenImpl()
{
Content = current.Value,
Location = new SourceLocation() { Column = current.Column, Line = current.Line },
Type = (TokenType)(int)current.Kind,
};
tokens.Add(token);
} while (current.Kind != TokenKind.EOF);
return tokens;
}
public GameStringTable(TextReader reader)
{
Tok = new StringTokenizer(reader);
Tok.IgnoreWhiteSpace = true;
Tok.SymbolChars = new char[] { ',', '{', '}' };
Strings = new Dictionary<string, string>();
}
//break up string into words
protected ArrayList tokenizeString(string a)
{
ArrayList al = new ArrayList();
StringTokenizer tok = new StringTokenizer(a);
tok.IgnoreWhiteSpace = true;
tok.SymbolChars = new char[]{',', ':', '!', '?'};
Token token;
do
{
token = tok.Next();
al.Add(token.Value);
} while (token.Kind != TokenKind.EOF);
return al;
}
private void TokenizeLine(string line)
{
StringTokenizer tokenizer = new StringTokenizer(line) { IgnoreWhiteSpace = true, RemoveQuotes = true };
Token token;
int tokenIndex = 0;
this.Tokens.Args = new List<string>();
do
{
token = tokenizer.Next();
switch (tokenIndex++)
{
case 0:
{
this.Tokens.Command = token.Value;
break;
}
default:
{
if (!string.IsNullOrWhiteSpace(token.Value))
{
this.Tokens.Args.Add(token.Value.TrimStart(new char[] { '\\' }).TrimEnd(new char[] { '\\' }));
}
break;
}
}
} while (token.Kind != TokenKind.EOF);
}
