IReadOnlyList<IToken> ITokenizer.GetTokens(string text) { List<IToken> tokens = new List<IToken>(); StringTokenizer tokenizer = new StringTokenizer(text); tokenizer.IgnoreWhiteSpace = true; Token current = null; do { current = tokenizer.Next(); if (current.Kind == TokenKind.EOL) { } IToken token = new TokenImpl() { Content = current.Value, Location = new SourceLocation() { Column = current.Column, Line = current.Line }, Type = (TokenType)(int)current.Kind, }; tokens.Add(token); } while (current.Kind != TokenKind.EOF); return tokens; }
public GameStringTable(TextReader reader) { Tok = new StringTokenizer(reader); Tok.IgnoreWhiteSpace = true; Tok.SymbolChars = new char[] { ',', '{', '}' }; Strings = new Dictionary<string, string>(); }
//break up string into words protected ArrayList tokenizeString(string a) { ArrayList al = new ArrayList(); StringTokenizer tok = new StringTokenizer(a); tok.IgnoreWhiteSpace = true; tok.SymbolChars = new char[]{',', ':', '!', '?'}; Token token; do { token = tok.Next(); al.Add(token.Value); } while (token.Kind != TokenKind.EOF); return al; }
private void TokenizeLine(string line) { StringTokenizer tokenizer = new StringTokenizer(line) { IgnoreWhiteSpace = true, RemoveQuotes = true }; Token token; int tokenIndex = 0; this.Tokens.Args = new List<string>(); do { token = tokenizer.Next(); switch (tokenIndex++) { case 0: { this.Tokens.Command = token.Value; break; } default: { if (!string.IsNullOrWhiteSpace(token.Value)) { this.Tokens.Args.Add(token.Value.TrimStart(new char[] { '\\' }).TrimEnd(new char[] { '\\' })); } break; } } } while (token.Kind != TokenKind.EOF); }