public ClojureTokenTag(Token token) { _token = token; }
public Token Next() { if (!_source.HasMore) return null; char currentChar = _source.Next(); Token nextToken = null; if (currentChar == '(') { nextToken = new Token(TokenType.ListStart, currentChar.ToString(), _source.CurrentIndex - 1, 1); } else if (currentChar == ')') { nextToken = new Token(TokenType.ListEnd, currentChar.ToString(), _source.CurrentIndex - 1, 1); } else if (currentChar == '[') { nextToken = new Token(TokenType.VectorStart, currentChar.ToString(), _source.CurrentIndex - 1, 1); } else if (currentChar == ']') { nextToken = new Token(TokenType.VectorEnd, currentChar.ToString(), _source.CurrentIndex - 1, 1); } else if (currentChar == '{') { nextToken = new Token(TokenType.MapStart, currentChar.ToString(), _source.CurrentIndex - 1, 1); } else if (currentChar == '}') { nextToken = new Token(TokenType.MapEnd, currentChar.ToString(), _source.CurrentIndex - 1, 1); } else if (currentChar == ':') { _source.Push(currentChar); string keyword = ReadKeyword(); nextToken = new Token(TokenType.Keyword, keyword, _source.CurrentIndex - keyword.Length, keyword.Length); } else if (IsString(currentChar, "#_")) { ReadChars(1); nextToken = new Token(TokenType.IgnoreReaderMacro, "#_", _source.CurrentIndex - 2, 2); } else if (BuiltInFunctions.Find(f => IsString(currentChar, f)) != null) { string match = BuiltInFunctions.Find(f => IsString(currentChar, f)); ReadChars(match.Length - 1); nextToken = new Token(TokenType.BuiltIn, match, _source.CurrentIndex - match.Length, match.Length); } else if (currentChar == '\\' && !IsNextCharWhitespace()) { string character = ReadCharacter(currentChar); nextToken = new Token(TokenType.Character, character, _source.CurrentIndex - character.Length, character.Length); } else if (IsPrefix(currentChar, "0x")) { ReadChars(1); string number = "0x" + ReadNumber(); nextToken = new Token(TokenType.HexNumber, number, _source.CurrentIndex - number.Length, number.Length); } else if (Char.IsNumber(currentChar)) { _source.Push(currentChar); string number = ReadNumber(); nextToken = new Token(TokenType.Number, number, _source.CurrentIndex - number.Length, number.Length); } else if (currentChar == '"') { _source.Push(currentChar); string str = ReadString(); nextToken = new Token(TokenType.String, str, _source.CurrentIndex - str.Length, str.Length); } else if (IsWhitespace(currentChar)) { _source.Push(currentChar); string str = ReadWhitespace(); nextToken = new Token(TokenType.Whitespace, str, _source.CurrentIndex - str.Length, str.Length); } else if (currentChar == ';') { _source.Push(currentChar); string str = ReadComment(); nextToken = new Token(TokenType.Comment, str, _source.CurrentIndex - str.Length, str.Length); } else if (IsString(currentChar, "true")) { ReadChars(3); nextToken = new Token(TokenType.Boolean, "true", _source.CurrentIndex - 4, 4); } else if (IsString(currentChar, "false")) { ReadChars(4); nextToken = new Token(TokenType.Boolean, "false", _source.CurrentIndex - 5, 5); } else if (IsString(currentChar, "nil")) { ReadChars(2); nextToken = new Token(TokenType.Nil, "nil", _source.CurrentIndex - 3, 3); } else if (IsSymbolPrefix(currentChar)) { _source.Push(currentChar); string str = ReadSymbol(); nextToken = new Token(TokenType.Symbol, str, _source.CurrentIndex - str.Length, str.Length); } else { nextToken = new Token(TokenType.Unknown, currentChar.ToString(), _source.CurrentIndex - 1, 1); } return nextToken; }
private static void AssertTokensAreEqual(Token tokenOne, Token tokenTwo) { Assert.AreEqual(tokenOne.Length, tokenTwo.Length); Assert.AreEqual(tokenOne.Text, tokenTwo.Text); Assert.AreEqual(tokenOne.Type, tokenTwo.Type); }