public void recognizeStringLiteral2()
{
String input = "\"hello\"";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type);
Assert.AreEqual("hello", tokenizer.TokenText);
}
public void recognizeStringLiteral1EmbeddedSlash()
{
String input = @"'hell\\o'";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type);
Assert.AreEqual(@"hell\o", tokenizer.TokenText);
}
public void recognizeStringLiteral2EmbeddedQuote()
{
String input = @"""hell\'o""";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type);
Assert.AreEqual(@"hell'o", tokenizer.TokenText);
}
public void advancing_too_far_with_2_tokens_throws_exception()
{
var testObject = new QueryTokenizer();
testObject.Start("blah blah");
testObject.Advance();
testObject.Advance();
Assert.Throws <ApplicationException>(() => testObject.Advance());
}
public void tokenizer_recognizes_name()
{
var testObject = new QueryTokenizer();
var name = "blah";
testObject.Start(name);
Assert.True(testObject.Token == QueryTokens.Name);
Assert.True(testObject.TokenString == name);
}
public void tokenizer_recognizes_unique_id()
{
var testObject = new QueryTokenizer();
var name = "blah";
testObject.Start("#" + name);
Assert.True(testObject.Token == QueryTokens.UniqueID);
Assert.True(testObject.TokenString == name);
}
public void tokenizer_recognizes_name_with_preceding_whitespace()
{
var testObject = new QueryTokenizer();
var name = "blah";
testObject.Start(" " + name);
Assert.True(testObject.Token == QueryTokens.Name);
Assert.True(testObject.TokenString == name);
}
public void tokenizer_recognizes_empty_quoted_string_alt()
{
var testObject = new QueryTokenizer();
var str = string.Empty;
testObject.Start("'" + str + "'");
Assert.True(testObject.Token == QueryTokens.QuotedString);
Assert.True(testObject.TokenString == str);
}
public void tokenizer_recognizes_quoted_string()
{
var testObject = new QueryTokenizer();
var str = "hello there";
testObject.Start("\"" + str + "\"");
Assert.True(testObject.Token == QueryTokens.QuotedString);
Assert.True(testObject.TokenString == str);
}
public void TokenLineIncrementsForEachNewline()
{
String input = "SELECT\n*\nWHERE";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(3, tokenizer.TokenLine);
}
public void recognizePrefixCaseInsensitive()
{
String input = "Prefix";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordPrefix, tokenizer.Type);
Assert.AreEqual("Prefix", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void advancing_with_a_single_token_ends_tokenizer()
{
var testObject = new QueryTokenizer();
testObject.Start("blah");
testObject.Advance();
Assert.True(testObject.AtEnd);
Assert.Equal(QueryTokens.End, testObject.Token);
Assert.Equal(string.Empty, testObject.TokenString);
}
public void advancing_to_broken_unique_id_causes_exception()
{
var testObject = new QueryTokenizer();
testObject.Start("name #");
Assert.Throws <ApplicationException>(() => testObject.Advance());
Assert.Equal(QueryTokens.End, testObject.Token);
Assert.Equal(string.Empty, testObject.TokenString);
Assert.True(testObject.AtEnd);
}
public void recognizeComment()
{
String input = "# this is a comment";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Comment, tokenizer.Type);
Assert.AreEqual("this is a comment", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void RecognizeVariableWithDollar()
{
String input = "$var";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type);
Assert.AreEqual("var", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeStringLiteralWithLanguage1()
{
String input = "'hello'@foo";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.LiteralLanguage, tokenizer.Type);
Assert.AreEqual("foo", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
}
public void recognizeWhere()
{
String input = "WHERE";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordWhere, tokenizer.Type);
Assert.AreEqual("WHERE", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeSelectCaseInsensitive()
{
String input = "sEleCt";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordSelect, tokenizer.Type);
Assert.AreEqual("sEleCt", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeSelect()
{
String input = "SELECT";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordSelect, tokenizer.Type);
Assert.AreEqual("SELECT", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void advancing_to_unterminated_quoted_string_throws_alt()
{
var testObject = new QueryTokenizer();
testObject.Start("name 'whatever");
Assert.Throws <ApplicationException>(() =>
{
testObject.Advance(); // Go past name
testObject.Advance(); // Go poast whitespace.
});
}
public void recognizeTwoStringLiteralLong1()
{
String input = "'''hello''' '''goodbye'''";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type);
Assert.AreEqual("hello", tokenizer.TokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type);
Assert.AreEqual("goodbye", tokenizer.TokenText);
}
protected IReadOnlyCollection <string> Run(string query)
{
var config = new QueryTokenizerConfig(
new []
{
new ContainerTokens("\"", "\""),
new ContainerTokens("'", "'"),
},
new [] { "=", "!=", ">", ">=", "<", "<=", "~", "!~" }
);
var sut = new QueryTokenizer(config);
return(sut.Tokenize(query));
}
public void recognizeVariableFollowedByBrace()
{
String input = "?var)";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type);
Assert.AreEqual("var", tokenizer.TokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void skipWhitespaceBetweenTokens()
{
String input = "?var1 ?var2 ?var3 \t?var4 \r?var5 \n?var6";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeQNameFollowedByBrace()
{
String input = ":x}";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.QName, tokenizer.Type);
Assert.AreEqual(":x", tokenizer.TokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void tokenizer_recognizes_question_mark_and_name_with_separator()
{
var testObject = new QueryTokenizer();
var name = "name";
testObject.Start("? " + name);
Assert.True(testObject.Token == QueryTokens.QuestionMark);
Assert.True(testObject.TokenString == "?");
testObject.Advance();
Assert.True(testObject.Token == QueryTokens.Name);
Assert.True(testObject.TokenString == name);
}
public void tokenizer_recognizes_greater_than()
{
var testObject = new QueryTokenizer();
var name = "name";
testObject.Start(">" + name);
Assert.True(testObject.Token == QueryTokens.GreaterThan);
Assert.True(testObject.TokenString == ">");
testObject.Advance();
Assert.True(testObject.Token == QueryTokens.Name);
Assert.True(testObject.TokenString == name);
}
public void tokenizer_recognizes_slash()
{
var testObject = new QueryTokenizer();
var name = "name";
testObject.Start("/" + name);
Assert.True(testObject.Token == QueryTokens.Slash);
Assert.True(testObject.TokenString == "/");
testObject.Advance();
Assert.True(testObject.Token == QueryTokens.Name);
Assert.True(testObject.TokenString == name);
}
public void tokenizer_recognizes_colon_and_name_separated_by_whitespace()
{
var testObject = new QueryTokenizer();
var name = "name";
testObject.Start(": " + name);
Assert.True(testObject.Token == QueryTokens.Colon);
Assert.True(testObject.TokenString == ":");
testObject.Advance();
Assert.True(testObject.Token == QueryTokens.Name);
Assert.True(testObject.TokenString == name);
}
public void tokenizer_recognizes_exclamation_mark_and_name_with_whitespace_in_between()
{
var testObject = new QueryTokenizer();
var name = "name";
testObject.Start("! " + name);
Assert.True(testObject.Token == QueryTokens.ExclamationMark);
Assert.True(testObject.TokenString == "!");
testObject.Advance();
Assert.True(testObject.Token == QueryTokens.Name);
Assert.True(testObject.TokenString == name);
}