public void TokenPositions()
{
String input = "SELECT\n * \nWHERE {?x ?y ?z}";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(1, tokenizer.TokenLine, "SELECT line number");
Assert.AreEqual(0, tokenizer.TokenAbsolutePosition, "SELECT absolute character position");
Assert.AreEqual(0, tokenizer.TokenLinePosition, "SELECT line character position");
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(2, tokenizer.TokenLine, "* line number");
Assert.AreEqual(9, tokenizer.TokenAbsolutePosition, "* absolute character position");
Assert.AreEqual(2, tokenizer.TokenLinePosition, "* line character position");
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(3, tokenizer.TokenLine, "WHERE line number");
Assert.AreEqual(12, tokenizer.TokenAbsolutePosition, "WHERE absolute character position");
Assert.AreEqual(0, tokenizer.TokenLinePosition, "WHERE line character position");
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(3, tokenizer.TokenLine, "{ line number");
Assert.AreEqual(18, tokenizer.TokenAbsolutePosition, "{ absolute character position");
Assert.AreEqual(6, tokenizer.TokenLinePosition, "{ line character position");
}
public void TokenLineIncrementsForEachNewline()
{
String input = "SELECT\n*\nWHERE";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(3, tokenizer.TokenLine);
}
public void recognizeComment()
{
String input = "# this is a comment";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Comment, tokenizer.Type);
Assert.AreEqual("this is a comment", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeStringLiteralWithLanguage1()
{
String input = "'hello'@foo";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.LiteralLanguage, tokenizer.Type);
Assert.AreEqual("foo", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
}
public void recognizeWhere()
{
String input = "WHERE";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordWhere, tokenizer.Type);
Assert.AreEqual("WHERE", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void RecognizeVariableWithDollar()
{
String input = "$var";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type);
Assert.AreEqual("var", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizePrefixCaseInsensitive()
{
String input = "Prefix";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordPrefix, tokenizer.Type);
Assert.AreEqual("Prefix", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeSelect()
{
String input = "SELECT";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordSelect, tokenizer.Type);
Assert.AreEqual("SELECT", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeSelectCaseInsensitive()
{
String input = "sEleCt";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordSelect, tokenizer.Type);
Assert.AreEqual("sEleCt", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeTwoStringLiteralLong1()
{
String input = "'''hello''' '''goodbye'''";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type);
Assert.AreEqual("hello", tokenizer.TokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type);
Assert.AreEqual("goodbye", tokenizer.TokenText);
}
public void recognizeVariableFollowedByBrace()
{
String input = "?var)";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type);
Assert.AreEqual("var", tokenizer.TokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeQNameFollowedByBrace()
{
String input = ":x}";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.QName, tokenizer.Type);
Assert.AreEqual(":x", tokenizer.TokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
private void AssertTokenIsRecognized(string tokenText, QueryTokenizer.TokenType expectedToken)
{
QueryTokenizer tokenizer = new QueryTokenizer(tokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(expectedToken, tokenizer.Type);
Assert.AreEqual(tokenText, tokenizer.TokenText);
}
public void recognizeTwoVariables()
{
String input = "?var ?var2";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type);
Assert.AreEqual("var", tokenizer.TokenText);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type);
Assert.AreEqual("var2", tokenizer.TokenText);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void recognizeStringLiteralLong1MalformedEndOneQuote()
{
String input = "'''recognizeStringLiteralLong1MalformedEnd'";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.ERROR, tokenizer.Type);
}
public void recognizeStringLiteral2EmbeddedCarriageReturnReturnsError()
{
String input = "\"hell\ro\"";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.ERROR, tokenizer.Type);
}
public void recognizeStringLiteral1EmbeddedNewlineReturnsError()
{
String input = "'hell\no'";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.ERROR, tokenizer.Type);
}
public void RecognizeBrackettedVariableSequence()
{
String input = "(?o)";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.BeginGroup, tokenizer.Type);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type);
Assert.AreEqual(false, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type);
}
public void TokenAbsolutePositionForLiteral()
{
String input = " \"\"\"foo\"\"\"";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(4, tokenizer.TokenAbsolutePosition);
}
public void TokenLineAfterWhitespace()
{
String input = " ?var";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(1, tokenizer.TokenLine);
}
public void TokenAbsolutePositionAfterWhitespace()
{
String input = " ?var";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(4, tokenizer.TokenAbsolutePosition);
}
public void recognizeQuotedIRIRef()
{
String input = "<http://example.com/subj>";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.QuotedIRIRef, tokenizer.Type);
Assert.AreEqual("http://example.com/subj", tokenizer.TokenText);
}
public void recognizeWildcard()
{
String input = "*";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.Wildcard, tokenizer.Type);
Assert.AreEqual("*", tokenizer.TokenText);
}
public void recognizeStatementTerminator()
{
String input = ".";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.StatementTerminator, tokenizer.Type);
Assert.AreEqual(".", tokenizer.TokenText);
}
public void recognizePredicateDelimiter()
{
String input = ",";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.PredicateDelimiter, tokenizer.Type);
Assert.AreEqual(",", tokenizer.TokenText);
}
public void recognizeBlankNode()
{
String input = "_:a";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.BlankNode, tokenizer.Type);
Assert.AreEqual("a", tokenizer.TokenText);
}
public void recognizeKeywordACaseInsensitive()
{
String input = "A";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.KeywordA, tokenizer.Type);
Assert.AreEqual("A", tokenizer.TokenText);
}
public void recognizeLiteralDatatype()
{
String input = "^^<http://example.com/type>";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.LiteralDatatype, tokenizer.Type);
Assert.AreEqual("http://example.com/type", tokenizer.TokenText);
}
public void recognizeBeginGroupWithRoundParenthesis()
{
String input = "(";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.BeginGroup, tokenizer.Type);
Assert.AreEqual("(", tokenizer.TokenText);
}
public void recognizeEndGroup()
{
String input = "}";
QueryTokenizer tokenizer = new QueryTokenizer(input);
Assert.AreEqual(true, tokenizer.MoveNext());
Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type);
Assert.AreEqual("}", tokenizer.TokenText);
}