public void recognizeStringLiteral2() { String input = "\"hello\""; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type); Assert.AreEqual("hello", tokenizer.TokenText); }
public void recognizeStringLiteral1EmbeddedSlash() { String input = @"'hell\\o'"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type); Assert.AreEqual(@"hell\o", tokenizer.TokenText); }
public void recognizeStringLiteral2EmbeddedQuote() { String input = @"""hell\'o"""; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type); Assert.AreEqual(@"hell'o", tokenizer.TokenText); }
public void advancing_too_far_with_2_tokens_throws_exception() { var testObject = new QueryTokenizer(); testObject.Start("blah blah"); testObject.Advance(); testObject.Advance(); Assert.Throws <ApplicationException>(() => testObject.Advance()); }
public void tokenizer_recognizes_name() { var testObject = new QueryTokenizer(); var name = "blah"; testObject.Start(name); Assert.True(testObject.Token == QueryTokens.Name); Assert.True(testObject.TokenString == name); }
public void tokenizer_recognizes_unique_id() { var testObject = new QueryTokenizer(); var name = "blah"; testObject.Start("#" + name); Assert.True(testObject.Token == QueryTokens.UniqueID); Assert.True(testObject.TokenString == name); }
public void tokenizer_recognizes_name_with_preceding_whitespace() { var testObject = new QueryTokenizer(); var name = "blah"; testObject.Start(" " + name); Assert.True(testObject.Token == QueryTokens.Name); Assert.True(testObject.TokenString == name); }
public void tokenizer_recognizes_empty_quoted_string_alt() { var testObject = new QueryTokenizer(); var str = string.Empty; testObject.Start("'" + str + "'"); Assert.True(testObject.Token == QueryTokens.QuotedString); Assert.True(testObject.TokenString == str); }
public void tokenizer_recognizes_quoted_string() { var testObject = new QueryTokenizer(); var str = "hello there"; testObject.Start("\"" + str + "\""); Assert.True(testObject.Token == QueryTokens.QuotedString); Assert.True(testObject.TokenString == str); }
public void TokenLineIncrementsForEachNewline() { String input = "SELECT\n*\nWHERE"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(3, tokenizer.TokenLine); }
public void recognizePrefixCaseInsensitive() { String input = "Prefix"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.KeywordPrefix, tokenizer.Type); Assert.AreEqual("Prefix", tokenizer.TokenText); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void advancing_with_a_single_token_ends_tokenizer() { var testObject = new QueryTokenizer(); testObject.Start("blah"); testObject.Advance(); Assert.True(testObject.AtEnd); Assert.Equal(QueryTokens.End, testObject.Token); Assert.Equal(string.Empty, testObject.TokenString); }
public void advancing_to_broken_unique_id_causes_exception() { var testObject = new QueryTokenizer(); testObject.Start("name #"); Assert.Throws <ApplicationException>(() => testObject.Advance()); Assert.Equal(QueryTokens.End, testObject.Token); Assert.Equal(string.Empty, testObject.TokenString); Assert.True(testObject.AtEnd); }
public void recognizeComment() { String input = "# this is a comment"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.Comment, tokenizer.Type); Assert.AreEqual("this is a comment", tokenizer.TokenText); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void RecognizeVariableWithDollar() { String input = "$var"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type); Assert.AreEqual("var", tokenizer.TokenText); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void recognizeStringLiteralWithLanguage1() { String input = "'hello'@foo"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.LiteralLanguage, tokenizer.Type); Assert.AreEqual("foo", tokenizer.TokenText); Assert.AreEqual(false, tokenizer.MoveNext()); }
public void recognizeWhere() { String input = "WHERE"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.KeywordWhere, tokenizer.Type); Assert.AreEqual("WHERE", tokenizer.TokenText); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void recognizeSelectCaseInsensitive() { String input = "sEleCt"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.KeywordSelect, tokenizer.Type); Assert.AreEqual("sEleCt", tokenizer.TokenText); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void recognizeSelect() { String input = "SELECT"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.KeywordSelect, tokenizer.Type); Assert.AreEqual("SELECT", tokenizer.TokenText); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void advancing_to_unterminated_quoted_string_throws_alt() { var testObject = new QueryTokenizer(); testObject.Start("name 'whatever"); Assert.Throws <ApplicationException>(() => { testObject.Advance(); // Go past name testObject.Advance(); // Go poast whitespace. }); }
public void recognizeTwoStringLiteralLong1() { String input = "'''hello''' '''goodbye'''"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type); Assert.AreEqual("hello", tokenizer.TokenText); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.RDFLiteral, tokenizer.Type); Assert.AreEqual("goodbye", tokenizer.TokenText); }
protected IReadOnlyCollection <string> Run(string query) { var config = new QueryTokenizerConfig( new [] { new ContainerTokens("\"", "\""), new ContainerTokens("'", "'"), }, new [] { "=", "!=", ">", ">=", "<", "<=", "~", "!~" } ); var sut = new QueryTokenizer(config); return(sut.Tokenize(query)); }
public void recognizeVariableFollowedByBrace() { String input = "?var)"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.Variable, tokenizer.Type); Assert.AreEqual("var", tokenizer.TokenText); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void skipWhitespaceBetweenTokens() { String input = "?var1 ?var2 ?var3 \t?var4 \r?var5 \n?var6"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void recognizeQNameFollowedByBrace() { String input = ":x}"; QueryTokenizer tokenizer = new QueryTokenizer(input); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.QName, tokenizer.Type); Assert.AreEqual(":x", tokenizer.TokenText); Assert.AreEqual(true, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EndGroup, tokenizer.Type); Assert.AreEqual(false, tokenizer.MoveNext()); Assert.AreEqual(QueryTokenizer.TokenType.EOF, tokenizer.Type); }
public void tokenizer_recognizes_question_mark_and_name_with_separator() { var testObject = new QueryTokenizer(); var name = "name"; testObject.Start("? " + name); Assert.True(testObject.Token == QueryTokens.QuestionMark); Assert.True(testObject.TokenString == "?"); testObject.Advance(); Assert.True(testObject.Token == QueryTokens.Name); Assert.True(testObject.TokenString == name); }
public void tokenizer_recognizes_greater_than() { var testObject = new QueryTokenizer(); var name = "name"; testObject.Start(">" + name); Assert.True(testObject.Token == QueryTokens.GreaterThan); Assert.True(testObject.TokenString == ">"); testObject.Advance(); Assert.True(testObject.Token == QueryTokens.Name); Assert.True(testObject.TokenString == name); }
public void tokenizer_recognizes_slash() { var testObject = new QueryTokenizer(); var name = "name"; testObject.Start("/" + name); Assert.True(testObject.Token == QueryTokens.Slash); Assert.True(testObject.TokenString == "/"); testObject.Advance(); Assert.True(testObject.Token == QueryTokens.Name); Assert.True(testObject.TokenString == name); }
public void tokenizer_recognizes_colon_and_name_separated_by_whitespace() { var testObject = new QueryTokenizer(); var name = "name"; testObject.Start(": " + name); Assert.True(testObject.Token == QueryTokens.Colon); Assert.True(testObject.TokenString == ":"); testObject.Advance(); Assert.True(testObject.Token == QueryTokens.Name); Assert.True(testObject.TokenString == name); }
public void tokenizer_recognizes_exclamation_mark_and_name_with_whitespace_in_between() { var testObject = new QueryTokenizer(); var name = "name"; testObject.Start("! " + name); Assert.True(testObject.Token == QueryTokens.ExclamationMark); Assert.True(testObject.TokenString == "!"); testObject.Advance(); Assert.True(testObject.Token == QueryTokens.Name); Assert.True(testObject.TokenString == name); }