public IFilter Parse()
{
var tokenizer = new Tokenizer(Filter);
tokens = tokenizer.Tokenize();
pos = -2;
string conjunction;
ListFilter.JoinType joinType;
FilterSet filter = new FilterAndSet();
IFilter thisFilter;
IFilter last;
FilterOrSet orSet;
while (++pos < tokens.Count)
{
conjunction = pos == -1 ? "," : tokens[pos];
var name = GetNextToken("field name");
var oper = GetNextToken("operator");
var value = GetNextToken("value");
if (Tokens.IsListStart(value))
{
joinType = ListFilter.JoinType.And;
var values = new List<string>();
while (!Tokens.IsListEnd(tokens[pos]) && !Tokens.IsListEnd(GetNextToken("value")))
{
values.Add(tokens[pos]);
if (GetNextToken("conjunction or list end") == "|")
joinType = ListFilter.JoinType.Or;
}
thisFilter = new ListFilter(name, oper, values, joinType);
}
else
thisFilter = new Filter(name, oper, value);
if (conjunction == ",") // and
filter.Add(thisFilter);
else if (conjunction == "|") // or
{
last = filter.Last();
if (!(last is FilterOrSet))
{
filter.RemoveAt(filter.Count - 1);
orSet = new FilterOrSet();
orSet.Add(last);
orSet.Add(thisFilter);
filter.Add(orSet);
}
else
((FilterOrSet)last).Add(thisFilter);
}
else
throw new ParseException("Parser: Unexpected '{0}', expected conjunction instead.", conjunction);
}
return filter;
}
private void AssertTokenizer(Tokenizer tokenizer, string[] expected)
{
var tokens = tokenizer.Tokenize();
int i = 0;
foreach (var expectedItem in expected)
Assert.Equal(expectedItem, tokens[i++]);
Assert.Equal(i, tokens.Count);
}
public void Tokenizer()
{
Tokenizer tokenizer;
string[] expectedTokens;
tokenizer = new Tokenizer(@"Field=Value,AnotherField $= 'AnotherValue'| NumericField> 5, DoubleQuoted ^= ""double quoted""");
expectedTokens = new string[] { "Field", "=", "Value", ",", "AnotherField", "$=", "AnotherValue", "|", "NumericField", ">", "5", ",",
"DoubleQuoted", "^=", "double quoted" };
AssertTokenizer(tokenizer, expectedTokens);
tokenizer = new Tokenizer(@"EscapedField='hoi\'test\\more' ,ComplicatedField=""test\\\"""" | withoutQuotes=hello\,more");
expectedTokens = new string[] { "EscapedField", "=", @"hoi'test\more", ",", "ComplicatedField", "=", @"test\""", "|",
"withoutQuotes", "=", "hello,more" };
AssertTokenizer(tokenizer, expectedTokens);
tokenizer = new Tokenizer(@"Field=Name With Whitespace, AnotherField ^= Prefixed and Suffixed space | QuotedField=' Includes space '");
expectedTokens = new string[] { "Field", "=", "Name With Whitespace", ",", "AnotherField", "^=", "Prefixed and Suffixed space", "|",
"QuotedField", "=", " Includes space " };
AssertTokenizer(tokenizer, expectedTokens);
tokenizer = new Tokenizer(@"SomeField = Contains A Value, FirstList = [ 'hello',another , more values ], PipeList*=[xyz|abc def|ghi|'quot|ed']");
expectedTokens = new string[] { "SomeField", "=", "Contains A Value", ",", "FirstList", "=", "[", "hello", ",", "another", ",", "more values", "]", ",",
"PipeList", "*=", "[", "xyz", "|", "abc def", "|", "ghi", "|", "quot|ed", "]" };
AssertTokenizer(tokenizer, expectedTokens);
tokenizer = new Tokenizer(@"Field = [Item1, Item2, Item3] | SecondField = [Item4 | Item5], ThirdField = [Item6]");
expectedTokens = new string[] { "Field", "=", "[", "Item1", ",", "Item2", ",", "Item3", "]", "|", "SecondField", "=", "[", "Item4", "|", "Item5", "]",
",", "ThirdField", "=", "Item6", "]" };
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, NoValue=").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Missing'Operator'").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Missing='Operator', InSecond'Field'").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Unexpected=End,").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Unexpected=End,").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed='test', Invalid='Quoting").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed='test', Invalid='Quoting\'").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=""test"", Invalid='Quoting""").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Field=[Invalid|Mixing,Conjunctions]").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"WithoutValue= ").Tokenize());
Assert.DoesNotThrow(() => new Tokenizer(@"WithoutValue=' '").Tokenize());
}
public IFilter Parse()
{
var tokenizer = new Tokenizer(Filter);
tokens = tokenizer.Tokenize();
pos = -2;
string conjunction;
FilterSet filter = new FilterAndSet();
Filter thisFilter;
IFilter last;
FilterOrSet orSet;
while (++pos < tokens.Count)
{
conjunction = pos == -1 ? "," : tokens[pos];
thisFilter = new Filter(GetNextToken("field name"), GetNextToken("operator"), GetNextToken("value"));
if (conjunction == ",") // and
filter.Add(thisFilter);
else if (conjunction == "|") // or
{
last = filter.Last();
if (!(last is FilterOrSet))
{
filter.RemoveAt(filter.Count - 1);
orSet = new FilterOrSet();
orSet.Add(last);
orSet.Add(thisFilter);
filter.Add(orSet);
}
else
((FilterOrSet)last).Add(thisFilter);
}
else
throw new ParseException("Parser: Unexpected '{0}', expected conjunction instead.", conjunction);
}
return filter;
}
public void Tokenizer()
{
Tokenizer tokenizer;
List<string> tokens;
tokenizer = new Tokenizer(@"Field=Value,AnotherField $= 'AnotherValue'| NumericField> 5, DoubleQuoted ^= ""double quoted""");
tokens = tokenizer.Tokenize();
Assert.Equal("Field", tokens[0]);
Assert.Equal("=", tokens[1]);
Assert.Equal("Value", tokens[2]);
Assert.Equal(",", tokens[3]);
Assert.Equal("AnotherField", tokens[4]);
Assert.Equal("$=", tokens[5]);
Assert.Equal("AnotherValue", tokens[6]);
Assert.Equal("|", tokens[7]);
Assert.Equal("NumericField", tokens[8]);
Assert.Equal(">", tokens[9]);
Assert.Equal("5", tokens[10]);
Assert.Equal(",", tokens[11]);
Assert.Equal("DoubleQuoted", tokens[12]);
Assert.Equal("^=", tokens[13]);
Assert.Equal("double quoted", tokens[14]);
Assert.Equal(15, tokens.Count);
tokenizer = new Tokenizer(@"EscapedField='hoi\'test\\more' ,ComplicatedField=""test\\\"""" | withoutQuotes=hello\,more");
tokens = tokenizer.Tokenize();
Assert.Equal("EscapedField", tokens[0]);
Assert.Equal("=", tokens[1]);
Assert.Equal(@"hoi'test\more", tokens[2]);
Assert.Equal(",", tokens[3]);
Assert.Equal("ComplicatedField", tokens[4]);
Assert.Equal("=", tokens[5]);
Assert.Equal(@"test\""", tokens[6]);
Assert.Equal("|", tokens[7]);
Assert.Equal("withoutQuotes", tokens[8]);
Assert.Equal("=", tokens[9]);
Assert.Equal("hello,more", tokens[10]);
Assert.Equal(11, tokens.Count);
tokenizer = new Tokenizer(@"Field=Name With Whitespace, AnotherField ^= Prefixed and Suffixed space | QuotedField=' Includes space '");
tokens = tokenizer.Tokenize();
Assert.Equal("Field", tokens[0]);
Assert.Equal("=", tokens[1]);
Assert.Equal("Name With Whitespace", tokens[2]);
Assert.Equal(",", tokens[3]);
Assert.Equal("AnotherField", tokens[4]);
Assert.Equal("^=", tokens[5]);
Assert.Equal("Prefixed and Suffixed space", tokens[6]);
Assert.Equal("|", tokens[7]);
Assert.Equal("QuotedField", tokens[8]);
Assert.Equal("=", tokens[9]);
Assert.Equal(" Includes space ", tokens[10]);
Assert.Equal(11, tokens.Count);
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, NoValue=").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Missing'Operator'").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Missing='Operator', InSecond'Field'").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Unexpected=End,").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=test, Unexpected=End,").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed='test', Invalid='Quoting").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed='test', Invalid='Quoting\'").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"Seed=""test"", Invalid='Quoting""").Tokenize());
Assert.Throws<ParseException>(() => new Tokenizer(@"WithoutValue= ").Tokenize());
Assert.DoesNotThrow(() => new Tokenizer(@"WithoutValue=' '").Tokenize());
}
