public virtual void TestToDot()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1, 0, 4) });
StringWriter w = new StringWriter();
(new TokenStreamToDot("abcd", ts, (TextWriter)(w))).ToDot();
Assert.IsTrue(w.ToString().IndexOf("abc / abcd", StringComparison.Ordinal) != -1);
}
public virtual void TestSynHangingOverEnd()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("a", 1, 1), Token("X", 0, 10) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = BasicOperations.Union(BasicAutomata.MakeString("a"), BasicAutomata.MakeString("X"));
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestSingleToken()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = BasicAutomata.MakeString("abc");
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestSynOverHole2()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("xyz", 1, 1), Token("abc", 0, 3), Token("def", 2, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = BasicOperations.Union(Join(S2a("xyz"), SEP_A, HOLE_A, SEP_A, S2a("def")), BasicAutomata.MakeString("abc"));
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestMultipleHoles()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("a", 1, 1), Token("b", 3, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = Join(S2a("a"), SEP_A, HOLE_A, SEP_A, HOLE_A, SEP_A, S2a("b"));
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestTwoTokens()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1), Token("def", 1, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = Join("abc", "def");
//toDot(actual);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestStartsWithHole()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 2, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = Join(HOLE_A, SEP_A, S2a("abc"));
//toDot(actual);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestSynOverHole()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("a", 1, 1), Token("X", 0, 2), Token("b", 2, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton a1 = BasicOperations.Union(Join(S2a("a"), SEP_A, HOLE_A), BasicAutomata.MakeString("X"));
Automaton expected = BasicOperations.Concatenate(a1, Join(SEP_A, S2a("b")));
//toDot(actual);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestHole()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1), Token("def", 2, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = Join(S2a("abc"), SEP_A, HOLE_A, SEP_A, S2a("def"));
//toDot(actual);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestOverlappedTokensSausage()
{
// Two tokens on top of each other (sausage):
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1), Token("xyz", 0, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton a1 = BasicAutomata.MakeString("abc");
Automaton a2 = BasicAutomata.MakeString("xyz");
Automaton expected = BasicOperations.Union(a1, a2);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestOverlappedTokensLattice2()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1), Token("xyz", 0, 3), Token("def", 1, 1), Token("ghi", 1, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton a1 = BasicAutomata.MakeString("xyz");
Automaton a2 = Join("abc", "def", "ghi");
Automaton expected = BasicOperations.Union(a1, a2);
//toDot(actual);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestSingleToken()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = BasicAutomata.MakeString("abc");
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestOverlappedTokensLattice2()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1), Token("xyz", 0, 3), Token("def", 1, 1), Token("ghi", 1, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton a1 = BasicAutomata.MakeString("xyz");
Automaton a2 = Join("abc", "def", "ghi");
Automaton expected = BasicOperations.Union(a1, a2);
//toDot(actual);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestSynOverHole2()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("xyz", 1, 1), Token("abc", 0, 3), Token("def", 2, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = BasicOperations.Union(Join(S2a("xyz"), SEP_A, HOLE_A, SEP_A, S2a("def")), BasicAutomata.MakeString("abc"));
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestSynOverHole()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("a", 1, 1), Token("X", 0, 2), Token("b", 2, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton a1 = BasicOperations.Union(Join(S2a("a"), SEP_A, HOLE_A), BasicAutomata.MakeString("X"));
Automaton expected = BasicOperations.Concatenate(a1, Join(SEP_A, S2a("b")));
//toDot(actual);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestOverlappedTokensSausage()
{
// Two tokens on top of each other (sausage):
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1), Token("xyz", 0, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton a1 = BasicAutomata.MakeString("abc");
Automaton a2 = BasicAutomata.MakeString("xyz");
Automaton expected = BasicOperations.Union(a1, a2);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestSynOverMultipleHoles()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("a", 1, 1), Token("x", 0, 3), Token("b", 3, 1) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton a1 = Join(S2a("a"), SEP_A, HOLE_A, SEP_A, HOLE_A, SEP_A, S2a("b"));
Automaton a2 = Join(S2a("x"), SEP_A, S2a("b"));
Automaton expected = BasicOperations.Union(a1, a2);
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestToDot()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("abc", 1, 1, 0, 4) });
StringWriter w = new StringWriter();
(new TokenStreamToDot("abcd", ts, (TextWriter)(w))).ToDot();
Assert.IsTrue(w.ToString().IndexOf("abc / abcd") != -1);
}
public virtual void TestMixupMultiValued()
{
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
Document doc = new Document();
Field field = new TextField("field", "", Field.Store.NO);
TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
field.TokenStream = ts;
doc.Add(field);
Field field2 = new TextField("field", "", Field.Store.NO);
Token withPayload = new Token("withPayload", 0, 11);
withPayload.Payload = new BytesRef("test");
ts = new CannedTokenStream(withPayload);
Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>());
field2.TokenStream = ts;
doc.Add(field2);
Field field3 = new TextField("field", "", Field.Store.NO);
ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
field3.TokenStream = ts;
doc.Add(field3);
writer.AddDocument(doc);
DirectoryReader reader = writer.Reader;
SegmentReader sr = GetOnlySegmentReader(reader);
DocsAndPositionsEnum de = sr.TermPositionsEnum(new Term("field", "withPayload"));
de.NextDoc();
de.NextPosition();
Assert.AreEqual(new BytesRef("test"), de.Payload);
writer.Dispose();
reader.Dispose();
dir.Dispose();
}
public virtual void TestSynHangingOverEnd()
{
TokenStream ts = new CannedTokenStream(new Token[] { Token("a", 1, 1), Token("X", 0, 10) });
Automaton actual = (new TokenStreamToAutomaton()).ToAutomaton(ts);
Automaton expected = BasicOperations.Union(BasicAutomata.MakeString("a"), BasicAutomata.MakeString("X"));
Assert.IsTrue(BasicOperations.SameLanguage(expected, actual));
}
public virtual void TestMixupDocs()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
Document doc = new Document();
Field field = new TextField("field", "", Field.Store.NO);
TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
field.TokenStream = ts;
doc.Add(field);
writer.AddDocument(doc);
Token withPayload = new Token("withPayload", 0, 11);
withPayload.Payload = new BytesRef("test");
ts = new CannedTokenStream(withPayload);
Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>());
field.TokenStream = ts;
writer.AddDocument(doc);
ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true);
Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
field.TokenStream = ts;
writer.AddDocument(doc);
DirectoryReader reader = writer.Reader;
AtomicReader sr = SlowCompositeReaderWrapper.Wrap(reader);
DocsAndPositionsEnum de = sr.TermPositionsEnum(new Term("field", "withPayload"));
de.NextDoc();
de.NextPosition();
Assert.AreEqual(new BytesRef("test"), de.Payload);
writer.Dispose();
reader.Dispose();
dir.Dispose();
}