/// <summary> Writes the document to the directory in the given segment</summary> /// <param name="">dir /// </param> /// <param name="">segment /// </param> /// <param name="">doc /// </param> public static void WriteDoc(Directory dir, System.String segment, Document doc) { Analyzer analyzer = new WhitespaceAnalyzer(); Similarity similarity = Similarity.GetDefault(); WriteDoc(dir, analyzer, similarity, segment, doc); }
public static Lucene.Net.Analysis.Analyzer GetAnalyzerByName(string analyzerName) { Lucene.Net.Analysis.Analyzer result; Lucene.Net.Util.Version AppLuceneVersion = Lucene.Net.Util.Version.LUCENE_30; switch (analyzerName) { case "SimpleAnalyzer": result = new Lucene.Net.Analysis.SimpleAnalyzer(); break; case "StandardAnalyzer": result = new Lucene.Net.Analysis.Standard.StandardAnalyzer(AppLuceneVersion); break; case "KeywordAnalyzer": result = new Lucene.Net.Analysis.KeywordAnalyzer(); break; case "StopAnalyzer": result = new Lucene.Net.Analysis.StopAnalyzer(AppLuceneVersion); break; case "WhitespaceAnalyzer": result = new Lucene.Net.Analysis.WhitespaceAnalyzer(); break; default: result = new Lucene.Net.Analysis.SimpleAnalyzer(); break; } return(result); }
public virtual void TestEscaped() { Analyzer a = new WhitespaceAnalyzer(); /*AssertQueryEquals("\\[brackets", a, "\\[brackets"); * AssertQueryEquals("\\[brackets", null, "brackets"); * AssertQueryEquals("\\\\", a, "\\\\"); * AssertQueryEquals("\\+blah", a, "\\+blah"); * AssertQueryEquals("\\(blah", a, "\\(blah"); * * AssertQueryEquals("\\-blah", a, "\\-blah"); * AssertQueryEquals("\\!blah", a, "\\!blah"); * AssertQueryEquals("\\{blah", a, "\\{blah"); * AssertQueryEquals("\\}blah", a, "\\}blah"); * AssertQueryEquals("\\:blah", a, "\\:blah"); * AssertQueryEquals("\\^blah", a, "\\^blah"); * AssertQueryEquals("\\[blah", a, "\\[blah"); * AssertQueryEquals("\\]blah", a, "\\]blah"); * AssertQueryEquals("\\\"blah", a, "\\\"blah"); * AssertQueryEquals("\\(blah", a, "\\(blah"); * AssertQueryEquals("\\)blah", a, "\\)blah"); * AssertQueryEquals("\\~blah", a, "\\~blah"); * AssertQueryEquals("\\*blah", a, "\\*blah"); * AssertQueryEquals("\\?blah", a, "\\?blah"); * //AssertQueryEquals("foo \\&\\& bar", a, "foo \\&\\& bar"); * //AssertQueryEquals("foo \\|| bar", a, "foo \\|| bar"); * //AssertQueryEquals("foo \\AND bar", a, "foo \\AND bar");*/ AssertQueryEquals("a\\-b:c", a, "a-b:c"); AssertQueryEquals("a\\+b:c", a, "a+b:c"); AssertQueryEquals("a\\:b:c", a, "a:b:c"); AssertQueryEquals("a\\\\b:c", a, "a\\b:c"); AssertQueryEquals("a:b\\-c", a, "a:b-c"); AssertQueryEquals("a:b\\+c", a, "a:b+c"); AssertQueryEquals("a:b\\:c", a, "a:b:c"); AssertQueryEquals("a:b\\\\c", a, "a:b\\c"); AssertQueryEquals("a:b\\-c*", a, "a:b-c*"); AssertQueryEquals("a:b\\+c*", a, "a:b+c*"); AssertQueryEquals("a:b\\:c*", a, "a:b:c*"); AssertQueryEquals("a:b\\\\c*", a, "a:b\\c*"); AssertQueryEquals("a:b\\-?c", a, "a:b-?c"); AssertQueryEquals("a:b\\+?c", a, "a:b+?c"); AssertQueryEquals("a:b\\:?c", a, "a:b:?c"); AssertQueryEquals("a:b\\\\?c", a, "a:b\\?c"); AssertQueryEquals("a:b\\-c~", a, "a:b-c~0.5"); AssertQueryEquals("a:b\\+c~", a, "a:b+c~0.5"); AssertQueryEquals("a:b\\:c~", a, "a:b:c~0.5"); AssertQueryEquals("a:b\\\\c~", a, "a:b\\c~0.5"); AssertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]"); AssertQueryEquals("[ a\\: TO a\\~ ]", null, "[a: TO a~]"); AssertQueryEquals("[ a\\\\ TO a\\* ]", null, "[a\\ TO a*]"); }
public virtual void TestPunct() { Analyzer a = new WhitespaceAnalyzer(); AssertQueryEquals("a&b", a, "a&b"); AssertQueryEquals("a&&b", a, "a&&b"); AssertQueryEquals(".NET", a, ".NET"); }
public virtual void TestAddDocument() { Document testDoc = new Document(); DocHelper.SetupDoc(testDoc); Analyzer analyzer = new WhitespaceAnalyzer(); IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); writer.AddDocument(testDoc); writer.Flush(); SegmentInfo info = writer.NewestSegment(); writer.Close(); //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.Get(info); Assert.IsTrue(reader != null); Document doc = reader.Document(0); Assert.IsTrue(doc != null); //System.out.println("Document: " + doc); Fieldable[] fields = doc.GetFields("textField2"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_2_TEXT)); Assert.IsTrue(fields[0].IsTermVectorStored()); fields = doc.GetFields("textField1"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_1_TEXT)); Assert.IsFalse(fields[0].IsTermVectorStored()); fields = doc.GetFields("keyField"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.KEYWORD_TEXT)); fields = doc.GetFields(DocHelper.NO_NORMS_KEY); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.NO_NORMS_TEXT)); fields = doc.GetFields(DocHelper.TEXT_FIELD_3_KEY); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_3_TEXT)); // test that the norms are not present in the segment if // omitNorms is true for (int i = 0; i < reader.core_ForNUnit.fieldInfos_ForNUnit.Size(); i++) { FieldInfo fi = reader.core_ForNUnit.fieldInfos_ForNUnit.FieldInfo(i); if (fi.isIndexed_ForNUnit) { Assert.IsTrue(fi.omitNorms_ForNUnit == !reader.HasNorms(fi.name_ForNUnit)); } } }
public virtual void TestIncrementingPositions() { Analyzer analyzer = new WhitespaceAnalyzer(); TokenStream ts = analyzer.TokenStream("Field", new System.IO.StringReader("one two three four five")); while (true) { Token token = ts.Next(); if (token == null) { break; } Assert.AreEqual(1, token.GetPositionIncrement(), token.TermText()); } }
public void Test_SegmentTermVector_IndexOf() { Lucene.Net.Store.RAMDirectory directory = new Lucene.Net.Store.RAMDirectory(); Lucene.Net.Analysis.Analyzer analyzer = new Lucene.Net.Analysis.WhitespaceAnalyzer(); Lucene.Net.Index.IndexWriter writer = new Lucene.Net.Index.IndexWriter(directory, analyzer, Lucene.Net.Index.IndexWriter.MaxFieldLength.LIMITED); Lucene.Net.Documents.Document document = new Lucene.Net.Documents.Document(); document.Add(new Lucene.Net.Documents.Field("contents", new System.IO.StreamReader(new System.IO.MemoryStream(System.Text.Encoding.ASCII.GetBytes("a_ a0"))), Lucene.Net.Documents.Field.TermVector.WITH_OFFSETS)); writer.AddDocument(document); Lucene.Net.Index.IndexReader reader = writer.GetReader(); Lucene.Net.Index.TermPositionVector tpv = reader.GetTermFreqVector(0, "contents") as Lucene.Net.Index.TermPositionVector; //Console.WriteLine("tpv: " + tpv); int index = tpv.IndexOf("a_"); Assert.AreEqual(index, 1, "See the issue: LUCENENET-183"); }
public virtual void TestAddDocument() { Analyzer analyzer = new WhitespaceAnalyzer(); Similarity similarity = Similarity.GetDefault(); DocumentWriter writer = new DocumentWriter(dir, analyzer, similarity, 50); Assert.IsTrue(writer != null); try { writer.AddDocument("test", testDoc); //After adding the document, we should be able to read it back in SegmentReader reader = new SegmentReader(new SegmentInfo("test", 1, dir)); Assert.IsTrue(reader != null); Document doc = reader.Document(0); Assert.IsTrue(doc != null); //System.out.println("Document: " + doc); Field[] fields = doc.GetFields("textField2"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_2_TEXT)); Assert.IsTrue(fields[0].IsTermVectorStored() == true); fields = doc.GetFields("textField1"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_1_TEXT)); Assert.IsTrue(fields[0].IsTermVectorStored() == false); fields = doc.GetFields("keyField"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.KEYWORD_TEXT)); } catch (System.IO.IOException e) { System.Console.Error.WriteLine(e.StackTrace); Assert.IsTrue(false); } }
public virtual void TestAddDocument() { Analyzer analyzer = new WhitespaceAnalyzer(); Similarity similarity = Similarity.GetDefault(); DocumentWriter writer = new DocumentWriter(dir, analyzer, similarity, 50); Assert.IsTrue(writer != null); try { writer.AddDocument("test", testDoc); //After adding the document, we should be able to read it back in SegmentReader reader = new SegmentReader(new SegmentInfo("test", 1, dir)); Assert.IsTrue(reader != null); Document doc = reader.Document(0); Assert.IsTrue(doc != null); //System.out.println("Document: " + doc); Field[] fields = doc.GetFields("textField2"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_2_TEXT)); Assert.IsTrue(fields[0].IsTermVectorStored() == true); fields = doc.GetFields("textField1"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_1_TEXT)); Assert.IsTrue(fields[0].IsTermVectorStored() == false); fields = doc.GetFields("keyField"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.KEYWORD_TEXT)); } catch (System.IO.IOException e) { System.Console.Error.WriteLine(e.StackTrace); Assert.IsTrue(false); } }
private float CheckPhraseQuery(Document doc, PhraseQuery query, int slop, int expectedNumResults) { query.SetSlop(slop); RAMDirectory ramDir = new RAMDirectory(); WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(); IndexWriter writer = new IndexWriter(ramDir, analyzer, MaxFieldLength.UNLIMITED); writer.AddDocument(doc); writer.Close(); IndexSearcher searcher = new IndexSearcher(ramDir); TopDocs td = searcher.Search(query, null, 10); //System.out.println("slop: "+slop+" query: "+query+" doc: "+doc+" Expecting number of hits: "+expectedNumResults+" maxScore="+td.getMaxScore()); Assert.AreEqual(expectedNumResults, td.TotalHits, "slop: " + slop + " query: " + query + " doc: " + doc + " Wrong number of hits"); //QueryUtils.check(query,searcher); searcher.Close(); ramDir.Close(); return(td.GetMaxScore()); }
public virtual void TestPunct() { Analyzer a = new WhitespaceAnalyzer(); AssertQueryEquals("a&b", a, "a&b"); AssertQueryEquals("a&&b", a, "a&&b"); AssertQueryEquals(".NET", a, ".NET"); }
public void Test_SegmentTermVector_IndexOf() { Lucene.Net.Store.RAMDirectory directory = new Lucene.Net.Store.RAMDirectory(); Lucene.Net.Analysis.Analyzer analyzer = new Lucene.Net.Analysis.WhitespaceAnalyzer(); Lucene.Net.Index.IndexWriter writer = new Lucene.Net.Index.IndexWriter(directory, analyzer, Lucene.Net.Index.IndexWriter.MaxFieldLength.LIMITED); Lucene.Net.Documents.Document document = new Lucene.Net.Documents.Document(); document.Add(new Lucene.Net.Documents.Field("contents", new System.IO.StreamReader(new System.IO.MemoryStream(System.Text.Encoding.ASCII.GetBytes("a_ a0"))), Lucene.Net.Documents.Field.TermVector.WITH_OFFSETS)); writer.AddDocument(document); Lucene.Net.Index.IndexReader reader = writer.GetReader(); Lucene.Net.Index.TermPositionVector tpv = reader.GetTermFreqVector(0, "contents") as Lucene.Net.Index.TermPositionVector; //Console.WriteLine("tpv: " + tpv); int index = tpv.IndexOf("a_"); Assert.AreEqual(index, 1, "See the issue: LUCENENET-183"); }
private float CheckPhraseQuery(Document doc, PhraseQuery query, int slop, int expectedNumResults) { query.SetSlop(slop); RAMDirectory ramDir = new RAMDirectory(); WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(); IndexWriter writer = new IndexWriter(ramDir, analyzer, MaxFieldLength.UNLIMITED); writer.AddDocument(doc); writer.Close(); IndexSearcher searcher = new IndexSearcher(ramDir); TopDocs td = searcher.Search(query, null, 10); //System.out.println("slop: "+slop+" query: "+query+" doc: "+doc+" Expecting number of hits: "+expectedNumResults+" maxScore="+td.getMaxScore()); Assert.AreEqual(expectedNumResults, td.totalHits, "slop: " + slop + " query: " + query + " doc: " + doc + " Wrong number of hits"); //QueryUtils.check(query,searcher); searcher.Close(); ramDir.Close(); return td.GetMaxScore(); }
/// <summary> Writes the document to the directory in the given segment</summary> /// <param name="">dir /// </param> /// <param name="">segment /// </param> /// <param name="">doc /// </param> public static void WriteDoc(Directory dir, System.String segment, Document doc) { Analyzer analyzer = new WhitespaceAnalyzer(); Similarity similarity = Similarity.GetDefault(); WriteDoc(dir, analyzer, similarity, segment, doc); }
public virtual void TestEscaped() { Analyzer a = new WhitespaceAnalyzer(); /*AssertQueryEquals("\\[brackets", a, "\\[brackets"); AssertQueryEquals("\\[brackets", null, "brackets"); AssertQueryEquals("\\\\", a, "\\\\"); AssertQueryEquals("\\+blah", a, "\\+blah"); AssertQueryEquals("\\(blah", a, "\\(blah"); AssertQueryEquals("\\-blah", a, "\\-blah"); AssertQueryEquals("\\!blah", a, "\\!blah"); AssertQueryEquals("\\{blah", a, "\\{blah"); AssertQueryEquals("\\}blah", a, "\\}blah"); AssertQueryEquals("\\:blah", a, "\\:blah"); AssertQueryEquals("\\^blah", a, "\\^blah"); AssertQueryEquals("\\[blah", a, "\\[blah"); AssertQueryEquals("\\]blah", a, "\\]blah"); AssertQueryEquals("\\\"blah", a, "\\\"blah"); AssertQueryEquals("\\(blah", a, "\\(blah"); AssertQueryEquals("\\)blah", a, "\\)blah"); AssertQueryEquals("\\~blah", a, "\\~blah"); AssertQueryEquals("\\*blah", a, "\\*blah"); AssertQueryEquals("\\?blah", a, "\\?blah"); //AssertQueryEquals("foo \\&\\& bar", a, "foo \\&\\& bar"); //AssertQueryEquals("foo \\|| bar", a, "foo \\|| bar"); //AssertQueryEquals("foo \\AND bar", a, "foo \\AND bar");*/ AssertQueryEquals("a\\-b:c", a, "a-b:c"); AssertQueryEquals("a\\+b:c", a, "a+b:c"); AssertQueryEquals("a\\:b:c", a, "a:b:c"); AssertQueryEquals("a\\\\b:c", a, "a\\b:c"); AssertQueryEquals("a:b\\-c", a, "a:b-c"); AssertQueryEquals("a:b\\+c", a, "a:b+c"); AssertQueryEquals("a:b\\:c", a, "a:b:c"); AssertQueryEquals("a:b\\\\c", a, "a:b\\c"); AssertQueryEquals("a:b\\-c*", a, "a:b-c*"); AssertQueryEquals("a:b\\+c*", a, "a:b+c*"); AssertQueryEquals("a:b\\:c*", a, "a:b:c*"); AssertQueryEquals("a:b\\\\c*", a, "a:b\\c*"); AssertQueryEquals("a:b\\-?c", a, "a:b-?c"); AssertQueryEquals("a:b\\+?c", a, "a:b+?c"); AssertQueryEquals("a:b\\:?c", a, "a:b:?c"); AssertQueryEquals("a:b\\\\?c", a, "a:b\\?c"); AssertQueryEquals("a:b\\-c~", a, "a:b-c~0.5"); AssertQueryEquals("a:b\\+c~", a, "a:b+c~0.5"); AssertQueryEquals("a:b\\:c~", a, "a:b:c~0.5"); AssertQueryEquals("a:b\\\\c~", a, "a:b\\c~0.5"); AssertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]"); AssertQueryEquals("[ a\\: TO a\\~ ]", null, "[a: TO a~]"); AssertQueryEquals("[ a\\\\ TO a\\* ]", null, "[a\\ TO a*]"); }
public virtual void TestIncrementingPositions() { Analyzer analyzer = new WhitespaceAnalyzer(); TokenStream ts = analyzer.TokenStream("field", new System.IO.StringReader("one two three four five")); while (true) { Token token = ts.Next(); if (token == null) break; Assert.AreEqual(1, token.GetPositionIncrement(), token.TermText()); } }
public virtual void TestQueryStringEscaping() { Analyzer a = new WhitespaceAnalyzer(); AssertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c"); AssertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c"); AssertEscapedQueryEquals("a:b:c", a, "a\\:b\\:c"); AssertEscapedQueryEquals("a\\b:c", a, "a\\\\b\\:c"); AssertEscapedQueryEquals("a:b-c", a, "a\\:b\\-c"); AssertEscapedQueryEquals("a:b+c", a, "a\\:b\\+c"); AssertEscapedQueryEquals("a:b:c", a, "a\\:b\\:c"); AssertEscapedQueryEquals("a:b\\c", a, "a\\:b\\\\c"); AssertEscapedQueryEquals("a:b-c*", a, "a\\:b\\-c\\*"); AssertEscapedQueryEquals("a:b+c*", a, "a\\:b\\+c\\*"); AssertEscapedQueryEquals("a:b:c*", a, "a\\:b\\:c\\*"); AssertEscapedQueryEquals("a:b\\\\c*", a, "a\\:b\\\\\\\\c\\*"); AssertEscapedQueryEquals("a:b-?c", a, "a\\:b\\-\\?c"); AssertEscapedQueryEquals("a:b+?c", a, "a\\:b\\+\\?c"); AssertEscapedQueryEquals("a:b:?c", a, "a\\:b\\:\\?c"); AssertEscapedQueryEquals("a:b?c", a, "a\\:b\\?c"); AssertEscapedQueryEquals("a:b-c~", a, "a\\:b\\-c\\~"); AssertEscapedQueryEquals("a:b+c~", a, "a\\:b\\+c\\~"); AssertEscapedQueryEquals("a:b:c~", a, "a\\:b\\:c\\~"); AssertEscapedQueryEquals("a:b\\c~", a, "a\\:b\\\\c\\~"); AssertEscapedQueryEquals("[ a - TO a+ ]", null, "\\[ a \\- TO a\\+ \\]"); AssertEscapedQueryEquals("[ a : TO a~ ]", null, "\\[ a \\: TO a\\~ \\]"); AssertEscapedQueryEquals("[ a\\ TO a* ]", null, "\\[ a\\\\ TO a\\* \\]"); // LUCENE-881 AssertEscapedQueryEquals("|| abc ||", a, "\\|\\| abc \\|\\|"); AssertEscapedQueryEquals("&& abc &&", a, "\\&\\& abc \\&\\&"); }
override public void Run() { IndexSearcher searcher = null; WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(); Query query = new TermQuery(new Term("content", "aaa")); for (int i = 0; i < this.numIteration; i++) { try { searcher = new IndexSearcher(dir); } catch (System.Exception e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Searcher: create hit unexpected exception: " + e.ToString()); System.Console.Error.WriteLine(e.StackTrace); break; } if (searcher != null) { Hits hits = null; try { hits = searcher.Search(query); } catch (System.IO.IOException e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Searcher: search hit unexpected exception: " + e.ToString()); System.Console.Error.WriteLine(e.StackTrace); break; } // System.out.println(hits.length() + " total results"); try { searcher.Close(); } catch (System.IO.IOException e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Searcher: close hit unexpected exception: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); break; } searcher = null; } } }
override public void Run() { WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(); IndexWriter writer = null; for (int i = 0; i < this.numIteration; i++) { try { writer = new IndexWriter(dir, analyzer, false, IndexWriter.MaxFieldLength.LIMITED); } catch (System.IO.IOException e) { if (e.ToString().IndexOf(" timed out:") == -1) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: creation hit unexpected IOException: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); } else { // lock obtain timed out // NOTE: we should at some point // consider this a failure? The lock // obtains, across IndexReader & // IndexWriters should be "fair" (ie // FIFO). } } catch (System.Exception e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: creation hit unexpected exception: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); break; } if (writer != null) { try { Enclosing_Instance.AddDoc(writer); } catch (System.IO.IOException e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: addDoc hit unexpected exception: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); break; } try { writer.Close(); } catch (System.IO.IOException e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: close hit unexpected exception: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); break; } writer = null; } } }
override public void Run() { WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(); IndexWriter writer = null; for (int i = 0; i < this.numIteration; i++) { try { writer = new IndexWriter(dir, analyzer, false, IndexWriter.MaxFieldLength.LIMITED); } catch (System.IO.IOException e) { if (e.ToString().IndexOf(" timed out:") == - 1) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: creation hit unexpected IOException: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); } else { // lock obtain timed out // NOTE: we should at some point // consider this a failure? The lock // obtains, across IndexReader & // IndexWriters should be "fair" (ie // FIFO). } } catch (System.Exception e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: creation hit unexpected exception: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); break; } if (writer != null) { try { Enclosing_Instance.AddDoc(writer); } catch (System.IO.IOException e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: addDoc hit unexpected exception: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); break; } try { writer.Close(); } catch (System.IO.IOException e) { hitException = true; System.Console.Out.WriteLine("Stress Test Index Writer: close hit unexpected exception: " + e.ToString()); System.Console.Out.WriteLine(e.StackTrace); break; } writer = null; } } }
public virtual void TestEndOffsetPositionWithTeeSinkTokenFilter() { MockRAMDirectory dir = new MockRAMDirectory(); Analyzer analyzer = new WhitespaceAnalyzer(); IndexWriter w = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); TeeSinkTokenFilter tee = new TeeSinkTokenFilter(analyzer.TokenStream("field", new System.IO.StringReader("abcd "))); TokenStream sink = tee.NewSinkTokenStream(); Field f1 = new Field("field", tee, Field.TermVector.WITH_POSITIONS_OFFSETS); Field f2 = new Field("field", sink, Field.TermVector.WITH_POSITIONS_OFFSETS); doc.Add(f1); doc.Add(f2); w.AddDocument(doc); w.Close(); IndexReader r = IndexReader.Open(dir); TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.GetTermFreqVector(0, "field")).GetOffsets(0); Assert.AreEqual(2, termOffsets.Length); Assert.AreEqual(0, termOffsets[0].GetStartOffset()); Assert.AreEqual(4, termOffsets[0].GetEndOffset()); Assert.AreEqual(8, termOffsets[1].GetStartOffset()); Assert.AreEqual(12, termOffsets[1].GetEndOffset()); r.Close(); dir.Close(); }
public virtual void TestAddDocument() { Document testDoc = new Document(); DocHelper.SetupDoc(testDoc); Analyzer analyzer = new WhitespaceAnalyzer(); IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); writer.AddDocument(testDoc); writer.Flush(); SegmentInfo info = writer.NewestSegment(); writer.Close(); //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.Get(info); Assert.IsTrue(reader != null); Document doc = reader.Document(0); Assert.IsTrue(doc != null); //System.out.println("Document: " + doc); Fieldable[] fields = doc.GetFields("textField2"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_2_TEXT)); Assert.IsTrue(fields[0].IsTermVectorStored()); fields = doc.GetFields("textField1"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_1_TEXT)); Assert.IsFalse(fields[0].IsTermVectorStored()); fields = doc.GetFields("keyField"); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.KEYWORD_TEXT)); fields = doc.GetFields(DocHelper.NO_NORMS_KEY); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.NO_NORMS_TEXT)); fields = doc.GetFields(DocHelper.TEXT_FIELD_3_KEY); Assert.IsTrue(fields != null && fields.Length == 1); Assert.IsTrue(fields[0].StringValue().Equals(DocHelper.FIELD_3_TEXT)); // test that the norms are not present in the segment if // omitNorms is true for (int i = 0; i < reader.core_ForNUnit.fieldInfos_ForNUnit.Size(); i++) { FieldInfo fi = reader.core_ForNUnit.fieldInfos_ForNUnit.FieldInfo(i); if (fi.isIndexed_ForNUnit) { Assert.IsTrue(fi.omitNorms_ForNUnit == !reader.HasNorms(fi.name_ForNUnit)); } } }
public virtual void TestEscaped() { Analyzer a = new WhitespaceAnalyzer(); /*assertQueryEquals("\\[brackets", a, "\\[brackets"); assertQueryEquals("\\[brackets", null, "brackets"); assertQueryEquals("\\\\", a, "\\\\"); assertQueryEquals("\\+blah", a, "\\+blah"); assertQueryEquals("\\(blah", a, "\\(blah"); assertQueryEquals("\\-blah", a, "\\-blah"); assertQueryEquals("\\!blah", a, "\\!blah"); assertQueryEquals("\\{blah", a, "\\{blah"); assertQueryEquals("\\}blah", a, "\\}blah"); assertQueryEquals("\\:blah", a, "\\:blah"); assertQueryEquals("\\^blah", a, "\\^blah"); assertQueryEquals("\\[blah", a, "\\[blah"); assertQueryEquals("\\]blah", a, "\\]blah"); assertQueryEquals("\\\"blah", a, "\\\"blah"); assertQueryEquals("\\(blah", a, "\\(blah"); assertQueryEquals("\\)blah", a, "\\)blah"); assertQueryEquals("\\~blah", a, "\\~blah"); assertQueryEquals("\\*blah", a, "\\*blah"); assertQueryEquals("\\?blah", a, "\\?blah"); //assertQueryEquals("foo \\&\\& bar", a, "foo \\&\\& bar"); //assertQueryEquals("foo \\|| bar", a, "foo \\|| bar"); //assertQueryEquals("foo \\AND bar", a, "foo \\AND bar");*/ AssertQueryEquals("\\a", a, "a"); AssertQueryEquals("a\\-b:c", a, "a-b:c"); AssertQueryEquals("a\\+b:c", a, "a+b:c"); AssertQueryEquals("a\\:b:c", a, "a:b:c"); AssertQueryEquals("a\\\\b:c", a, "a\\b:c"); AssertQueryEquals("a:b\\-c", a, "a:b-c"); AssertQueryEquals("a:b\\+c", a, "a:b+c"); AssertQueryEquals("a:b\\:c", a, "a:b:c"); AssertQueryEquals("a:b\\\\c", a, "a:b\\c"); AssertQueryEquals("a:b\\-c*", a, "a:b-c*"); AssertQueryEquals("a:b\\+c*", a, "a:b+c*"); AssertQueryEquals("a:b\\:c*", a, "a:b:c*"); AssertQueryEquals("a:b\\\\c*", a, "a:b\\c*"); AssertQueryEquals("a:b\\-?c", a, "a:b-?c"); AssertQueryEquals("a:b\\+?c", a, "a:b+?c"); AssertQueryEquals("a:b\\:?c", a, "a:b:?c"); AssertQueryEquals("a:b\\\\?c", a, "a:b\\?c"); AssertQueryEquals("a:b\\-c~", a, "a:b-c~0.5"); AssertQueryEquals("a:b\\+c~", a, "a:b+c~0.5"); AssertQueryEquals("a:b\\:c~", a, "a:b:c~0.5"); AssertQueryEquals("a:b\\\\c~", a, "a:b\\c~0.5"); AssertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]"); AssertQueryEquals("[ a\\: TO a\\~ ]", null, "[a: TO a~]"); AssertQueryEquals("[ a\\\\ TO a\\* ]", null, "[a\\ TO a*]"); AssertQueryEquals("[\"c\\:\\\\temp\\\\\\~foo0.txt\" TO \"c\\:\\\\temp\\\\\\~foo9.txt\"]", a, "[c:\\temp\\~foo0.txt TO c:\\temp\\~foo9.txt]"); AssertQueryEquals("a\\\\\\+b", a, "a\\+b"); AssertQueryEquals("a \\\"b c\\\" d", a, "a \"b c\" d"); AssertQueryEquals("\"a \\\"b c\\\" d\"", a, "\"a \"b c\" d\""); AssertQueryEquals("\"a \\+b c d\"", a, "\"a +b c d\""); AssertQueryEquals("c\\:\\\\temp\\\\\\~foo.txt", a, "c:\\temp\\~foo.txt"); AssertParseException("XY\\"); // there must be a character after the escape char // test unicode escaping AssertQueryEquals("a\\u0062c", a, "abc"); AssertQueryEquals("XY\\u005a", a, "XYZ"); AssertQueryEquals("XY\\u005A", a, "XYZ"); AssertQueryEquals("\"a \\\\\\u0028\\u0062\\\" c\"", a, "\"a \\(b\" c\""); AssertParseException("XY\\u005G"); // test non-hex character in escaped unicode sequence AssertParseException("XY\\u005"); // test incomplete escaped unicode sequence // Tests bug LUCENE-800 AssertQueryEquals("(item:\\\\ item:ABCD\\\\)", a, "item:\\ item:ABCD\\"); AssertParseException("(item:\\\\ item:ABCD\\\\))"); // unmatched closing paranthesis AssertQueryEquals("\\*", a, "*"); AssertQueryEquals("\\\\", a, "\\"); // escaped backslash AssertParseException("\\"); // a backslash must always be escaped // LUCENE-1189 AssertQueryEquals("(\"a\\\\\") or (\"b\")", a, "a\\ or b"); }