/// <summary> Current version number from segments file.</summary> public static long ReadCurrentVersion(Directory directory) { IndexInput input = directory.OpenInput(IndexFileNames.SEGMENTS); int format = 0; long version = 0; try { format = input.ReadInt(); if (format < 0) { if (format < FORMAT) throw new System.IO.IOException("Unknown format version: " + format); version = input.ReadLong(); // read version } } finally { input.Close(); } if (format < 0) return version; // We cannot be sure about the format of the file. // Therefore we have to read the whole file and cannot simply seek to the version entry. SegmentInfos sis = new SegmentInfos(); sis.Read(directory); return sis.GetVersion(); }
public virtual void TestNumDocsLimit() { // tests that the max merge docs constraint is applied during forceMerge. Directory dir = new RAMDirectory(); // Prepare an index w/ several small segments and a large one. IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); AddDocs(writer, 3); AddDocs(writer, 3); AddDocs(writer, 5); AddDocs(writer, 3); AddDocs(writer, 3); AddDocs(writer, 3); AddDocs(writer, 3); writer.Dispose(); conf = NewWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); lmp.MaxMergeDocs = 3; conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); // Should only be 3 segments in the index, because one of them exceeds the size limit SegmentInfos sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(3, sis.Size()); }
protected internal override object DoBody(string segmentFileName) { var sis = new SegmentInfos(); sis.Read(directory, segmentFileName); var readers = new SegmentReader[sis.Size()]; for (int i = sis.Size() - 1; i >= 0; i--) { System.IO.IOException prior = null; bool success = false; try { readers[i] = new SegmentReader(sis.Info(i), termInfosIndexDivisor, IOContext.READ); success = true; } catch (System.IO.IOException ex) { prior = ex; } finally { if (!success) { IOUtils.CloseWhileHandlingException(prior, readers); } } } return new StandardDirectoryReader(directory, readers, null, sis, termInfosIndexDivisor, false); }
public virtual void TestOneLargeOneSmall() { Directory dir = new RAMDirectory(); IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); AddDocs(writer, 3); AddDocs(writer, 5); AddDocs(writer, 3); AddDocs(writer, 5); writer.Dispose(); conf = NewWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); lmp.MaxMergeDocs = 3; conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(4, sis.Size()); }
public virtual void TestSingleMergeableSegment() { Directory dir = new RAMDirectory(); IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); AddDocs(writer, 3); AddDocs(writer, 5); AddDocs(writer, 3); // delete the last document, so that the last segment is merged. writer.DeleteDocuments(new Term("id", "10")); writer.Dispose(); conf = NewWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); lmp.MaxMergeDocs = 3; conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); // Verify that the last segment does not have deletions. SegmentInfos sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(3, sis.Count); Assert.IsFalse(sis.Info(2).HasDeletions); }
public IndexSplitter(DirectoryInfo dir) { this.dir = dir; fsDir = FSDirectory.Open(dir); Infos = new SegmentInfos(); Infos.Read(fsDir); }
protected internal override object DoBody(System.String segmentFileName) { SegmentInfos infos = new SegmentInfos(); infos.Read(directory, segmentFileName); DirectoryIndexReader reader; if (infos.Count == 1) { // index is optimized reader = SegmentReader.Get(readOnly, infos, infos.Info(0), closeDirectory); } else if (readOnly) { reader = new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory); } else { reader = new MultiSegmentReader(directory, infos, closeDirectory, false); } reader.SetDeletionPolicy(deletionPolicy); return(reader); }
public virtual void TestSingleNonMergeableSegment() { Directory dir = new RAMDirectory(); IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); AddDocs(writer, 3, true); writer.Dispose(); conf = NewWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); lmp.MaxMergeDocs = 3; conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); // Verify that the last segment does not have deletions. SegmentInfos sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(1, sis.Size()); }
public virtual void TestSingleMergeableTooLargeSegment() { Directory dir = new RAMDirectory(); IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); AddDocs(writer, 5, true); // delete the last document writer.DeleteDocuments(new Term("id", "4")); writer.Dispose(); conf = NewWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); lmp.MaxMergeDocs = 2; conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); // Verify that the last segment does not have deletions. SegmentInfos sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(1, sis.Size()); Assert.IsTrue(sis.Info(0).HasDeletions()); }
/// <summary> Returns userData from latest segments file</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public static System.Collections.Generic.IDictionary <string, string> ReadCurrentUserData(Directory directory, IState state) { var sis = new SegmentInfos(); sis.Read(directory, state); return(sis.UserData); }
protected internal override object DoBody(string segmentFileName) { var sis = new SegmentInfos(); sis.Read(directory, segmentFileName); var readers = new SegmentReader[sis.Count]; // LUCENENET: Ported over changes from 4.8.1 to this method for (int i = sis.Count - 1; i >= 0; i--) { //IOException prior = null; // LUCENENET: Not used bool success = false; try { readers[i] = new SegmentReader(sis[i], termInfosIndexDivisor, IOContext.READ); success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(readers); } } } return(new StandardDirectoryReader(directory, readers, null, sis, termInfosIndexDivisor, false)); }
public virtual void TestAllSegmentsLarge() { Directory dir = new RAMDirectory(); IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); AddDocs(writer, 3); AddDocs(writer, 3); AddDocs(writer, 3); writer.Dispose(); conf = NewWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); lmp.MaxMergeDocs = 2; conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(3, sis.Size()); }
private int GetNumberOfSegments(Directory dir) { SegmentInfos infos = new SegmentInfos(); infos.Read(dir); return(infos.Size()); }
/// <summary>Merges all segments from an array of indexes into this index. /// /// <p>This may be used to parallelize batch indexing. A large document /// collection can be broken into sub-collections. Each sub-collection can be /// indexed in parallel, on a different thread, process or machine. The /// complete index can then be created by merging sub-collection indexes /// with this method. /// /// <p>After this completes, the index is optimized. /// </summary> public virtual void AddIndexes(Directory[] dirs) { lock (this) { Optimize(); // start with zero or 1 seg int start = segmentInfos.Count; for (int i = 0; i < dirs.Length; i++) { SegmentInfos sis = new SegmentInfos(); // read infos from dir sis.Read(dirs[i]); for (int j = 0; j < sis.Count; j++) { segmentInfos.Add(sis.Info(j)); // add each info } } // merge newly added segments in log(n) passes while (segmentInfos.Count > start + mergeFactor) { for (int base_Renamed = start; base_Renamed < segmentInfos.Count; base_Renamed++) { int end = System.Math.Min(segmentInfos.Count, base_Renamed + mergeFactor); if (end - base_Renamed > 1) { MergeSegments(base_Renamed, end); } } } Optimize(); // final cleanup } }
/// <summary> Returns userData from latest segments file</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public static System.Collections.Generic.IDictionary <string, string> ReadCurrentUserData(Directory directory) { SegmentInfos sis = new SegmentInfos(); sis.Read(directory); return(sis.GetUserData()); }
public virtual void TestMergeFactor() { Directory dir = new RAMDirectory(); IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); AddDocs(writer, 3); AddDocs(writer, 3); AddDocs(writer, 3); AddDocs(writer, 3); AddDocs(writer, 5); AddDocs(writer, 3); AddDocs(writer, 3); writer.Dispose(); conf = NewWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); lmp.MaxMergeDocs = 3; lmp.MergeFactor = 2; conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); // Should only be 4 segments in the index, because of the merge factor and // max merge docs settings. SegmentInfos sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(4, sis.Size()); }
protected internal override object DoBody(string segmentFileName) { var sis = new SegmentInfos(); sis.Read(directory, segmentFileName); var readers = new SegmentReader[sis.Count]; for (int i = sis.Count - 1; i >= 0; i--) { System.IO.IOException prior = null; bool success = false; try { readers[i] = new SegmentReader(sis.Info(i), termInfosIndexDivisor, IOContext.READ); success = true; } catch (System.IO.IOException ex) { prior = ex; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(prior, readers); } } } return(new StandardDirectoryReader(directory, readers, null, sis, termInfosIndexDivisor, false)); }
protected internal override object DoBody(string segmentFileName) { SegmentInfos infos = new SegmentInfos(); infos.Read(outerInstance.m_directory, segmentFileName); return(outerInstance.DoOpenIfChanged(infos)); }
public virtual void TestByteSizeLimit() { // tests that the max merge size constraint is applied during forceMerge. Directory dir = new RAMDirectory(); // Prepare an index w/ several small segments and a large one. IndexWriterConfig conf = NewWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); const int numSegments = 15; for (int i = 0; i < numSegments; i++) { int numDocs = i == 7 ? 30 : 1; AddDocs(writer, numDocs); } writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); double min = sis.Info(0).SizeInBytes(); conf = NewWriterConfig(); LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(); lmp.MaxMergeMBForForcedMerge = (min + 1) / (1 << 20); conf.SetMergePolicy(lmp); writer = new IndexWriter(dir, conf); writer.ForceMerge(1); writer.Dispose(); // Should only be 3 segments in the index, because one of them exceeds the size limit sis = new SegmentInfos(); sis.Read(dir); Assert.AreEqual(3, sis.Size()); }
public /*protected internal*/ override System.Object DoBody(System.String segmentFileName) { var infos = new SegmentInfos(); infos.Read(directory, segmentFileName); if (readOnly) return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor); else return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor); }
public virtual void DoTestDocument() { sis.Read(dir); IndexReader reader = OpenReader(); Assert.IsTrue(reader != null); Document newDoc1 = reader.Document(0); Assert.IsTrue(newDoc1 != null); Assert.IsTrue(DocHelper.NumFields(newDoc1) == DocHelper.NumFields(doc1) - DocHelper.unstored.Count); Document newDoc2 = reader.Document(1); Assert.IsTrue(newDoc2 != null); Assert.IsTrue(DocHelper.NumFields(newDoc2) == DocHelper.NumFields(doc2) - DocHelper.unstored.Count); TermFreqVector vector = reader.GetTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY); Assert.IsTrue(vector != null); TestSegmentReader.CheckNorms(reader); }
/// <summary> Current version number from segments file.</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public static long ReadCurrentVersion(Directory directory) { // Fully read the segments file: this ensures that it's // completely written so that if // IndexWriter.prepareCommit has been called (but not // yet commit), then the reader will still see itself as // current: SegmentInfos sis = new SegmentInfos(); sis.Read(directory); return(sis.version); }
public virtual void TestMaxNumSegments2([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler) { Directory dir = NewDirectory(); Document doc = new Document(); doc.Add(NewStringField("content", "aaa", Field.Store.NO)); LogDocMergePolicy ldmp = new LogDocMergePolicy(); ldmp.MinMergeDocs = 1; ldmp.MergeFactor = 4; var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)) .SetMaxBufferedDocs(2) .SetMergePolicy(ldmp) .SetMergeScheduler(newScheduler()); IndexWriter writer = new IndexWriter(dir, config); for (int iter = 0; iter < 10; iter++) { for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Commit(); writer.WaitForMerges(); writer.Commit(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); int segCount = sis.Count; writer.ForceMerge(7); writer.Commit(); writer.WaitForMerges(); sis = new SegmentInfos(); sis.Read(dir); int optSegCount = sis.Count; if (segCount < 7) { Assert.AreEqual(segCount, optSegCount); } else { Assert.AreEqual(7, optSegCount, "seg: " + segCount); } } writer.Dispose(); dir.Dispose(); }
/** @see IndexReader#listCommits */ public new static ICollection <IndexCommitPoint> ListCommits(Directory dir) { string[] files = dir.List(); if (files == null) { throw new System.IO.IOException("cannot read directory " + dir + ": list() returned null"); } ICollection <IndexCommitPoint> commits = new List <IndexCommitPoint>(); SegmentInfos latest = new SegmentInfos(); latest.Read(dir); long currentGen = latest.GetGeneration(); commits.Add(new ReaderCommit(latest, dir)); for (int i = 0; i < files.Length; i++) { String fileName = files[i]; if (fileName.StartsWith(IndexFileNames.SEGMENTS) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN) && SegmentInfos.GenerationFromSegmentsFileName(fileName) < currentGen) { SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt sis.Read(dir, fileName); } catch (System.Exception) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist sis = null; } if (sis != null) { commits.Add(new ReaderCommit(sis, dir)); } } } return(commits); }
public override void SetUp() { base.SetUp(); dir = new RAMDirectory(); doc1 = new Document(); doc2 = new Document(); DocHelper.SetupDoc(doc1); DocHelper.SetupDoc(doc2); DocHelper.WriteDoc(dir, doc1); DocHelper.WriteDoc(dir, doc2); sis = new SegmentInfos(); sis.Read(dir); }
public virtual void TestAddIndexes() { Directory dir1 = NewDirectory(); Directory dir2 = NewDirectory(); IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES)); Document d1 = new Document(); d1.Add(new TextField("f1", "first field", Field.Store.YES)); d1.Add(new TextField("f2", "second field", Field.Store.YES)); writer.AddDocument(d1); writer.Dispose(); writer = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES)); Document d2 = new Document(); FieldType customType2 = new FieldType(TextField.TYPE_STORED); customType2.StoreTermVectors = true; d2.Add(new TextField("f2", "second field", Field.Store.YES)); d2.Add(new Field("f1", "first field", customType2)); d2.Add(new TextField("f3", "third field", Field.Store.YES)); d2.Add(new TextField("f4", "fourth field", Field.Store.YES)); writer.AddDocument(d2); writer.Dispose(); writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES)); writer.AddIndexes(dir2); writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir1); Assert.AreEqual(2, sis.Count); FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0)); FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1)); Assert.AreEqual("f1", fis1.FieldInfo(0).Name); Assert.AreEqual("f2", fis1.FieldInfo(1).Name); // make sure the ordering of the "external" segment is preserved Assert.AreEqual("f2", fis2.FieldInfo(0).Name); Assert.AreEqual("f1", fis2.FieldInfo(1).Name); Assert.AreEqual("f3", fis2.FieldInfo(2).Name); Assert.AreEqual("f4", fis2.FieldInfo(3).Name); dir1.Dispose(); dir2.Dispose(); }
/// <summary> /// Returns all commit points that exist in the <see cref="Store.Directory"/>. /// Normally, because the default is /// <see cref="KeepOnlyLastCommitDeletionPolicy"/>, there would be only /// one commit point. But if you're using a custom /// <see cref="IndexDeletionPolicy"/> then there could be many commits. /// Once you have a given commit, you can open a reader on /// it by calling <see cref="DirectoryReader.Open(IndexCommit)"/> /// There must be at least one commit in /// the <see cref="Store.Directory"/>, else this method throws /// <see cref="IndexNotFoundException"/>. Note that if a commit is in /// progress while this method is running, that commit /// may or may not be returned. /// </summary> /// <returns> a sorted list of <see cref="Index.IndexCommit"/>s, from oldest /// to latest. </returns> public static IList <IndexCommit> ListCommits(Directory dir) { string[] files = dir.ListAll(); List <IndexCommit> commits = new List <IndexCommit>(); SegmentInfos latest = new SegmentInfos(); latest.Read(dir); long currentGen = latest.Generation; commits.Add(new StandardDirectoryReader.ReaderCommit(latest, dir)); for (int i = 0; i < files.Length; i++) { string fileName = files[i]; if (fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN, StringComparison.Ordinal) && SegmentInfos.GenerationFromSegmentsFileName(fileName) < currentGen) { SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt sis.Read(dir, fileName); } catch (Exception fnfe) when(fnfe.IsNoSuchFileExceptionOrFileNotFoundException()) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist sis = null; } if (sis != null) { commits.Add(new StandardDirectoryReader.ReaderCommit(sis, dir)); } } } // Ensure that the commit points are sorted in ascending order. commits.Sort(); return(commits); }
/// <summary> Current version number from segments file.</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public static long ReadCurrentVersion(Directory directory) { // Fully read the segments file: this ensures that it's // completely written so that if // IndexWriter.prepareCommit has been called (but not // yet commit), then the reader will still see itself as // current: var sis = new SegmentInfos(); sis.Read(directory); return(sis.version); //return (long) ((System.Int64) new AnonymousClassFindSegmentsFile1(directory).Run()); //DIGY: AnonymousClassFindSegmentsFile1 can safely be deleted }
public override System.Object DoBody() { SegmentInfos infos = new SegmentInfos(); infos.Read(directory); if (infos.Count == 1) { // index is optimized return SegmentReader.Get(infos, infos.Info(0), closeDirectory); } IndexReader[] readers = new IndexReader[infos.Count]; for (int i = 0; i < infos.Count; i++) readers[i] = SegmentReader.Get(infos.Info(i)); return new MultiReader(directory, infos, closeDirectory, readers); }
public virtual void TestErrorInDocsWriterAdd() { MockRAMDirectory.Failure failure = new AnonymousClassFailure1(this); // create a couple of files System.String[] keywords = new System.String[] { "1", "2" }; System.String[] unindexed = new System.String[] { "Netherlands", "Italy" }; System.String[] unstored = new System.String[] { "Amsterdam has lots of bridges", "Venice has lots of canals" }; System.String[] text = new System.String[] { "Amsterdam", "Venice" }; for (int pass = 0; pass < 2; pass++) { bool autoCommit = (0 == pass); MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true); dir.FailOn(failure.Reset()); for (int i = 0; i < keywords.Length; i++) { Document doc = new Document(); doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO)); doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED)); doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED)); try { modifier.AddDocument(doc); } catch (System.IO.IOException io) { break; } } System.String[] startFiles = dir.ListAll(); SegmentInfos infos = new SegmentInfos(); infos.Read(dir); new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null); System.String[] endFiles = dir.ListAll(); if (!SupportClass.CollectionsHelper.CompareStringArrays(startFiles, endFiles)) { Assert.Fail("docswriter abort() failed to delete unreferenced files:\n before delete:\n " + ArrayToString(startFiles) + "\n after delete:\n " + ArrayToString(endFiles)); } modifier.Close(); } }
protected internal override System.Object DoBody(System.String segmentFileName) { SegmentInfos infos = new SegmentInfos(); infos.Read(directory, segmentFileName); DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos); if (Enclosing_Instance != newReader) { newReader.Init(directory, infos, Enclosing_Instance.closeDirectory); newReader.deletionPolicy = Enclosing_Instance.deletionPolicy; } return newReader; }
private int CheckAllSegmentsUpgraded(Directory dir) { SegmentInfos infos = new SegmentInfos(); infos.Read(dir); if (VERBOSE) { Console.WriteLine("checkAllSegmentsUpgraded: " + infos); } foreach (SegmentCommitInfo si in infos.Segments) { Assert.AreEqual(Constants.LUCENE_MAIN_VERSION, si.Info.Version); } return(infos.Size()); }
public virtual void TestManyFields() { int NUM_DOCS = AtLeast(200); int MAX_FIELDS = AtLeast(50); int[][] docs = RectangularArrays.ReturnRectangularArray <int>(NUM_DOCS, 4); for (int i = 0; i < docs.Length; i++) { for (int j = 0; j < docs[i].Length; j++) { docs[i][j] = Random.Next(MAX_FIELDS); } } Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))); for (int i = 0; i < NUM_DOCS; i++) { Document d = new Document(); for (int j = 0; j < docs[i].Length; j++) { d.Add(GetField(docs[i][j])); } writer.AddDocument(d); } writer.ForceMerge(1); writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); foreach (SegmentCommitInfo si in sis.Segments) { FieldInfos fis = SegmentReader.ReadFieldInfos(si); foreach (FieldInfo fi in fis) { Field expected = GetField(Convert.ToInt32(fi.Name)); Assert.AreEqual(expected.FieldType.IsIndexed, fi.IsIndexed); Assert.AreEqual(expected.FieldType.StoreTermVectors, fi.HasVectors); } } dir.Dispose(); }
public virtual void TestAddIndexes() { Directory dir1 = NewDirectory(); Directory dir2 = NewDirectory(); IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES)); Document d1 = new Document(); d1.Add(new TextField("f1", "first field", Field.Store.YES)); d1.Add(new TextField("f2", "second field", Field.Store.YES)); writer.AddDocument(d1); writer.Dispose(); writer = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES)); Document d2 = new Document(); FieldType customType2 = new FieldType(TextField.TYPE_STORED); customType2.StoreTermVectors = true; d2.Add(new TextField("f2", "second field", Field.Store.YES)); d2.Add(new Field("f1", "first field", customType2)); d2.Add(new TextField("f3", "third field", Field.Store.YES)); d2.Add(new TextField("f4", "fourth field", Field.Store.YES)); writer.AddDocument(d2); writer.Dispose(); writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES)); writer.AddIndexes(dir2); writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir1); Assert.AreEqual(2, sis.Size()); FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0)); FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1)); Assert.AreEqual("f1", fis1.FieldInfo(0).Name); Assert.AreEqual("f2", fis1.FieldInfo(1).Name); // make sure the ordering of the "external" segment is preserved Assert.AreEqual("f2", fis2.FieldInfo(0).Name); Assert.AreEqual("f1", fis2.FieldInfo(1).Name); Assert.AreEqual("f3", fis2.FieldInfo(2).Name); Assert.AreEqual("f4", fis2.FieldInfo(3).Name); dir1.Dispose(); dir2.Dispose(); }
public override System.Object DoBody() { SegmentInfos infos = new SegmentInfos(); infos.Read(directory); if (infos.Count == 1) { // index is optimized return(SegmentReader.Get(infos, infos.Info(0), closeDirectory)); } IndexReader[] readers = new IndexReader[infos.Count]; for (int i = 0; i < infos.Count; i++) { readers[i] = SegmentReader.Get(infos.Info(i)); } return(new MultiReader(directory, infos, closeDirectory, readers)); }
public virtual void TestPartialMerge() { Directory dir = NewDirectory(); Document doc = new Document(); doc.Add(NewStringField("content", "aaa", Field.Store.NO)); int incrMin = TEST_NIGHTLY ? 15 : 40; for (int numDocs = 10; numDocs < 500; numDocs += TestUtil.NextInt32(Random, incrMin, 5 * incrMin)) { LogDocMergePolicy ldmp = new LogDocMergePolicy(); ldmp.MinMergeDocs = 1; ldmp.MergeFactor = 5; IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMaxBufferedDocs(2).SetMergePolicy(ldmp)); for (int j = 0; j < numDocs; j++) { writer.AddDocument(doc); } writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); int segCount = sis.Count; ldmp = new LogDocMergePolicy(); ldmp.MergeFactor = 5; writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(ldmp)); writer.ForceMerge(3); writer.Dispose(); sis = new SegmentInfos(); sis.Read(dir); int optSegCount = sis.Count; if (segCount < 3) { Assert.AreEqual(segCount, optSegCount); } else { Assert.AreEqual(3, optSegCount); } } dir.Dispose(); }
public virtual void TestPartialMerge() { Directory dir = NewDirectory(); Document doc = new Document(); doc.Add(NewStringField("content", "aaa", Field.Store.NO)); int incrMin = TEST_NIGHTLY ? 15 : 40; for (int numDocs = 10; numDocs < 500; numDocs += TestUtil.NextInt(Random(), incrMin, 5 * incrMin)) { LogDocMergePolicy ldmp = new LogDocMergePolicy(); ldmp.MinMergeDocs = 1; ldmp.MergeFactor = 5; IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.CREATE).SetMaxBufferedDocs(2).SetMergePolicy(ldmp)); for (int j = 0; j < numDocs; j++) { writer.AddDocument(doc); } writer.Dispose(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); int segCount = sis.Size(); ldmp = new LogDocMergePolicy(); ldmp.MergeFactor = 5; writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(ldmp)); writer.ForceMerge(3); writer.Dispose(); sis = new SegmentInfos(); sis.Read(dir); int optSegCount = sis.Size(); if (segCount < 3) { Assert.AreEqual(segCount, optSegCount); } else { Assert.AreEqual(3, optSegCount); } } dir.Dispose(); }
public virtual void TestBackgroundForceMerge() { Directory dir = NewDirectory(); for (int pass = 0; pass < 2; pass++) { IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(51))); Document doc = new Document(); doc.Add(NewStringField("field", "aaa", Field.Store.NO)); for (int i = 0; i < 100; i++) { writer.AddDocument(doc); } writer.ForceMerge(1, false); if (0 == pass) { writer.Dispose(); DirectoryReader reader = DirectoryReader.Open(dir); Assert.AreEqual(1, reader.Leaves.Count); reader.Dispose(); } else { // Get another segment to flush so we can verify it is // NOT included in the merging writer.AddDocument(doc); writer.AddDocument(doc); writer.Dispose(); DirectoryReader reader = DirectoryReader.Open(dir); Assert.IsTrue(reader.Leaves.Count > 1); reader.Dispose(); SegmentInfos infos = new SegmentInfos(); infos.Read(dir); Assert.AreEqual(2, infos.Count); } } dir.Dispose(); }
protected internal override System.Object DoBody(System.String segmentFileName) { SegmentInfos infos = new SegmentInfos(); infos.Read(directory, segmentFileName); DirectoryIndexReader reader; if (infos.Count == 1) { // index is optimized reader = SegmentReader.Get(infos, infos.Info(0), closeDirectory); } else { reader = new MultiSegmentReader(directory, infos, closeDirectory); } reader.SetDeletionPolicy(deletionPolicy); return reader; }
public virtual void TestBackgroundForceMerge() { Directory dir = NewDirectory(); for (int pass = 0; pass < 2; pass++) { IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.CREATE).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(51))); Document doc = new Document(); doc.Add(NewStringField("field", "aaa", Field.Store.NO)); for (int i = 0; i < 100; i++) { writer.AddDocument(doc); } writer.ForceMerge(1, false); if (0 == pass) { writer.Dispose(); DirectoryReader reader = DirectoryReader.Open(dir); Assert.AreEqual(1, reader.Leaves.Count); reader.Dispose(); } else { // Get another segment to flush so we can verify it is // NOT included in the merging writer.AddDocument(doc); writer.AddDocument(doc); writer.Dispose(); DirectoryReader reader = DirectoryReader.Open(dir); Assert.IsTrue(reader.Leaves.Count > 1); reader.Dispose(); SegmentInfos infos = new SegmentInfos(); infos.Read(dir); Assert.AreEqual(2, infos.Size()); } } dir.Dispose(); }
public override System.Object DoBody(System.String segmentFileName) { SegmentInfos infos = new SegmentInfos(); infos.Read(directory, segmentFileName); if (infos.Count == 1) { // index is optimized return SegmentReader.Get(infos, infos.Info(0), closeDirectory); } else { // To reduce the chance of hitting FileNotFound // (and having to retry), we open segments in // reverse because IndexWriter merges & deletes // the newest segments first. IndexReader[] readers = new IndexReader[infos.Count]; for (int i = infos.Count - 1; i >= 0; i--) { try { readers[i] = SegmentReader.Get(infos.Info(i)); } catch (System.IO.IOException e) { // Close all readers we had opened: for (i++; i < infos.Count; i++) { readers[i].Close(); } throw e; } } return new MultiReader(directory, infos, closeDirectory, readers); } }
public virtual void TestMaxNumSegments2() { Directory dir = NewDirectory(); Document doc = new Document(); doc.Add(NewStringField("content", "aaa", Field.Store.NO)); LogDocMergePolicy ldmp = new LogDocMergePolicy(); ldmp.MinMergeDocs = 1; ldmp.MergeFactor = 4; IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(ldmp).SetMergeScheduler(new ConcurrentMergeScheduler())); for (int iter = 0; iter < 10; iter++) { for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Commit(); writer.WaitForMerges(); writer.Commit(); SegmentInfos sis = new SegmentInfos(); sis.Read(dir); int segCount = sis.Size(); writer.ForceMerge(7); writer.Commit(); writer.WaitForMerges(); sis = new SegmentInfos(); sis.Read(dir); int optSegCount = sis.Size(); if (segCount < 7) { Assert.AreEqual(segCount, optSegCount); } else { Assert.AreEqual(7, optSegCount, "seg: " + segCount); } } writer.Dispose(); dir.Dispose(); }
/// <summary> Current version number from segments file.</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public static long ReadCurrentVersion(Directory directory) { // Fully read the segments file: this ensures that it's // completely written so that if // IndexWriter.prepareCommit has been called (but not // yet commit), then the reader will still see itself as // current: SegmentInfos sis = new SegmentInfos(); sis.Read(directory); return sis.version; //return (long) ((System.Int64) new AnonymousClassFindSegmentsFile1(directory).Run()); //DIGY: AnonymousClassFindSegmentsFile1 can safely be deleted }
public static IndexSegmentsInfo GetCurrentSegmentsInfo(string indexName, Lucene.Net.Store.Directory directory) { var segmentInfos = new SegmentInfos(); var result = new IndexSegmentsInfo(); try { segmentInfos.Read(directory); result.Generation = segmentInfos.Generation; result.SegmentsFileName = segmentInfos.GetCurrentSegmentFileName(); result.ReferencedFiles = segmentInfos.Files(directory, false); } catch (CorruptIndexException ex) { log.WarnException(string.Format("Could not read segment information for an index '{0}'", indexName), ex); result.IsIndexCorrupted = true; } return result; }
/// <summary>Returns a <see cref="Status" /> instance detailing /// the state of the index. /// /// </summary> /// <param name="onlySegments">list of specific segment names to check /// /// <p/>As this method checks every byte in the specified /// segments, on a large index it can take quite a long /// time to run. /// /// <p/><b>WARNING</b>: make sure /// you only call this when the index is not opened by any /// writer. /// </param> public virtual Status CheckIndex_Renamed_Method(List<string> onlySegments) { System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat; SegmentInfos sis = new SegmentInfos(); Status result = new Status(); result.dir = dir; try { sis.Read(dir); } catch (System.Exception t) { Msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; if (infoStream != null) infoStream.WriteLine(t.StackTrace); return result; } int numSegments = sis.Count; var segmentsFileName = sis.GetCurrentSegmentFileName(); IndexInput input = null; try { input = dir.OpenInput(segmentsFileName); } catch (System.Exception t) { Msg("ERROR: could not open segments file in directory"); if (infoStream != null) infoStream.WriteLine(t.StackTrace); result.cantOpenSegments = true; return result; } int format = 0; try { format = input.ReadInt(); } catch (System.Exception t) { Msg("ERROR: could not read segment file version in directory"); if (infoStream != null) infoStream.WriteLine(t.StackTrace); result.missingSegmentVersion = true; return result; } finally { if (input != null) input.Close(); } System.String sFormat = ""; bool skip = false; if (format == SegmentInfos.FORMAT) sFormat = "FORMAT [Lucene Pre-2.1]"; if (format == SegmentInfos.FORMAT_LOCKLESS) sFormat = "FORMAT_LOCKLESS [Lucene 2.1]"; else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE) sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]"; else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE) sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]"; else { if (format == SegmentInfos.FORMAT_CHECKSUM) sFormat = "FORMAT_CHECKSUM [Lucene 2.4]"; else if (format == SegmentInfos.FORMAT_DEL_COUNT) sFormat = "FORMAT_DEL_COUNT [Lucene 2.4]"; else if (format == SegmentInfos.FORMAT_HAS_PROX) sFormat = "FORMAT_HAS_PROX [Lucene 2.4]"; else if (format == SegmentInfos.FORMAT_USER_DATA) sFormat = "FORMAT_USER_DATA [Lucene 2.9]"; else if (format == SegmentInfos.FORMAT_DIAGNOSTICS) sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]"; else if (format < SegmentInfos.CURRENT_FORMAT) { sFormat = "int=" + format + " [newer version of Lucene than this tool]"; skip = true; } else { sFormat = format + " [Lucene 1.3 or prior]"; } } result.segmentsFileName = segmentsFileName; result.numSegments = numSegments; result.segmentFormat = sFormat; result.userData = sis.UserData; System.String userDataString; if (sis.UserData.Count > 0) { userDataString = " userData=" + CollectionsHelper.CollectionToString(sis.UserData); } else { userDataString = ""; } Msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString); if (onlySegments != null) { result.partial = true; if (infoStream != null) infoStream.Write("\nChecking only these segments:"); foreach(string s in onlySegments) { if (infoStream != null) { infoStream.Write(" " + s); } } result.segmentsChecked.AddRange(onlySegments); Msg(":"); } if (skip) { Msg("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting"); result.toolOutOfDate = true; return result; } result.newSegments = (SegmentInfos) sis.Clone(); result.newSegments.Clear(); for (int i = 0; i < numSegments; i++) { SegmentInfo info = sis.Info(i); if (onlySegments != null && !onlySegments.Contains(info.name)) continue; var segInfoStat = new Status.SegmentInfoStatus(); result.segmentInfos.Add(segInfoStat); Msg(" " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount); segInfoStat.name = info.name; segInfoStat.docCount = info.docCount; int toLoseDocCount = info.docCount; SegmentReader reader = null; try { Msg(" compound=" + info.GetUseCompoundFile()); segInfoStat.compound = info.GetUseCompoundFile(); Msg(" hasProx=" + info.HasProx); segInfoStat.hasProx = info.HasProx; Msg(" numFiles=" + info.Files().Count); segInfoStat.numFiles = info.Files().Count; Msg(System.String.Format(nf, " size (MB)={0:f}", new System.Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) })); segInfoStat.sizeMB = info.SizeInBytes() / (1024.0 * 1024.0); IDictionary<string, string> diagnostics = info.Diagnostics; segInfoStat.diagnostics = diagnostics; if (diagnostics.Count > 0) { Msg(" diagnostics = " + CollectionsHelper.CollectionToString(diagnostics)); } int docStoreOffset = info.DocStoreOffset; if (docStoreOffset != - 1) { Msg(" docStoreOffset=" + docStoreOffset); segInfoStat.docStoreOffset = docStoreOffset; Msg(" docStoreSegment=" + info.DocStoreSegment); segInfoStat.docStoreSegment = info.DocStoreSegment; Msg(" docStoreIsCompoundFile=" + info.DocStoreIsCompoundFile); segInfoStat.docStoreCompoundFile = info.DocStoreIsCompoundFile; } System.String delFileName = info.GetDelFileName(); if (delFileName == null) { Msg(" no deletions"); segInfoStat.hasDeletions = false; } else { Msg(" has deletions [delFileName=" + delFileName + "]"); segInfoStat.hasDeletions = true; segInfoStat.deletionsFileName = delFileName; } if (infoStream != null) infoStream.Write(" test: open reader........."); reader = SegmentReader.Get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); segInfoStat.openReaderPassed = true; int numDocs = reader.NumDocs(); toLoseDocCount = numDocs; if (reader.HasDeletions) { if (reader.deletedDocs.Count() != info.GetDelCount()) { throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs deletedDocs.count()=" + reader.deletedDocs.Count()); } if (reader.deletedDocs.Count() > reader.MaxDoc) { throw new System.SystemException("too many deleted docs: MaxDoc=" + reader.MaxDoc + " vs deletedDocs.count()=" + reader.deletedDocs.Count()); } if (info.docCount - numDocs != info.GetDelCount()) { throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs reader=" + (info.docCount - numDocs)); } segInfoStat.numDeleted = info.docCount - numDocs; Msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]"); } else { if (info.GetDelCount() != 0) { throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs reader=" + (info.docCount - numDocs)); } Msg("OK"); } if (reader.MaxDoc != info.docCount) throw new System.SystemException("SegmentReader.MaxDoc " + reader.MaxDoc + " != SegmentInfos.docCount " + info.docCount); // Test getFieldNames() if (infoStream != null) { infoStream.Write(" test: fields.............."); } ICollection<string> fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL); Msg("OK [" + fieldNames.Count + " fields]"); segInfoStat.numFields = fieldNames.Count; // Test Field Norms segInfoStat.fieldNormStatus = TestFieldNorms(fieldNames, reader); // Test the Term Index segInfoStat.termIndexStatus = TestTermIndex(info, reader); // Test Stored Fields segInfoStat.storedFieldStatus = TestStoredFields(info, reader, nf); // Test Term Vectors segInfoStat.termVectorStatus = TestTermVectors(info, reader, nf); // Rethrow the first exception we encountered // This will cause stats for failed segments to be incremented properly if (segInfoStat.fieldNormStatus.error != null) { throw new SystemException("Field Norm test failed"); } else if (segInfoStat.termIndexStatus.error != null) { throw new SystemException("Term Index test failed"); } else if (segInfoStat.storedFieldStatus.error != null) { throw new SystemException("Stored Field test failed"); } else if (segInfoStat.termVectorStatus.error != null) { throw new System.SystemException("Term Vector test failed"); } Msg(""); } catch (System.Exception t) { Msg("FAILED"); const string comment = "fixIndex() would remove reference to this segment"; Msg(" WARNING: " + comment + "; full exception:"); if (infoStream != null) infoStream.WriteLine(t.StackTrace); Msg(""); result.totLoseDocCount += toLoseDocCount; result.numBadSegments++; continue; } finally { if (reader != null) reader.Close(); } // Keeper result.newSegments.Add((SegmentInfo)info.Clone()); } if (0 == result.numBadSegments) { result.clean = true; Msg("No problems were detected with this index.\n"); } else Msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected"); return result; }
private void Init(Directory d, Analyzer a, bool create, bool closeDir, IndexDeletionPolicy deletionPolicy, bool autoCommit, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit) { this.closeDir = closeDir; directory = d; analyzer = a; SetMessageID(defaultInfoStream); this.maxFieldLength = maxFieldLength; if (indexingChain == null) indexingChain = DocumentsWriter.DefaultIndexingChain; if (create) { // Clear the write lock in case it's leftover: directory.ClearLock(WRITE_LOCK_NAME); } Lock writeLock = directory.MakeLock(WRITE_LOCK_NAME); if (!writeLock.Obtain(writeLockTimeout)) // obtain write lock { throw new LockObtainFailedException("Index locked for write: " + writeLock); } this.writeLock = writeLock; // save it bool success = false; try { if (create) { // Try to read first. This is to allow create // against an index that's currently open for // searching. In this case we write the next // segments_N file with no segments: bool doCommit; try { segmentInfos.Read(directory); segmentInfos.Clear(); doCommit = false; } catch (System.IO.IOException e) { // Likely this means it's a fresh directory doCommit = true; } if (autoCommit || doCommit) { // Always commit if autoCommit=true, else only // commit if there is no segments file in this dir // already. segmentInfos.Commit(directory); SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true)); } else { // Record that we have a change (zero out all // segments) pending: changeCount++; } } else { segmentInfos.Read(directory); if (commit != null) { // Swap out all segments, but, keep metadata in // SegmentInfos, like version & generation, to // preserve write-once. This is important if // readers are open against the future commit // points. if (commit.GetDirectory() != directory) throw new System.ArgumentException("IndexCommit's directory doesn't match my directory"); SegmentInfos oldInfos = new SegmentInfos(); oldInfos.Read(directory, commit.GetSegmentsFileName()); segmentInfos.Replace(oldInfos); changeCount++; if (infoStream != null) Message("init: loaded commit \"" + commit.GetSegmentsFileName() + "\""); } // We assume that this segments_N was previously // properly sync'd: SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true)); } this.autoCommit = autoCommit; SetRollbackSegmentInfos(segmentInfos); docWriter = new DocumentsWriter(directory, this, indexingChain); docWriter.SetInfoStream(infoStream); docWriter.SetMaxFieldLength(maxFieldLength); // Default deleter (for backwards compatibility) is // KeepOnlyLastCommitDeleter: deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter,synced); if (deleter.startingCommitDeleted) // Deletion policy deleted the "head" commit point. // We have to mark ourself as changed so that if we // are closed w/o any further changes we write a new // segments_N file. changeCount++; PushMaxBufferedDocs(); if (infoStream != null) { Message("init: create=" + create); MessageState(); } success = true; } finally { if (!success) { if (infoStream != null) { Message("init: hit exception on init; releasing write lock"); } try { writeLock.Release(); } catch (Exception t) { // don't mask the original exception } writeLock = null; } } }
/** @see IndexReader#listCommits */ public static new ICollection<IndexCommitPoint> ListCommits(Directory dir) { string[] files = dir.List(); if (files == null) throw new System.IO.IOException("cannot read directory " + dir + ": list() returned null"); ICollection<IndexCommitPoint> commits = new List<IndexCommitPoint>(); SegmentInfos latest = new SegmentInfos(); latest.Read(dir); long currentGen = latest.GetGeneration(); commits.Add(new ReaderCommit(latest, dir)); for (int i = 0; i < files.Length; i++) { String fileName = files[i]; if (fileName.StartsWith(IndexFileNames.SEGMENTS) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN) && SegmentInfos.GenerationFromSegmentsFileName(fileName) < currentGen) { SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt sis.Read(dir, fileName); } catch (System.Exception) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist sis = null; } if (sis != null) commits.Add(new ReaderCommit(sis, dir)); } } return commits; }
protected internal override object DoBody(System.String segmentFileName) { SegmentInfos infos = new SegmentInfos(); infos.Read(directory, segmentFileName); DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos); if (Enclosing_Instance != newReader) { newReader.Init(directory, infos, Enclosing_Instance.closeDirectory, Enclosing_Instance.readOnly); newReader.deletionPolicy = Enclosing_Instance.deletionPolicy; } return newReader; }
public virtual void TestErrorInDocsWriterAdd() { MockRAMDirectory.Failure failure = new AnonymousClassFailure1(this); // create a couple of files System.String[] keywords = new System.String[]{"1", "2"}; System.String[] unindexed = new System.String[]{"Netherlands", "Italy"}; System.String[] unstored = new System.String[]{"Amsterdam has lots of bridges", "Venice has lots of canals"}; System.String[] text = new System.String[]{"Amsterdam", "Venice"}; for (int pass = 0; pass < 2; pass++) { bool autoCommit = (0 == pass); MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true); dir.FailOn(failure.Reset()); for (int i = 0; i < keywords.Length; i++) { Document doc = new Document(); doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO)); doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED)); doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED)); try { modifier.AddDocument(doc); } catch (System.IO.IOException io) { break; } } System.String[] startFiles = dir.ListAll(); SegmentInfos infos = new SegmentInfos(); infos.Read(dir); new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null,null); System.String[] endFiles = dir.ListAll(); if (!SupportClass.CollectionsHelper.CompareStringArrays(startFiles, endFiles)) { Assert.Fail("docswriter abort() failed to delete unreferenced files:\n before delete:\n " + ArrayToString(startFiles) + "\n after delete:\n " + ArrayToString(endFiles)); } modifier.Close(); } }
private int GetNumberOfSegments(Directory dir) { SegmentInfos infos = new SegmentInfos(); infos.Read(dir); return infos.Size(); }
/// <summary> Merges all segments from an array of indexes into this /// index. /// /// <p/>This may be used to parallelize batch indexing. A large document /// collection can be broken into sub-collections. Each sub-collection can be /// indexed in parallel, on a different thread, process or machine. The /// complete index can then be created by merging sub-collection indexes /// with this method. /// /// <p/><b>NOTE:</b> the index in each Directory must not be /// changed (opened by a writer) while this method is /// running. This method does not acquire a write lock in /// each input Directory, so it is up to the caller to /// enforce this. /// /// <p/><b>NOTE:</b> while this is running, any attempts to /// add or delete documents (with another thread) will be /// paused until this method completes. /// /// <p/>This method is transactional in how Exceptions are /// handled: it does not commit a new segments_N file until /// all indexes are added. This means if an Exception /// occurs (for example disk full), then either no indexes /// will have been added or they all will have been.<p/> /// /// <p/>Note that this requires temporary free space in the /// Directory up to 2X the sum of all input indexes /// (including the starting index). If readers/searchers /// are open against the starting index, then temporary /// free space required will be higher by the size of the /// starting index (see {@link #Optimize()} for details). /// <p/> /// /// <p/>Once this completes, the final size of the index /// will be less than the sum of all input index sizes /// (including the starting index). It could be quite a /// bit smaller (if there were many pending deletes) or /// just slightly smaller.<p/> /// /// <p/> /// This requires this index not be among those to be added. /// /// <p/><b>NOTE</b>: if this method hits an OutOfMemoryError /// you should immediately close the writer. See <a /// href="#OOME">above</a> for details.<p/> /// /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public virtual void AddIndexesNoOptimize(Directory[] dirs) { EnsureOpen(); NoDupDirs(dirs); // Do not allow add docs or deletes while we are running: docWriter.PauseAllThreads(); try { if (infoStream != null) Message("flush at addIndexesNoOptimize"); Flush(true, false, true); bool success = false; StartTransaction(false); try { int docCount = 0; lock (this) { EnsureOpen(); for (int i = 0; i < dirs.Length; i++) { if (directory == dirs[i]) { // cannot add this index: segments may be deleted in merge before added throw new System.ArgumentException("Cannot add this index to itself"); } SegmentInfos sis = new SegmentInfos(); // read infos from dir sis.Read(dirs[i]); for (int j = 0; j < sis.Count; j++) { SegmentInfo info = sis.Info(j); System.Diagnostics.Debug.Assert(!segmentInfos.Contains(info), "dup info dir=" + info.dir + " name=" + info.name); docCount += info.docCount; segmentInfos.Add(info); // add each info } } } // Notify DocumentsWriter that the flushed count just increased docWriter.UpdateFlushedDocCount(docCount); MaybeMerge(); EnsureOpen(); // If after merging there remain segments in the index // that are in a different directory, just copy these // over into our index. This is necessary (before // finishing the transaction) to avoid leaving the // index in an unusable (inconsistent) state. ResolveExternalSegments(); EnsureOpen(); success = true; } finally { if (success) { CommitTransaction(); } else { RollbackTransaction(); } } } catch (System.OutOfMemoryException oom) { HandleOOM(oom, "addIndexesNoOptimize"); } finally { if (docWriter != null) { docWriter.ResumeAllThreads(); } } }
/// <summary> /// Adds all segments from an array of indexes into this index. /// /// <p>this may be used to parallelize batch indexing. A large document /// collection can be broken into sub-collections. Each sub-collection can be /// indexed in parallel, on a different thread, process or machine. The /// complete index can then be created by merging sub-collection indexes /// with this method. /// /// <p> /// <b>NOTE:</b> this method acquires the write lock in /// each directory, to ensure that no {@code IndexWriter} /// is currently open or tries to open while this is /// running. /// /// <p>this method is transactional in how Exceptions are /// handled: it does not commit a new segments_N file until /// all indexes are added. this means if an Exception /// occurs (for example disk full), then either no indexes /// will have been added or they all will have been. /// /// <p>Note that this requires temporary free space in the /// <seealso cref="Directory"/> up to 2X the sum of all input indexes /// (including the starting index). If readers/searchers /// are open against the starting index, then temporary /// free space required will be higher by the size of the /// starting index (see <seealso cref="#forceMerge(int)"/> for details). /// /// <p> /// <b>NOTE:</b> this method only copies the segments of the incoming indexes /// and does not merge them. Therefore deleted documents are not removed and /// the new segments are not merged with the existing ones. /// /// <p>this requires this index not be among those to be added. /// /// <p> /// <b>NOTE</b>: if this method hits an OutOfMemoryError /// you should immediately close the writer. See <a /// href="#OOME">above</a> for details. /// </summary> /// <exception cref="CorruptIndexException"> if the index is corrupt </exception> /// <exception cref="IOException"> if there is a low-level IO error </exception> /// <exception cref="LockObtainFailedException"> if we were unable to /// acquire the write lock in at least one directory </exception> public virtual void AddIndexes(params Directory[] dirs) { EnsureOpen(); NoDupDirs(dirs); IEnumerable<Lock> locks = AcquireWriteLocks(dirs); bool successTop = false; try { if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "flush at addIndexes(Directory...)"); } Flush(false, true); IList<SegmentCommitInfo> infos = new List<SegmentCommitInfo>(); bool success = false; try { foreach (Directory dir in dirs) { if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "addIndexes: process directory " + dir); } SegmentInfos sis = new SegmentInfos(); // read infos from dir sis.Read(dir); HashSet<string> dsFilesCopied = new HashSet<string>(); IDictionary<string, string> dsNames = new Dictionary<string, string>(); HashSet<string> copiedFiles = new HashSet<string>(); foreach (SegmentCommitInfo info in sis.Segments) { Debug.Assert(!infos.Contains(info), "dup info dir=" + info.Info.Dir + " name=" + info.Info.Name); string newSegName = NewSegmentName(); if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "addIndexes: process segment origName=" + info.Info.Name + " newName=" + newSegName + " info=" + info); } IOContext context = new IOContext(new MergeInfo(info.Info.DocCount, info.SizeInBytes(), true, -1)); foreach (FieldInfo fi in SegmentReader.ReadFieldInfos(info)) { GlobalFieldNumberMap.AddOrGet(fi.Name, fi.Number, fi.DocValuesType); } infos.Add(CopySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context, copiedFiles)); } } success = true; } finally { if (!success) { foreach (SegmentCommitInfo sipc in infos) { foreach (string file in sipc.Files()) { try { directory.DeleteFile(file); } catch (Exception) { } } } } } lock (this) { success = false; try { EnsureOpen(); success = true; } finally { if (!success) { foreach (SegmentCommitInfo sipc in infos) { foreach (string file in sipc.Files()) { try { directory.DeleteFile(file); } catch (Exception) { } } } } } segmentInfos.AddAll(infos); Checkpoint(); } successTop = true; } catch (System.OutOfMemoryException oom) { HandleOOM(oom, "addIndexes(Directory...)"); } finally { if (locks != null) { foreach (var lk in locks) { lk.Release(); } } if (successTop) { IOUtils.Close(locks); } else { IOUtils.CloseWhileHandlingException(locks); } } }
/// <summary> Initialize the deleter: find all previous commits in /// the Directory, incref the files they reference, call /// the policy to let it delete commits. This will remove /// any files not referenced by any of the commits. /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, System.Collections.Generic.Dictionary<string, string> synced) { this.docWriter = docWriter; this.infoStream = infoStream; this.synced = synced; if (infoStream != null) { Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.GetGeneration(); IndexFileNameFilter filter = IndexFileNameFilter.GetFilter(); System.String[] files = directory.ListAll(); CommitPoint currentCommitPoint = null; for (int i = 0; i < files.Length; i++) { System.String fileName = files[i]; if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS)) { // This is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (infoStream != null) { Message("init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } catch (System.IO.FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream != null) { Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } catch (System.IO.IOException e) { if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen) { throw e; } else { // Most likely we are opening an index that // has an aborted "future" commit, so suppress // exc in this case sis = null; } } if (sis != null) { CommitPoint commitPoint = new CommitPoint(this,commitsToDelete, directory, sis); if (sis.GetGeneration() == segmentInfos.GetGeneration()) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); if (lastSegmentInfos == null || sis.GetGeneration() > lastSegmentInfos.GetGeneration()) { lastSegmentInfos = sis; } } } } } if (currentCommitPoint == null) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. This can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, segmentInfos.GetCurrentSegmentFileName()); } catch (System.IO.IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } if (infoStream != null) Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName()); currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): commits.Sort(); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. System.Collections.Generic.IEnumerator<System.Collections.Generic.KeyValuePair<System.String, RefCount>> it = refCounts.GetEnumerator(); while (it.MoveNext()) { System.String fileName = (System.String) it.Current.Key; RefCount rc = (RefCount) refCounts[fileName]; if (0 == rc.count) { if (infoStream != null) { Message("init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: policy.OnInit(commits); // Always protect the incoming segmentInfos since // sometime it may not be the most recent commit Checkpoint(segmentInfos, false); startingCommitDeleted = currentCommitPoint.IsDeleted(); DeleteCommits(); }
internal readonly Codec Codec; // for writing new segments /// <summary> /// Constructs a new IndexWriter per the settings given in <code>conf</code>. /// If you want to make "live" changes to this writer instance, use /// <seealso cref="#getConfig()"/>. /// /// <p> /// <b>NOTE:</b> after ths writer is created, the given configuration instance /// cannot be passed to another writer. If you intend to do so, you should /// <seealso cref="IndexWriterConfig#clone() clone"/> it beforehand. /// </summary> /// <param name="d"> /// the index directory. The index is either created or appended /// according <code>conf.getOpenMode()</code>. </param> /// <param name="conf"> /// the configuration settings according to which IndexWriter should /// be initialized. </param> /// <exception cref="IOException"> /// if the directory cannot be read/written to, or if it does not /// exist and <code>conf.getOpenMode()</code> is /// <code>OpenMode.APPEND</code> or if there is any other low-level /// IO error </exception> public IndexWriter(Directory d, IndexWriterConfig conf) { /*if (!InstanceFieldsInitialized) { InitializeInstanceFields(); InstanceFieldsInitialized = true; }*/ readerPool = new ReaderPool(this); conf.SetIndexWriter(this); // prevent reuse by other instances Config_Renamed = new LiveIndexWriterConfig(conf); directory = d; analyzer = Config_Renamed.Analyzer; infoStream = Config_Renamed.InfoStream; mergePolicy = Config_Renamed.MergePolicy; mergePolicy.IndexWriter = this; mergeScheduler = Config_Renamed.MergeScheduler; Codec = Config_Renamed.Codec; BufferedUpdatesStream = new BufferedUpdatesStream(infoStream); PoolReaders = Config_Renamed.ReaderPooling; WriteLock = directory.MakeLock(WRITE_LOCK_NAME); if (!WriteLock.Obtain(Config_Renamed.WriteLockTimeout)) // obtain write lock { throw new LockObtainFailedException("Index locked for write: " + WriteLock); } bool success = false; try { OpenMode_e? mode = Config_Renamed.OpenMode; bool create; if (mode == OpenMode_e.CREATE) { create = true; } else if (mode == OpenMode_e.APPEND) { create = false; } else { // CREATE_OR_APPEND - create only if an index does not exist create = !DirectoryReader.IndexExists(directory); } // If index is too old, reading the segments will throw // IndexFormatTooOldException. segmentInfos = new SegmentInfos(); bool initialIndexExists = true; if (create) { // Try to read first. this is to allow create // against an index that's currently open for // searching. In this case we write the next // segments_N file with no segments: try { segmentInfos.Read(directory); segmentInfos.Clear(); } catch (IOException) { // Likely this means it's a fresh directory initialIndexExists = false; } // Record that we have a change (zero out all // segments) pending: Changed(); } else { segmentInfos.Read(directory); IndexCommit commit = Config_Renamed.IndexCommit; if (commit != null) { // Swap out all segments, but, keep metadata in // SegmentInfos, like version & generation, to // preserve write-once. this is important if // readers are open against the future commit // points. if (commit.Directory != directory) { throw new System.ArgumentException("IndexCommit's directory doesn't match my directory"); } SegmentInfos oldInfos = new SegmentInfos(); oldInfos.Read(directory, commit.SegmentsFileName); segmentInfos.Replace(oldInfos); Changed(); if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "init: loaded commit \"" + commit.SegmentsFileName + "\""); } } } RollbackSegments = segmentInfos.CreateBackupSegmentInfos(); // start with previous field numbers, but new FieldInfos GlobalFieldNumberMap = FieldNumberMap; Config_Renamed.FlushPolicy.Init(Config_Renamed); DocWriter = new DocumentsWriter(this, Config_Renamed, directory); eventQueue = DocWriter.EventQueue(); // Default deleter (for backwards compatibility) is // KeepOnlyLastCommitDeleter: lock (this) { Deleter = new IndexFileDeleter(directory, Config_Renamed.DelPolicy, segmentInfos, infoStream, this, initialIndexExists); } if (Deleter.StartingCommitDeleted) { // Deletion policy deleted the "head" commit point. // We have to mark ourself as changed so that if we // are closed w/o any further changes we write a new // segments_N file. Changed(); } if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "init: create=" + create); MessageState(); } success = true; } finally { if (!success) { if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "init: hit exception on init; releasing write lock"); } WriteLock.Release(); IOUtils.CloseWhileHandlingException(WriteLock); WriteLock = null; } } }
public virtual void TestRollingUpdates_Mem() { Random random = new Random(Random().Next()); BaseDirectoryWrapper dir = NewDirectory(); LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues()); //provider.register(new MemoryCodec()); if ((!"Lucene3x".Equals(Codec.Default.Name)) && Random().NextBoolean()) { Codec.Default = TestUtil.AlwaysPostingsFormat(new MemoryPostingsFormat(Random().nextBoolean(), random.NextFloat())); } MockAnalyzer analyzer = new MockAnalyzer(Random()); analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH); IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); int SIZE = AtLeast(20); int id = 0; IndexReader r = null; IndexSearcher s = null; int numUpdates = (int)(SIZE * (2 + (TEST_NIGHTLY ? 200 * Random().NextDouble() : 5 * Random().NextDouble()))); if (VERBOSE) { Console.WriteLine("TEST: numUpdates=" + numUpdates); } int updateCount = 0; // TODO: sometimes update ids not in order... for (int docIter = 0; docIter < numUpdates; docIter++) { Documents.Document doc = docs.NextDoc(); string myID = "" + id; if (id == SIZE - 1) { id = 0; } else { id++; } if (VERBOSE) { Console.WriteLine(" docIter=" + docIter + " id=" + id); } ((Field)doc.GetField("docid")).StringValue = myID; Term idTerm = new Term("docid", myID); bool doUpdate; if (s != null && updateCount < SIZE) { TopDocs hits = s.Search(new TermQuery(idTerm), 1); Assert.AreEqual(1, hits.TotalHits); doUpdate = !w.TryDeleteDocument(r, hits.ScoreDocs[0].Doc); if (VERBOSE) { if (doUpdate) { Console.WriteLine(" tryDeleteDocument failed"); } else { Console.WriteLine(" tryDeleteDocument succeeded"); } } } else { doUpdate = true; if (VERBOSE) { Console.WriteLine(" no searcher: doUpdate=true"); } } updateCount++; if (doUpdate) { w.UpdateDocument(idTerm, doc); } else { w.AddDocument(doc); } if (docIter >= SIZE && Random().Next(50) == 17) { if (r != null) { r.Dispose(); } bool applyDeletions = Random().NextBoolean(); if (VERBOSE) { Console.WriteLine("TEST: reopen applyDeletions=" + applyDeletions); } r = w.GetReader(applyDeletions); if (applyDeletions) { s = NewSearcher(r); } else { s = null; } Assert.IsTrue(!applyDeletions || r.NumDocs == SIZE, "applyDeletions=" + applyDeletions + " r.NumDocs=" + r.NumDocs + " vs SIZE=" + SIZE); updateCount = 0; } } if (r != null) { r.Dispose(); } w.Commit(); Assert.AreEqual(SIZE, w.NumDocs()); w.Dispose(); TestIndexWriter.AssertNoUnreferencedFiles(dir, "leftover files after rolling updates"); docs.Dispose(); // LUCENE-4455: SegmentInfos infos = new SegmentInfos(); infos.Read(dir); long totalBytes = 0; foreach (SegmentCommitInfo sipc in infos.Segments) { totalBytes += sipc.SizeInBytes(); } long totalBytes2 = 0; foreach (string fileName in dir.ListAll()) { if (!fileName.StartsWith(IndexFileNames.SEGMENTS)) { totalBytes2 += dir.FileLength(fileName); } } Assert.AreEqual(totalBytes2, totalBytes); dir.Dispose(); }
private int CheckAllSegmentsUpgraded(Directory dir) { SegmentInfos infos = new SegmentInfos(); infos.Read(dir); if (VERBOSE) { Console.WriteLine("checkAllSegmentsUpgraded: " + infos); } foreach (SegmentCommitInfo si in infos.Segments) { Assert.AreEqual(Constants.LUCENE_MAIN_VERSION, si.Info.Version); } return infos.Size(); }
public virtual void AddIndexes(Directory[] dirs) { EnsureOpen(); NoDupDirs(dirs); // Do not allow add docs or deletes while we are running: docWriter.PauseAllThreads(); try { if (infoStream != null) Message("flush at addIndexes"); Flush(true, false, true); bool success = false; StartTransaction(false); try { int docCount = 0; lock (this) { EnsureOpen(); for (int i = 0; i < dirs.Length; i++) { SegmentInfos sis = new SegmentInfos(); // read infos from dir sis.Read(dirs[i]); for (int j = 0; j < sis.Count; j++) { SegmentInfo info = sis.Info(j); docCount += info.docCount; System.Diagnostics.Debug.Assert(!segmentInfos.Contains(info)); segmentInfos.Add(info); // add each info } } } // Notify DocumentsWriter that the flushed count just increased docWriter.UpdateFlushedDocCount(docCount); Optimize(); success = true; } finally { if (success) { CommitTransaction(); } else { RollbackTransaction(); } } } catch (System.OutOfMemoryException oom) { HandleOOM(oom, "addIndexes(Directory[])"); } finally { if (docWriter != null) { docWriter.ResumeAllThreads(); } } }
protected internal override object DoBody(System.String segmentFileName) { IndexInput input = directory.OpenInput(segmentFileName); int format = 0; long version = 0; try { format = input.ReadInt(); if (format < 0) { if (format < Lucene.Net.Index.SegmentInfos.CURRENT_FORMAT) throw new CorruptIndexException("Unknown format version: " + format); version = input.ReadLong(); // read version } } finally { input.Close(); } if (format < 0) return (long) version; // We cannot be sure about the format of the file. // Therefore we have to read the whole file and cannot simply seek to the version entry. SegmentInfos sis = new SegmentInfos(); sis.Read(directory, segmentFileName); return (long) sis.GetVersion(); }
/// <summary> Returns userData from latest segments file</summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public static System.Collections.Generic.IDictionary<string, string> ReadCurrentUserData(Directory directory) { SegmentInfos sis = new SegmentInfos(); sis.Read(directory); return sis.GetUserData(); }