public virtual void Split(DirectoryInfo destDir, string[] segs) { destDir.Create(); FSDirectory destFSDir = FSDirectory.Open(destDir); SegmentInfos destInfos = new SegmentInfos(); destInfos.Counter = infos.Counter; foreach (string n in segs) { SegmentCommitInfo infoPerCommit = GetInfo(n); SegmentInfo info = infoPerCommit.Info; // Same info just changing the dir: SegmentInfo newInfo = new SegmentInfo(destFSDir, info.Version, info.Name, info.DocCount, info.UseCompoundFile, info.Codec, info.Diagnostics); destInfos.Add(new SegmentCommitInfo(newInfo, infoPerCommit.DelCount, infoPerCommit.DelGen, infoPerCommit.FieldInfosGen)); // now copy files over ICollection <string> files = infoPerCommit.Files(); foreach (string srcName in files) { FileInfo srcFile = new FileInfo(Path.Combine(dir.FullName, srcName)); FileInfo destFile = new FileInfo(Path.Combine(destDir.FullName, srcName)); CopyFile(srcFile, destFile); } } destInfos.Changed(); destInfos.Commit(destFSDir); // Console.WriteLine("destDir:"+destDir.getAbsolutePath()); }
/// <summary> /// Copies the segment files as-is into the IndexWriter's directory. </summary> private SegmentCommitInfo CopySegmentAsIs(SegmentCommitInfo info, string segName, IDictionary<string, string> dsNames, ISet<string> dsFilesCopied, IOContext context, ISet<string> copiedFiles) { // Determine if the doc store of this segment needs to be copied. It's // only relevant for segments that share doc store with others, // because the DS might have been copied already, in which case we // just want to update the DS name of this SegmentInfo. string dsName = Lucene3xSegmentInfoFormat.GetDocStoreSegment(info.Info); Debug.Assert(dsName != null); string newDsName; if (dsNames.ContainsKey(dsName)) { newDsName = dsNames[dsName]; } else { dsNames[dsName] = segName; newDsName = segName; } // note: we don't really need this fis (its copied), but we load it up // so we don't pass a null value to the si writer FieldInfos fis = SegmentReader.ReadFieldInfos(info); ISet<string> docStoreFiles3xOnly = Lucene3xCodec.GetDocStoreFiles(info.Info); IDictionary<string, string> attributes; // copy the attributes map, we might modify it below. // also we need to ensure its read-write, since we will invoke the SIwriter (which might want to set something). if (info.Info.Attributes() == null) { attributes = new Dictionary<string, string>(); } else { attributes = new Dictionary<string, string>(info.Info.Attributes()); } if (docStoreFiles3xOnly != null) { // only violate the codec this way if it's preflex & // shares doc stores // change docStoreSegment to newDsName attributes[Lucene3xSegmentInfoFormat.DS_NAME_KEY] = newDsName; } //System.out.println("copy seg=" + info.info.name + " version=" + info.info.getVersion()); // Same SI as before but we change directory, name and docStoreSegment: SegmentInfo newInfo = new SegmentInfo(directory, info.Info.Version, segName, info.Info.DocCount, info.Info.UseCompoundFile, info.Info.Codec, info.Info.Diagnostics, attributes); SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.DelCount, info.DelGen, info.FieldInfosGen); HashSet<string> segFiles = new HashSet<string>(); // Build up new segment's file names. Must do this // before writing SegmentInfo: foreach (string file in info.Files()) { string newFileName; if (docStoreFiles3xOnly != null && docStoreFiles3xOnly.Contains(file)) { newFileName = newDsName + Lucene.Net.Index.IndexFileNames.StripSegmentName(file); } else { newFileName = segName + Lucene.Net.Index.IndexFileNames.StripSegmentName(file); } segFiles.Add(newFileName); } newInfo.Files = segFiles; // We must rewrite the SI file because it references // segment name (its own name, if its 3.x, and doc // store segment name): TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory); Codec currentCodec = newInfo.Codec; try { currentCodec.SegmentInfoFormat().SegmentInfoWriter.Write(trackingDir, newInfo, fis, context); } catch (System.NotSupportedException uoe) { if (currentCodec is Lucene3xCodec) { // OK: 3x codec cannot write a new SI file; // SegmentInfos will write this on commit } else { throw uoe; } } ICollection<string> siFiles = trackingDir.CreatedFiles; bool success = false; try { // Copy the segment's files foreach (string file in info.Files()) { string newFileName; if (docStoreFiles3xOnly != null && docStoreFiles3xOnly.Contains(file)) { newFileName = newDsName + Lucene.Net.Index.IndexFileNames.StripSegmentName(file); if (dsFilesCopied.Contains(newFileName)) { continue; } dsFilesCopied.Add(newFileName); } else { newFileName = segName + Lucene.Net.Index.IndexFileNames.StripSegmentName(file); } if (siFiles.Contains(newFileName)) { // We already rewrote this above continue; } Debug.Assert(!SlowFileExists(directory, newFileName), "file \"" + newFileName + "\" already exists; siFiles=" + siFiles); Debug.Assert(!copiedFiles.Contains(file), "file \"" + file + "\" is being copied more than once"); copiedFiles.Add(file); info.Info.Dir.Copy(directory, file, newFileName, context); } success = true; } finally { if (!success) { foreach (string file in newInfo.Files) { try { directory.DeleteFile(file); } catch (Exception) { } } } } return newInfoPerCommit; }
/// <summary> /// Merges the provided indexes into this index. /// /// <p> /// The provided IndexReaders are not closed. /// /// <p> /// See <seealso cref="#addIndexes"/> for details on transactional semantics, temporary /// free space required in the Directory, and non-CFS segments on an Exception. /// /// <p> /// <b>NOTE</b>: if this method hits an OutOfMemoryError you should immediately /// close the writer. See <a href="#OOME">above</a> for details. /// /// <p> /// <b>NOTE:</b> empty segments are dropped by this method and not added to this /// index. /// /// <p> /// <b>NOTE:</b> this method merges all given <seealso cref="IndexReader"/>s in one /// merge. If you intend to merge a large number of readers, it may be better /// to call this method multiple times, each time with a small set of readers. /// In principle, if you use a merge policy with a {@code mergeFactor} or /// {@code maxMergeAtOnce} parameter, you should pass that many readers in one /// call. Also, if the given readers are <seealso cref="DirectoryReader"/>s, they can be /// opened with {@code termIndexInterval=-1} to save RAM, since during merge /// the in-memory structure is not used. See /// <seealso cref="DirectoryReader#open(Directory, int)"/>. /// /// <p> /// <b>NOTE</b>: if you call <seealso cref="#close(boolean)"/> with <tt>false</tt>, which /// aborts all running merges, then any thread still running this method might /// hit a <seealso cref="MergePolicy.MergeAbortedException"/>. /// </summary> /// <exception cref="CorruptIndexException"> /// if the index is corrupt </exception> /// <exception cref="IOException"> /// if there is a low-level IO error </exception> public virtual void AddIndexes(params IndexReader[] readers) { EnsureOpen(); int numDocs = 0; try { if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "flush at addIndexes(IndexReader...)"); } Flush(false, true); string mergedName = NewSegmentName(); IList<AtomicReader> mergeReaders = new List<AtomicReader>(); foreach (IndexReader indexReader in readers) { numDocs += indexReader.NumDocs(); foreach (AtomicReaderContext ctx in indexReader.Leaves()) { mergeReaders.Add(ctx.AtomicReader); } } IOContext context = new IOContext(new MergeInfo(numDocs, -1, true, -1)); // TODO: somehow we should fix this merge so it's // abortable so that IW.close(false) is able to stop it TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory); SegmentInfo info = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergedName, -1, false, Codec, null); SegmentMerger merger = new SegmentMerger(mergeReaders, info, infoStream, trackingDir, Config_Renamed.TermIndexInterval, MergeState.CheckAbort.NONE, GlobalFieldNumberMap, context, Config_Renamed.CheckIntegrityAtMerge); if (!merger.ShouldMerge()) { return; } MergeState mergeState; bool success = false; try { mergeState = merger.Merge(); // merge 'em success = true; } finally { if (!success) { lock (this) { Deleter.Refresh(info.Name); } } } SegmentCommitInfo infoPerCommit = new SegmentCommitInfo(info, 0, -1L, -1L); info.Files = new HashSet<string>(trackingDir.CreatedFiles); trackingDir.CreatedFiles.Clear(); SetDiagnostics(info, SOURCE_ADDINDEXES_READERS); bool useCompoundFile; lock (this) // Guard segmentInfos { if (StopMerges) { Deleter.DeleteNewFiles(infoPerCommit.Files()); return; } EnsureOpen(); useCompoundFile = mergePolicy.UseCompoundFile(segmentInfos, infoPerCommit); } // Now create the compound file if needed if (useCompoundFile) { ICollection<string> filesToDelete = infoPerCommit.Files(); try { CreateCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, info, context); } finally { // delete new non cfs files directly: they were never // registered with IFD lock (this) { Deleter.DeleteNewFiles(filesToDelete); } } info.UseCompoundFile = true; } // Have codec write SegmentInfo. Must do this after // creating CFS so that 1) .si isn't slurped into CFS, // and 2) .si reflects useCompoundFile=true change // above: success = false; try { Codec.SegmentInfoFormat().SegmentInfoWriter.Write(trackingDir, info, mergeState.FieldInfos, context); success = true; } finally { if (!success) { lock (this) { Deleter.Refresh(info.Name); } } } info.AddFiles(trackingDir.CreatedFiles); // Register the new segment lock (this) { if (StopMerges) { Deleter.DeleteNewFiles(info.Files); return; } EnsureOpen(); segmentInfos.Add(infoPerCommit); Checkpoint(); } } catch (System.OutOfMemoryException oom) { HandleOOM(oom, "addIndexes(IndexReader...)"); } }
/// <summary> /// Flush all pending docs to a new segment </summary> internal virtual FlushedSegment Flush() { Debug.Assert(numDocsInRAM > 0); Debug.Assert(DeleteSlice.Empty, "all deletes must be applied in prepareFlush"); SegmentInfo_Renamed.DocCount = numDocsInRAM; SegmentWriteState flushState = new SegmentWriteState(InfoStream, Directory, SegmentInfo_Renamed, FieldInfos.Finish(), IndexWriterConfig.TermIndexInterval, PendingUpdates, new IOContext(new FlushInfo(numDocsInRAM, BytesUsed()))); double startMBUsed = BytesUsed() / 1024.0 / 1024.0; // Apply delete-by-docID now (delete-byDocID only // happens when an exception is hit processing that // doc, eg if analyzer has some problem w/ the text): if (PendingUpdates.DocIDs.Count > 0) { flushState.LiveDocs = Codec.LiveDocsFormat().NewLiveDocs(numDocsInRAM); foreach (int delDocID in PendingUpdates.DocIDs) { flushState.LiveDocs.Clear(delDocID); } flushState.DelCountOnFlush = PendingUpdates.DocIDs.Count; PendingUpdates.BytesUsed.AddAndGet(-PendingUpdates.DocIDs.Count * BufferedUpdates.BYTES_PER_DEL_DOCID); PendingUpdates.DocIDs.Clear(); } if (Aborting) { if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "flush: skip because aborting is set"); } return null; } if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "flush postings as segment " + flushState.SegmentInfo.Name + " numDocs=" + numDocsInRAM); } bool success = false; try { Consumer.Flush(flushState); PendingUpdates.Terms.Clear(); SegmentInfo_Renamed.Files = new HashSet<string>(Directory.CreatedFiles); SegmentCommitInfo segmentInfoPerCommit = new SegmentCommitInfo(SegmentInfo_Renamed, 0, -1L, -1L); if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "new segment has " + (flushState.LiveDocs == null ? 0 : (flushState.SegmentInfo.DocCount - flushState.DelCountOnFlush)) + " deleted docs"); InfoStream.Message("DWPT", "new segment has " + (flushState.FieldInfos.HasVectors() ? "vectors" : "no vectors") + "; " + (flushState.FieldInfos.HasNorms() ? "norms" : "no norms") + "; " + (flushState.FieldInfos.HasDocValues() ? "docValues" : "no docValues") + "; " + (flushState.FieldInfos.HasProx() ? "prox" : "no prox") + "; " + (flushState.FieldInfos.HasFreq() ? "freqs" : "no freqs")); InfoStream.Message("DWPT", "flushedFiles=" + segmentInfoPerCommit.Files()); InfoStream.Message("DWPT", "flushed codec=" + Codec); } BufferedUpdates segmentDeletes; if (PendingUpdates.Queries.Count == 0 && PendingUpdates.NumericUpdates.Count == 0 && PendingUpdates.BinaryUpdates.Count == 0) { PendingUpdates.Clear(); segmentDeletes = null; } else { segmentDeletes = PendingUpdates; } if (InfoStream.IsEnabled("DWPT")) { double newSegmentSize = segmentInfoPerCommit.SizeInBytes() / 1024.0 / 1024.0; InfoStream.Message("DWPT", "flushed: segment=" + SegmentInfo_Renamed.Name + " ramUsed=" + startMBUsed.ToString(Nf) + " MB" + " newFlushedSize(includes docstores)=" + newSegmentSize.ToString(Nf) + " MB" + " docs/MB=" + (flushState.SegmentInfo.DocCount / newSegmentSize).ToString(Nf)); } Debug.Assert(SegmentInfo_Renamed != null); FlushedSegment fs = new FlushedSegment(segmentInfoPerCommit, flushState.FieldInfos, segmentDeletes, flushState.LiveDocs, flushState.DelCountOnFlush); SealFlushedSegment(fs); success = true; return fs; } finally { if (!success) { Abort(FilesToDelete); } } }
/// <summary> /// Flush all pending docs to a new segment </summary> internal virtual FlushedSegment Flush() { Debug.Assert(numDocsInRAM > 0); Debug.Assert(DeleteSlice.Empty, "all deletes must be applied in prepareFlush"); SegmentInfo_Renamed.DocCount = numDocsInRAM; SegmentWriteState flushState = new SegmentWriteState(InfoStream, Directory, SegmentInfo_Renamed, FieldInfos.Finish(), IndexWriterConfig.TermIndexInterval, PendingUpdates, new IOContext(new FlushInfo(numDocsInRAM, BytesUsed()))); double startMBUsed = BytesUsed() / 1024.0 / 1024.0; // Apply delete-by-docID now (delete-byDocID only // happens when an exception is hit processing that // doc, eg if analyzer has some problem w/ the text): if (PendingUpdates.DocIDs.Count > 0) { flushState.LiveDocs = Codec.LiveDocsFormat().NewLiveDocs(numDocsInRAM); foreach (int delDocID in PendingUpdates.DocIDs) { flushState.LiveDocs.Clear(delDocID); } flushState.DelCountOnFlush = PendingUpdates.DocIDs.Count; PendingUpdates.BytesUsed.AddAndGet(-PendingUpdates.DocIDs.Count * BufferedUpdates.BYTES_PER_DEL_DOCID); PendingUpdates.DocIDs.Clear(); } if (Aborting) { if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "flush: skip because aborting is set"); } return(null); } if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "flush postings as segment " + flushState.SegmentInfo.Name + " numDocs=" + numDocsInRAM); } bool success = false; try { Consumer.Flush(flushState); PendingUpdates.Terms.Clear(); SegmentInfo_Renamed.Files = new HashSet <string>(Directory.CreatedFiles); SegmentCommitInfo segmentInfoPerCommit = new SegmentCommitInfo(SegmentInfo_Renamed, 0, -1L, -1L); if (InfoStream.IsEnabled("DWPT")) { InfoStream.Message("DWPT", "new segment has " + (flushState.LiveDocs == null ? 0 : (flushState.SegmentInfo.DocCount - flushState.DelCountOnFlush)) + " deleted docs"); InfoStream.Message("DWPT", "new segment has " + (flushState.FieldInfos.HasVectors() ? "vectors" : "no vectors") + "; " + (flushState.FieldInfos.HasNorms() ? "norms" : "no norms") + "; " + (flushState.FieldInfos.HasDocValues() ? "docValues" : "no docValues") + "; " + (flushState.FieldInfos.HasProx() ? "prox" : "no prox") + "; " + (flushState.FieldInfos.HasFreq() ? "freqs" : "no freqs")); InfoStream.Message("DWPT", "flushedFiles=" + segmentInfoPerCommit.Files()); InfoStream.Message("DWPT", "flushed codec=" + Codec); } BufferedUpdates segmentDeletes; if (PendingUpdates.Queries.Count == 0 && PendingUpdates.NumericUpdates.Count == 0 && PendingUpdates.BinaryUpdates.Count == 0) { PendingUpdates.Clear(); segmentDeletes = null; } else { segmentDeletes = PendingUpdates; } if (InfoStream.IsEnabled("DWPT")) { double newSegmentSize = segmentInfoPerCommit.SizeInBytes() / 1024.0 / 1024.0; InfoStream.Message("DWPT", "flushed: segment=" + SegmentInfo_Renamed.Name + " ramUsed=" + startMBUsed.ToString(Nf) + " MB" + " newFlushedSize(includes docstores)=" + newSegmentSize.ToString(Nf) + " MB" + " docs/MB=" + (flushState.SegmentInfo.DocCount / newSegmentSize).ToString(Nf)); } Debug.Assert(SegmentInfo_Renamed != null); FlushedSegment fs = new FlushedSegment(segmentInfoPerCommit, flushState.FieldInfos, segmentDeletes, flushState.LiveDocs, flushState.DelCountOnFlush); SealFlushedSegment(fs); success = true; return(fs); } finally { if (!success) { Abort(FilesToDelete); } } }