public override string ToString() { StringBuilder sb = new StringBuilder(); sb.Append("matchVersion=").Append(matchVersion).Append("\n"); sb.Append("analyzer=").Append(analyzer == null ? "null" : analyzer.GetType().Name).Append("\n"); sb.Append("ramBufferSizeMB=").Append(RAMBufferSizeMB).Append("\n"); sb.Append("maxBufferedDocs=").Append(MaxBufferedDocs).Append("\n"); sb.Append("maxBufferedDeleteTerms=").Append(MaxBufferedDeleteTerms).Append("\n"); sb.Append("mergedSegmentWarmer=").Append(MergedSegmentWarmer).Append("\n"); sb.Append("readerTermsIndexDivisor=").Append(ReaderTermsIndexDivisor).Append("\n"); sb.Append("termIndexInterval=").Append(TermIndexInterval).Append("\n"); // TODO: this should be private to the codec, not settable here sb.Append("delPolicy=").Append(IndexDeletionPolicy.GetType().Name).Append("\n"); IndexCommit commit = IndexCommit; sb.Append("commit=").Append(commit == null ? "null" : commit.ToString()).Append("\n"); sb.Append("openMode=").Append(OpenMode).Append("\n"); sb.Append("similarity=").Append(Similarity.GetType().Name).Append("\n"); sb.Append("mergeScheduler=").Append(MergeScheduler).Append("\n"); sb.Append("default WRITE_LOCK_TIMEOUT=").Append(IndexWriterConfig.WRITE_LOCK_TIMEOUT).Append("\n"); sb.Append("writeLockTimeout=").Append(WriteLockTimeout).Append("\n"); sb.Append("codec=").Append(Codec).Append("\n"); sb.Append("infoStream=").Append(InfoStream.GetType().Name).Append("\n"); sb.Append("mergePolicy=").Append(MergePolicy).Append("\n"); sb.Append("indexerThreadPool=").Append(IndexerThreadPool).Append("\n"); sb.Append("readerPooling=").Append(UseReaderPooling).Append("\n"); sb.Append("perThreadHardLimitMB=").Append(RAMPerThreadHardLimitMB).Append("\n"); sb.Append("useCompoundFile=").Append(UseCompoundFile).Append("\n"); sb.Append("checkIntegrityAtMerge=").Append(CheckIntegrityAtMerge).Append("\n"); return(sb.ToString()); }
/// <summary> /// Creates a new config that that handles the live <see cref="IndexWriter"/> /// settings. /// </summary> internal LiveIndexWriterConfig(IndexWriterConfig config) { maxBufferedDeleteTerms = config.MaxBufferedDeleteTerms; maxBufferedDocs = config.MaxBufferedDocs; mergedSegmentWarmer = config.MergedSegmentWarmer; ramBufferSizeMB = config.RAMBufferSizeMB; readerTermsIndexDivisor = config.ReaderTermsIndexDivisor; termIndexInterval = config.TermIndexInterval; matchVersion = config.matchVersion; analyzer = config.Analyzer; delPolicy = config.IndexDeletionPolicy; commit = config.IndexCommit; openMode = config.OpenMode; similarity = config.Similarity; mergeScheduler = config.MergeScheduler; writeLockTimeout = config.WriteLockTimeout; indexingChain = config.IndexingChain; codec = config.Codec; infoStream = config.InfoStream; mergePolicy = config.MergePolicy; indexerThreadPool = config.IndexerThreadPool; readerPooling = config.UseReaderPooling; flushPolicy = config.FlushPolicy; perThreadHardLimitMB = config.RAMPerThreadHardLimitMB; useCompoundFile = config.UseCompoundFile; checkIntegrityAtMerge = config.CheckIntegrityAtMerge; }
// used by IndexWriterConfig internal LiveIndexWriterConfig(Analyzer analyzer, LuceneVersion matchVersion) { this.analyzer = analyzer; this.matchVersion = matchVersion; ramBufferSizeMB = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB; maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS; maxBufferedDeleteTerms = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS; readerTermsIndexDivisor = IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR; mergedSegmentWarmer = null; termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here delPolicy = new KeepOnlyLastCommitDeletionPolicy(); commit = null; useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM; openMode = Index.OpenMode.CREATE_OR_APPEND; similarity = IndexSearcher.DefaultSimilarity; #if !FEATURE_CONCURRENTMERGESCHEDULER mergeScheduler = new TaskMergeScheduler(); #else mergeScheduler = new ConcurrentMergeScheduler(); #endif writeLockTimeout = IndexWriterConfig.WRITE_LOCK_TIMEOUT; indexingChain = DocumentsWriterPerThread.DefaultIndexingChain; codec = Codec.Default; if (codec == null) { throw new System.NullReferenceException(); } infoStream = Util.InfoStream.Default; mergePolicy = new TieredMergePolicy(); flushPolicy = new FlushByRamOrCountsPolicy(); readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING; indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES); perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB; }
/// <summary> /// <see cref="PersistentSnapshotDeletionPolicy"/> wraps another /// <see cref="IndexDeletionPolicy"/> to enable flexible snapshotting. /// </summary> /// <param name="primary"> /// the <see cref="IndexDeletionPolicy"/> that is used on non-snapshotted /// commits. Snapshotted commits, by definition, are not deleted until /// explicitly released via <see cref="Release(IndexCommit)"/>. </param> /// <param name="dir"> /// the <see cref="Directory"/> which will be used to persist the snapshots /// information. </param> /// <param name="mode"> /// specifies whether a new index should be created, deleting all /// existing snapshots information (immediately), or open an existing /// index, initializing the class with the snapshots information. </param> public PersistentSnapshotDeletionPolicy(IndexDeletionPolicy primary, Directory dir, OpenMode mode) : base(primary) { this.dir = dir; if (mode == OpenMode.CREATE) { ClearPriorSnapshots(); } LoadPriorSnapshots(); if (mode == OpenMode.APPEND && nextWriteGen == 0) { throw IllegalStateException.Create("no snapshots stored in this directory"); } }
/// <summary> /// Initialize the deleter: find all previous commits in /// the <see cref="Directory"/>, incref the files they reference, call /// the policy to let it delete commits. this will remove /// any files not referenced by any of the commits. </summary> /// <exception cref="IOException"> if there is a low-level IO error </exception> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, InfoStream infoStream, IndexWriter writer, bool initialIndexExists) { this.infoStream = infoStream; this.writer = writer; string currentSegmentsFile = segmentInfos.GetSegmentsFileName(); if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: current segments file is \"" + currentSegmentsFile + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.Generation; CommitPoint currentCommitPoint = null; string[] files /* = null*/; try { files = directory.ListAll(); } catch (DirectoryNotFoundException) // LUCENENET: IDE0059: Remove unnecessary value assignment { // it means the directory is empty, so ignore it. files = Arrays.Empty <string>(); } if (currentSegmentsFile != null) { Regex r = IndexFileNames.CODEC_FILE_PATTERN; foreach (string fileName in files) { if (!fileName.EndsWith("write.lock", StringComparison.Ordinal) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN, StringComparison.Ordinal) && (r.IsMatch(fileName) || fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal))) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal)) { // this is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } #pragma warning disable 168 catch (FileNotFoundException e) #pragma warning restore 168 { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } // LUCENENET specific - .NET (thankfully) only has one FileNotFoundException, so we don't need this //catch (NoSuchFileException) //{ // // LUCENE-948: on NFS (and maybe others), if // // you have writers switching back and forth // // between machines, it's very likely that the // // dir listing will be stale and will claim a // // file segments_X exists when in fact it // // doesn't. So, we catch this and handle it // // as if the file does not exist // if (infoStream.IsEnabled("IFD")) // { // infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); // } // sis = null; //} // LUCENENET specific - since NoSuchDirectoryException subclasses FileNotFoundException // in Lucene, we need to catch it here to be on the safe side. catch (DirectoryNotFoundException) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } catch (IOException /*e*/) { if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen && directory.FileLength(fileName) > 0) { throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } else { // Most likely we are opening an index that // has an aborted "future" commit, so suppress // exc in this case sis = null; } } if (sis != null) { CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis); if (sis.Generation == segmentInfos.Generation) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation) { lastSegmentInfos = sis; } } } } } } if (currentCommitPoint == null && currentSegmentsFile != null && initialIndexExists) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. this can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, currentSegmentsFile); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file \"" + currentSegmentsFile + "\"" + e.ToString(), e); } if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "forced open of current segments file " + segmentInfos.GetSegmentsFileName()); } currentCommitPoint = new CommitPoint(commitsToDelete, directory, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): CollectionUtil.TimSort(commits); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. foreach (KeyValuePair <string, RefCount> entry in refCounts) { RefCount rc = entry.Value; string fileName = entry.Key; if (0 == rc.count) { if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: this.policy.OnInit(commits); // Always protect the incoming segmentInfos since // sometime it may not be the most recent commit Checkpoint(segmentInfos, false); startingCommitDeleted = currentCommitPoint == null ? false : currentCommitPoint.IsDeleted; DeleteCommits(); }
/// <summary> /// Sole constructor, taking the incoming /// <see cref="IndexDeletionPolicy"/> to wrap. /// </summary> public SnapshotDeletionPolicy(IndexDeletionPolicy primary) { this.primary = primary; }
/// <summary> /// <see cref="PersistentSnapshotDeletionPolicy"/> wraps another /// <see cref="IndexDeletionPolicy"/> to enable flexible /// snapshotting, passing <see cref="OpenMode.CREATE_OR_APPEND"/> /// by default. /// </summary> /// <param name="primary"> /// the <see cref="IndexDeletionPolicy"/> that is used on non-snapshotted /// commits. Snapshotted commits, by definition, are not deleted until /// explicitly released via <see cref="Release(IndexCommit)"/>. </param> /// <param name="dir"> /// the <see cref="Directory"/> which will be used to persist the snapshots /// information. </param> public PersistentSnapshotDeletionPolicy(IndexDeletionPolicy primary, Directory dir) : this(primary, dir, OpenMode.CREATE_OR_APPEND) { }