public static GenerationFromSegmentsFileName ( string fileName ) : long | ||
fileName | string | |
return | long |
/** @see IndexReader#listCommits */ public new static ICollection <IndexCommitPoint> ListCommits(Directory dir) { string[] files = dir.List(); if (files == null) { throw new System.IO.IOException("cannot read directory " + dir + ": list() returned null"); } ICollection <IndexCommitPoint> commits = new List <IndexCommitPoint>(); SegmentInfos latest = new SegmentInfos(); latest.Read(dir); long currentGen = latest.GetGeneration(); commits.Add(new ReaderCommit(latest, dir)); for (int i = 0; i < files.Length; i++) { String fileName = files[i]; if (fileName.StartsWith(IndexFileNames.SEGMENTS) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN) && SegmentInfos.GenerationFromSegmentsFileName(fileName) < currentGen) { SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt sis.Read(dir, fileName); } catch (System.Exception) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist sis = null; } if (sis != null) { commits.Add(new ReaderCommit(sis, dir)); } } } return(commits); }
private void VerifyCommitOrder <T>(IList <T> commits) where T : IndexCommit { if (commits.Count == 0) { return; } IndexCommit firstCommit = commits[0]; long last = SegmentInfos.GenerationFromSegmentsFileName(firstCommit.SegmentsFileName); Assert.AreEqual(last, firstCommit.Generation); for (int i = 1; i < commits.Count; i++) { IndexCommit commit = commits[i]; long now = SegmentInfos.GenerationFromSegmentsFileName(commit.SegmentsFileName); Assert.IsTrue(now > last, "SegmentInfos commits are out-of-order"); Assert.AreEqual(now, commit.Generation); last = now; } }
private void VerifyCommitOrder(System.Collections.IList commits) { IndexCommit firstCommit = ((IndexCommit)commits[0]); long last = SegmentInfos.GenerationFromSegmentsFileName(firstCommit.GetSegmentsFileName()); Assert.AreEqual(last, firstCommit.GetGeneration()); long lastVersion = firstCommit.GetVersion(); long lastTimestamp = firstCommit.GetTimestamp(); for (int i = 1; i < commits.Count; i++) { IndexCommit commit = ((IndexCommit)commits[i]); long now = SegmentInfos.GenerationFromSegmentsFileName(commit.GetSegmentsFileName()); long nowVersion = commit.GetVersion(); long nowTimestamp = commit.GetTimestamp(); Assert.IsTrue(now > last, "SegmentInfos commits are out-of-order"); Assert.IsTrue(nowVersion > lastVersion, "SegmentInfos versions are out-of-order"); Assert.IsTrue(nowTimestamp >= lastTimestamp, "SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp); Assert.AreEqual(now, commit.GetGeneration()); last = now; lastVersion = nowVersion; lastTimestamp = nowTimestamp; } }
private void VerifyCommitOrder <T>(IList <T> commits) where T : IndexCommit { IndexCommit firstCommit = commits[0]; long last = SegmentInfos.GenerationFromSegmentsFileName(firstCommit.SegmentsFileName); Assert.AreEqual(last, firstCommit.Generation); long lastVersion = firstCommit.Version; long lastTimestamp = firstCommit.Timestamp; for (int i = 1; i < commits.Count; i++) { IndexCommit commit = commits[i]; long now = SegmentInfos.GenerationFromSegmentsFileName(commit.SegmentsFileName); long nowVersion = commit.Version; long nowTimestamp = commit.Timestamp; Assert.IsTrue(now > last, "SegmentInfos commits are out-of-order"); Assert.IsTrue(nowVersion > lastVersion, "SegmentInfos versions are out-of-order"); Assert.IsTrue(nowTimestamp >= lastTimestamp, "SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp); Assert.AreEqual(now, commit.Generation); last = now; lastVersion = nowVersion; lastTimestamp = nowTimestamp; } }
/// <summary> /// Initialize the deleter: find all previous commits in /// the <see cref="Directory"/>, incref the files they reference, call /// the policy to let it delete commits. this will remove /// any files not referenced by any of the commits. </summary> /// <exception cref="IOException"> if there is a low-level IO error </exception> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, InfoStream infoStream, IndexWriter writer, bool initialIndexExists) { this.infoStream = infoStream; this.writer = writer; string currentSegmentsFile = segmentInfos.GetSegmentsFileName(); if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: current segments file is \"" + currentSegmentsFile + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.Generation; CommitPoint currentCommitPoint = null; string[] files = null; try { files = directory.ListAll(); } #pragma warning disable 168 catch (DirectoryNotFoundException e) #pragma warning restore 168 { // it means the directory is empty, so ignore it. files = new string[0]; } if (currentSegmentsFile != null) { Regex r = IndexFileNames.CODEC_FILE_PATTERN; foreach (string fileName in files) { if (!fileName.EndsWith("write.lock", StringComparison.Ordinal) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN, StringComparison.Ordinal) && (r.IsMatch(fileName) || fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal))) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal)) { // this is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } #pragma warning disable 168 catch (FileNotFoundException e) #pragma warning restore 168 { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } // LUCENENET specific - .NET (thankfully) only has one FileNotFoundException, so we don't need this //catch (NoSuchFileException) //{ // // LUCENE-948: on NFS (and maybe others), if // // you have writers switching back and forth // // between machines, it's very likely that the // // dir listing will be stale and will claim a // // file segments_X exists when in fact it // // doesn't. So, we catch this and handle it // // as if the file does not exist // if (infoStream.IsEnabled("IFD")) // { // infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); // } // sis = null; //} // LUCENENET specific - since NoSuchDirectoryException subclasses FileNotFoundException // in Lucene, we need to catch it here to be on the safe side. catch (System.IO.DirectoryNotFoundException) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } catch (IOException /*e*/) { if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen && directory.FileLength(fileName) > 0) { throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } else { // Most likely we are opening an index that // has an aborted "future" commit, so suppress // exc in this case sis = null; } } if (sis != null) { CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis); if (sis.Generation == segmentInfos.Generation) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation) { lastSegmentInfos = sis; } } } } } } if (currentCommitPoint == null && currentSegmentsFile != null && initialIndexExists) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. this can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, currentSegmentsFile); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file \"" + currentSegmentsFile + "\"" + e.ToString(), e); } if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "forced open of current segments file " + segmentInfos.GetSegmentsFileName()); } currentCommitPoint = new CommitPoint(commitsToDelete, directory, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): CollectionUtil.TimSort(commits); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. foreach (KeyValuePair <string, RefCount> entry in refCounts) { RefCount rc = entry.Value; string fileName = entry.Key; if (0 == rc.count) { if (infoStream.IsEnabled("IFD")) { infoStream.Message("IFD", "init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: this.policy.OnInit(commits); // Always protect the incoming segmentInfos since // sometime it may not be the most recent commit Checkpoint(segmentInfos, false); startingCommitDeleted = currentCommitPoint == null ? false : currentCommitPoint.IsDeleted; DeleteCommits(); }
/// <summary> Initialize the deleter: find all previous commits in /// the Directory, incref the files they reference, call /// the policy to let it delete commits. The incoming /// segmentInfos must have been loaded from a commit point /// and not yet modified. This will remove any files not /// referenced by any of the commits. /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.TextWriter infoStream, DocumentsWriter docWriter) { this.docWriter = docWriter; this.infoStream = infoStream; if (infoStream != null) { Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.GetGeneration(); IndexFileNameFilter filter = IndexFileNameFilter.GetFilter(); System.String[] files = directory.List(); if (files == null) { throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null"); } CommitPoint currentCommitPoint = null; for (int i = 0; i < files.Length; i++) { System.String fileName = files[i]; if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS)) { // This is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen) { if (infoStream != null) { Message("init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } catch (System.IO.FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream != null) { Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } if (sis != null) { CommitPoint commitPoint = new CommitPoint(this, sis); if (sis.GetGeneration() == segmentInfos.GetGeneration()) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); } } } } } if (currentCommitPoint == null) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. This can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, segmentInfos.GetCurrentSegmentFileName()); } catch (System.IO.IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } if (infoStream != null) { Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName()); } currentCommitPoint = new CommitPoint(this, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): commits.Sort(); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. System.Collections.IEnumerator it = refCounts.Keys.GetEnumerator(); while (it.MoveNext()) { System.String fileName = (System.String)it.Current; RefCount rc = (RefCount)refCounts[fileName]; if (0 == rc.count) { if (infoStream != null) { Message("init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: policy.OnInit(commits); // It's OK for the onInit to remove the current commit // point; we just have to checkpoint our in-memory // SegmentInfos to protect those files that it uses: if (currentCommitPoint.deleted) { Checkpoint(segmentInfos, false); } DeleteCommits(); }
/// <summary> /// Returns all commit points that exist in the <see cref="Store.Directory"/>. /// Normally, because the default is /// <see cref="KeepOnlyLastCommitDeletionPolicy"/>, there would be only /// one commit point. But if you're using a custom /// <see cref="IndexDeletionPolicy"/> then there could be many commits. /// Once you have a given commit, you can open a reader on /// it by calling <see cref="DirectoryReader.Open(IndexCommit)"/> /// There must be at least one commit in /// the <see cref="Store.Directory"/>, else this method throws /// <see cref="IndexNotFoundException"/>. Note that if a commit is in /// progress while this method is running, that commit /// may or may not be returned. /// </summary> /// <returns> a sorted list of <see cref="Index.IndexCommit"/>s, from oldest /// to latest. </returns> public static IList <IndexCommit> ListCommits(Directory dir) { string[] files = dir.ListAll(); List <IndexCommit> commits = new List <IndexCommit>(); SegmentInfos latest = new SegmentInfos(); latest.Read(dir); long currentGen = latest.Generation; commits.Add(new StandardDirectoryReader.ReaderCommit(latest, dir)); for (int i = 0; i < files.Length; i++) { string fileName = files[i]; if (fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN, StringComparison.Ordinal) && SegmentInfos.GenerationFromSegmentsFileName(fileName) < currentGen) { SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt sis.Read(dir, fileName); } catch (FileNotFoundException) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist sis = null; } // LUCENENET specific - .NET (thankfully) only has one FileNotFoundException, so we don't need this //catch (NoSuchFileException) //{ // sis = null; //} // LUCENENET specific - since NoSuchDirectoryException subclasses FileNotFoundException // in Lucene, we need to catch it here to be on the safe side. catch (System.IO.DirectoryNotFoundException) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist sis = null; } if (sis != null) { commits.Add(new StandardDirectoryReader.ReaderCommit(sis, dir)); } } } // Ensure that the commit points are sorted in ascending order. commits.Sort(); return(commits); }
/// <summary> Initialize the deleter: find all previous commits in /// the Directory, incref the files they reference, call /// the policy to let it delete commits. This will remove /// any files not referenced by any of the commits. /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, HashSet <string> synced) { this.docWriter = docWriter; this.infoStream = infoStream; this.synced = synced; if (infoStream != null) { Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.Generation; IndexFileNameFilter filter = IndexFileNameFilter.Filter; System.String[] files = directory.ListAll(); CommitPoint currentCommitPoint = null; for (int i = 0; i < files.Length; i++) { System.String fileName = files[i]; if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS)) { // This is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (infoStream != null) { Message("init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } catch (System.IO.FileNotFoundException) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream != null) { Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } catch (System.IO.IOException) { if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen) { throw; } else { // Most likely we are opening an index that // has an aborted "future" commit, so suppress // exc in this case sis = null; } } if (sis != null) { CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis); if (sis.Generation == segmentInfos.Generation) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation) { lastSegmentInfos = sis; } } } } } if (currentCommitPoint == null) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. This can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, segmentInfos.GetCurrentSegmentFileName()); } catch (System.IO.IOException) { throw new CorruptIndexException("failed to locate current segments_N file"); } if (infoStream != null) { Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName()); } currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): commits.Sort(); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. foreach (KeyValuePair <string, RefCount> entry in refCounts) { string fileName = entry.Key; RefCount rc = refCounts[fileName]; if (0 == rc.count) { if (infoStream != null) { Message("init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: policy.OnInit(commits); // Always protect the incoming segmentInfos since // sometime it may not be the most recent commit Checkpoint(segmentInfos, false); startingCommitDeleted = currentCommitPoint.IsDeleted; DeleteCommits(); }