/// <summary>Take a snapshot of the most recent commit to the /// index. You must call release() to free this snapshot. /// Note that while the snapshot is held, the files it /// references will not be deleted, which will consume /// additional disk space in your index. If you take a /// snapshot at a particularly bad time (say just before /// you call optimize()) then in the worst case this could /// consume an extra 1X of your total index size, until /// you release the snapshot. /// </summary> // TODO 3.0: change this to return IndexCommit instead public virtual IndexCommitPoint Snapshot() { lock (this) { if (snapshot == null) { snapshot = lastCommit.GetSegmentsFileName(); } else { throw new System.SystemException("snapshot is already set; please call release() first"); } return(lastCommit); } }
internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy, IndexCommit commit, bool readOnly) { SegmentInfos.FindSegmentsFile finder = new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory, readOnly); if (commit == null) { return((DirectoryIndexReader)finder.Run()); } else { if (directory != commit.GetDirectory()) { throw new System.IO.IOException("the specified commit does not match the specified Directory"); } // this can and will directly throw IOException if the specified commit point has been deleted return((DirectoryIndexReader)finder.DoBody(commit.GetSegmentsFileName())); } }
public virtual void OnCommit(System.Collections.IList commits) { Enclosing_Instance.VerifyCommitOrder(commits); IndexCommit lastCommit = (IndexCommit)commits[commits.Count - 1]; // Any commit older than expireTime should be deleted: double expireTime = dir.FileModified(lastCommit.GetSegmentsFileName()) / 1000.0 - expirationTimeSeconds; System.Collections.IEnumerator it = commits.GetEnumerator(); while (it.MoveNext()) { IndexCommit commit = (IndexCommit)it.Current; double modTime = dir.FileModified(commit.GetSegmentsFileName()) / 1000.0; if (commit != lastCommit && modTime < expireTime) { commit.Delete(); numDelete += 1; } } }
private void VerifyCommitOrder(System.Collections.IList commits) { IndexCommit firstCommit = ((IndexCommit)commits[0]); long last = SegmentInfos.GenerationFromSegmentsFileName(firstCommit.GetSegmentsFileName()); Assert.AreEqual(last, firstCommit.GetGeneration()); long lastVersion = firstCommit.GetVersion(); long lastTimestamp = firstCommit.GetTimestamp(); for (int i = 1; i < commits.Count; i++) { IndexCommit commit = ((IndexCommit)commits[i]); long now = SegmentInfos.GenerationFromSegmentsFileName(commit.GetSegmentsFileName()); long nowVersion = commit.GetVersion(); long nowTimestamp = commit.GetTimestamp(); Assert.IsTrue(now > last, "SegmentInfos commits are out-of-order"); Assert.IsTrue(nowVersion > lastVersion, "SegmentInfos versions are out-of-order"); Assert.IsTrue(nowTimestamp >= lastTimestamp, "SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp); Assert.AreEqual(now, commit.GetGeneration()); last = now; lastVersion = nowVersion; lastTimestamp = nowTimestamp; } }
public override System.String GetSegmentsFileName() { return(cp.GetSegmentsFileName()); }
public System.Object Run(IndexCommit commit) { if (commit != null) { if (directory != commit.GetDirectory()) throw new System.IO.IOException("the specified commit does not match the specified Directory"); return DoBody(commit.GetSegmentsFileName()); } System.String segmentFileName = null; long lastGen = - 1; long gen = 0; int genLookaheadCount = 0; System.IO.IOException exc = null; bool retry = false; int method = 0; // Loop until we succeed in calling doBody() without // hitting an IOException. An IOException most likely // means a commit was in process and has finished, in // the time it took us to load the now-old infos files // (and segments files). It's also possible it's a // true error (corrupt index). To distinguish these, // on each retry we must see "forward progress" on // which generation we are trying to load. If we // don't, then the original error is real and we throw // it. // We have three methods for determining the current // generation. We try the first two in parallel, and // fall back to the third when necessary. while (true) { if (0 == method) { // Method 1: list the directory and use the highest // segments_N file. This method works well as long // as there is no stale caching on the directory // contents (NOTE: NFS clients often have such stale // caching): System.String[] files = null; long genA = - 1; files = directory.ListAll(); if (files != null) genA = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files); Lucene.Net.Index.SegmentInfos.Message("directory listing genA=" + genA); // Method 2: open segments.gen and read its // contents. Then we take the larger of the two // gen's. This way, if either approach is hitting // a stale cache (NFS) we have a better chance of // getting the right generation. long genB = - 1; for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++) { IndexInput genInput = null; try { genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN); } catch (System.IO.FileNotFoundException e) { Lucene.Net.Index.SegmentInfos.Message("segments.gen open: FileNotFoundException " + e); break; } catch (System.IO.IOException e) { Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e); } if (genInput != null) { try { int version = genInput.ReadInt(); if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS) { long gen0 = genInput.ReadLong(); long gen1 = genInput.ReadLong(); Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1); if (gen0 == gen1) { // The file is consistent. genB = gen0; break; } } } catch (System.IO.IOException err2) { // will retry } finally { genInput.Close(); } } try { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec)); } catch (System.Threading.ThreadInterruptedException ie) { // In 3.0 we will change this to throw // InterruptedException instead SupportClass.ThreadClass.Current().Interrupt(); throw new System.SystemException(ie.Message, ie); } } Lucene.Net.Index.SegmentInfos.Message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB); // Pick the larger of the two gen's: if (genA > genB) gen = genA; else gen = genB; if (gen == - 1) { // Neither approach found a generation System.String s; if (files != null) { s = ""; for (int i = 0; i < files.Length; i++) s += (" " + files[i]); } else s = " null"; throw new System.IO.FileNotFoundException("no segments* file found in " + directory + ": files:" + s); } } // Third method (fallback if first & second methods // are not reliable): since both directory cache and // file contents cache seem to be stale, just // advance the generation. if (1 == method || (0 == method && lastGen == gen && retry)) { method = 1; if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount) { gen++; genLookaheadCount++; Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen); } } if (lastGen == gen) { // This means we're about to try the same // segments_N last tried. This is allowed, // exactly once, because writer could have been in // the process of writing segments_N last time. if (retry) { // OK, we've tried the same segments_N file // twice in a row, so this must be a real // error. We throw the original exception we // got. throw exc; } else { retry = true; } } else if (0 == method) { // Segment file has advanced since our last loop, so // reset retry: retry = false; } lastGen = gen; segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); try { System.Object v = DoBody(segmentFileName); if (exc != null) { Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName); } return v; } catch (System.IO.IOException err) { // Save the original root cause: if (exc == null) { exc = err; } Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen); if (!retry && gen > 1) { // This is our first time trying this segments // file (because retry is false), and, there is // possibly a segments_(N-1) (because gen > 1). // So, check if the segments_(N-1) exists and // try it if so: System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1); bool prevExists; prevExists = directory.FileExists(prevSegmentFileName); if (prevExists) { Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'"); try { System.Object v = DoBody(prevSegmentFileName); if (exc != null) { Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName); } return v; } catch (System.IO.IOException err2) { Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry"); } } } } } }
public System.Object Run(IndexCommit commit) { if (commit != null) { if (directory != commit.GetDirectory()) { throw new System.IO.IOException("the specified commit does not match the specified Directory"); } return(DoBody(commit.GetSegmentsFileName())); } System.String segmentFileName = null; long lastGen = -1; long gen = 0; int genLookaheadCount = 0; System.IO.IOException exc = null; bool retry = false; int method = 0; // Loop until we succeed in calling doBody() without // hitting an IOException. An IOException most likely // means a commit was in process and has finished, in // the time it took us to load the now-old infos files // (and segments files). It's also possible it's a // true error (corrupt index). To distinguish these, // on each retry we must see "forward progress" on // which generation we are trying to load. If we // don't, then the original error is real and we throw // it. // We have three methods for determining the current // generation. We try the first two in parallel, and // fall back to the third when necessary. while (true) { if (0 == method) { // Method 1: list the directory and use the highest // segments_N file. This method works well as long // as there is no stale caching on the directory // contents (NOTE: NFS clients often have such stale // caching): System.String[] files = null; long genA = -1; files = directory.ListAll(); if (files != null) { genA = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files); } Lucene.Net.Index.SegmentInfos.Message("directory listing genA=" + genA); // Method 2: open segments.gen and read its // contents. Then we take the larger of the two // gen's. This way, if either approach is hitting // a stale cache (NFS) we have a better chance of // getting the right generation. long genB = -1; for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++) { IndexInput genInput = null; try { genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN); } catch (System.IO.FileNotFoundException e) { Lucene.Net.Index.SegmentInfos.Message("segments.gen open: FileNotFoundException " + e); break; } catch (System.IO.IOException e) { Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e); } if (genInput != null) { try { int version = genInput.ReadInt(); if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS) { long gen0 = genInput.ReadLong(); long gen1 = genInput.ReadLong(); Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1); if (gen0 == gen1) { // The file is consistent. genB = gen0; break; } } } catch (System.IO.IOException err2) { // will retry } finally { genInput.Close(); } } try { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec)); } catch (System.Threading.ThreadInterruptedException ie) { // In 3.0 we will change this to throw // InterruptedException instead SupportClass.ThreadClass.Current().Interrupt(); throw new System.SystemException(ie.Message, ie); } } Lucene.Net.Index.SegmentInfos.Message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB); // Pick the larger of the two gen's: if (genA > genB) { gen = genA; } else { gen = genB; } if (gen == -1) { // Neither approach found a generation System.String s; if (files != null) { s = ""; for (int i = 0; i < files.Length; i++) { s += (" " + files[i]); } } else { s = " null"; } throw new System.IO.FileNotFoundException("no segments* file found in " + directory + ": files:" + s); } } // Third method (fallback if first & second methods // are not reliable): since both directory cache and // file contents cache seem to be stale, just // advance the generation. if (1 == method || (0 == method && lastGen == gen && retry)) { method = 1; if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount) { gen++; genLookaheadCount++; Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen); } } if (lastGen == gen) { // This means we're about to try the same // segments_N last tried. This is allowed, // exactly once, because writer could have been in // the process of writing segments_N last time. if (retry) { // OK, we've tried the same segments_N file // twice in a row, so this must be a real // error. We throw the original exception we // got. throw exc; } else { retry = true; } } else if (0 == method) { // Segment file has advanced since our last loop, so // reset retry: retry = false; } lastGen = gen; segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); try { System.Object v = DoBody(segmentFileName); if (exc != null) { Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName); } return(v); } catch (System.IO.IOException err) { // Save the original root cause: if (exc == null) { exc = err; } Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen); if (!retry && gen > 1) { // This is our first time trying this segments // file (because retry is false), and, there is // possibly a segments_(N-1) (because gen > 1). // So, check if the segments_(N-1) exists and // try it if so: System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1); bool prevExists; prevExists = directory.FileExists(prevSegmentFileName); if (prevExists) { Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'"); try { System.Object v = DoBody(prevSegmentFileName); if (exc != null) { Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName); } return(v); } catch (System.IO.IOException err2) { Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry"); } } } } } }
internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy, IndexCommit commit, bool readOnly) { SegmentInfos.FindSegmentsFile finder = new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory, readOnly); if (commit == null) return (DirectoryIndexReader) finder.Run(); else { if (directory != commit.GetDirectory()) throw new System.IO.IOException("the specified commit does not match the specified Directory"); // this can and will directly throw IOException if the specified commit point has been deleted return (DirectoryIndexReader)finder.DoBody(commit.GetSegmentsFileName()); } }
private void Init(Directory d, Analyzer a, bool create, bool closeDir, IndexDeletionPolicy deletionPolicy, bool autoCommit, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit) { this.closeDir = closeDir; directory = d; analyzer = a; SetMessageID(defaultInfoStream); this.maxFieldLength = maxFieldLength; if (indexingChain == null) indexingChain = DocumentsWriter.DefaultIndexingChain; if (create) { // Clear the write lock in case it's leftover: directory.ClearLock(WRITE_LOCK_NAME); } Lock writeLock = directory.MakeLock(WRITE_LOCK_NAME); if (!writeLock.Obtain(writeLockTimeout)) // obtain write lock { throw new LockObtainFailedException("Index locked for write: " + writeLock); } this.writeLock = writeLock; // save it bool success = false; try { if (create) { // Try to read first. This is to allow create // against an index that's currently open for // searching. In this case we write the next // segments_N file with no segments: bool doCommit; try { segmentInfos.Read(directory); segmentInfos.Clear(); doCommit = false; } catch (System.IO.IOException e) { // Likely this means it's a fresh directory doCommit = true; } if (autoCommit || doCommit) { // Always commit if autoCommit=true, else only // commit if there is no segments file in this dir // already. segmentInfos.Commit(directory); SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true)); } else { // Record that we have a change (zero out all // segments) pending: changeCount++; } } else { segmentInfos.Read(directory); if (commit != null) { // Swap out all segments, but, keep metadata in // SegmentInfos, like version & generation, to // preserve write-once. This is important if // readers are open against the future commit // points. if (commit.GetDirectory() != directory) throw new System.ArgumentException("IndexCommit's directory doesn't match my directory"); SegmentInfos oldInfos = new SegmentInfos(); oldInfos.Read(directory, commit.GetSegmentsFileName()); segmentInfos.Replace(oldInfos); changeCount++; if (infoStream != null) Message("init: loaded commit \"" + commit.GetSegmentsFileName() + "\""); } // We assume that this segments_N was previously // properly sync'd: SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true)); } this.autoCommit = autoCommit; SetRollbackSegmentInfos(segmentInfos); docWriter = new DocumentsWriter(directory, this, indexingChain); docWriter.SetInfoStream(infoStream); docWriter.SetMaxFieldLength(maxFieldLength); // Default deleter (for backwards compatibility) is // KeepOnlyLastCommitDeleter: deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter,synced); if (deleter.startingCommitDeleted) // Deletion policy deleted the "head" commit point. // We have to mark ourself as changed so that if we // are closed w/o any further changes we write a new // segments_N file. changeCount++; PushMaxBufferedDocs(); if (infoStream != null) { Message("init: create=" + create); MessageState(); } success = true; } finally { if (!success) { if (infoStream != null) { Message("init: hit exception on init; releasing write lock"); } try { writeLock.Release(); } catch (Exception t) { // don't mask the original exception } writeLock = null; } } }
private IndexReader DoReopen(bool openReadOnly, IndexCommit commit) { lock (this) { EnsureOpen(); System.Diagnostics.Debug.Assert(commit == null || openReadOnly); // If we were obtained by writer.getReader(), re-ask the // writer to get a new reader. if (writer != null) { System.Diagnostics.Debug.Assert(readOnly); if (!openReadOnly) { throw new System.ArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)"); } if (commit != null) { throw new System.ArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit"); } if (!writer.IsOpen(true)) { throw new AlreadyClosedException("cannot reopen: the IndexWriter this reader was obtained from is now closed"); } // TODO: right now we *always* make a new reader; in // the future we could have write make some effort to // detect that no changes have occurred IndexReader reader = writer.GetReader(); reader.SetDisableFakeNorms(GetDisableFakeNorms()); return reader; } if (commit == null) { if (hasChanges) { // We have changes, which means we are not readOnly: System.Diagnostics.Debug.Assert(readOnly == false); // and we hold the write lock: System.Diagnostics.Debug.Assert(writeLock != null); // so no other writer holds the write lock, which // means no changes could have been done to the index: System.Diagnostics.Debug.Assert(IsCurrent()); if (openReadOnly) { return (IndexReader) Clone(openReadOnly); } else { return this; } } else if (IsCurrent()) { if (openReadOnly != readOnly) { // Just fallback to clone return (IndexReader) Clone(openReadOnly); } else { return this; } } } else { if (directory != commit.GetDirectory()) throw new System.IO.IOException("the specified commit does not match the specified Directory"); if (segmentInfos != null && commit.GetSegmentsFileName().Equals(segmentInfos.GetCurrentSegmentFileName())) { if (readOnly != openReadOnly) { // Just fallback to clone return (IndexReader) Clone(openReadOnly); } else { return this; } } } return (IndexReader) new AnonymousClassFindSegmentsFile1(openReadOnly, this, directory).Run(commit); } }
private IndexReader DoReopenNoWriter(bool openReadOnly, IndexCommit commit) { lock (this) { if (commit == null) { if (hasChanges) { // We have changes, which means we are not readOnly: System.Diagnostics.Debug.Assert(readOnly == false); // and we hold the write lock: System.Diagnostics.Debug.Assert(writeLock != null); // so no other writer holds the write lock, which // means no changes could have been done to the index: System.Diagnostics.Debug.Assert(IsCurrent()); if (openReadOnly) { return (IndexReader)Clone(openReadOnly); } else { return this; } } else if (IsCurrent()) { if (openReadOnly != readOnly) { // Just fallback to clone return (IndexReader)Clone(openReadOnly); } else { return this; } } } else { if (directory != commit.GetDirectory()) throw new System.IO.IOException("the specified commit does not match the specified Directory"); if (segmentInfos != null && commit.GetSegmentsFileName().Equals(segmentInfos.GetCurrentSegmentFileName())) { if (readOnly != openReadOnly) { // Just fallback to clone return (IndexReader)Clone(openReadOnly); } else { return this; } } } return (IndexReader)new AnonymousFindSegmentsFile(directory, openReadOnly, this).Run(commit); } }
public virtual System.String GetSegmentsFileName() { return(cp.GetSegmentsFileName()); }