protected internal virtual void PrepareIndexAndSnapshots(SnapshotDeletionPolicy sdp, IndexWriter writer, int numSnapshots) { for (int i = 0; i < numSnapshots; i++) { // create dummy document to trigger commit. writer.AddDocument(new Document()); writer.Commit(); Snapshots.Add(sdp.Snapshot()); } }
public void Initialize() { if (System.IO.Directory.Exists(path) == false) System.IO.Directory.CreateDirectory(path); directory = FSDirectory.Open(new DirectoryInfo(path)); if (IndexWriter.IsLocked(directory)) IndexWriter.Unlock(directory); analyzer = new LowerCaseKeywordAnalyzer(); snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED); writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler()); currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true)); }
/// <summary> /// Example showing how to use the SnapshotDeletionPolicy to take a backup. /// this method does not really do a backup; instead, it reads every byte of /// every file just to test that the files indeed exist and are readable even /// while the index is changing. /// </summary> public virtual void BackupIndex(Directory dir, SnapshotDeletionPolicy dp) { // To backup an index we first take a snapshot: IndexCommit snapshot = dp.Snapshot(); try { CopyFiles(dir, snapshot); } finally { // Make sure to release the snapshot, otherwise these // files will never be deleted during this IndexWriter // session: dp.Release(snapshot); } }
public virtual void TestReleaseSnapshot() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), DeletionPolicy)); SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy; PrepareIndexAndSnapshots(sdp, writer, 1); // Create another commit - we must do that, because otherwise the "snapshot" // files will still remain in the index, since it's the last commit. writer.AddDocument(new Document()); writer.Commit(); // Release string segFileName = Snapshots[0].SegmentsFileName; sdp.Release(Snapshots[0]); writer.DeleteUnusedFiles(); writer.Dispose(); Assert.IsFalse(SlowFileExists(dir, segFileName), "segments file should not be found in dirctory: " + segFileName); dir.Dispose(); }
public virtual void TestMultiThreadedSnapshotting() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), DeletionPolicy)); SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy)writer.Config.DelPolicy; ThreadClass[] threads = new ThreadClass[10]; IndexCommit[] snapshots = new IndexCommit[threads.Length]; for (int i = 0; i < threads.Length; i++) { int finalI = i; threads[i] = new ThreadAnonymousInnerClassHelper2(this, writer, sdp, snapshots, finalI); threads[i].Name = "t" + i; } foreach (ThreadClass t in threads) { t.Start(); } foreach (ThreadClass t in threads) { t.Join(); } // Do one last commit, so that after we release all snapshots, we stay w/ one commit writer.AddDocument(new Document()); writer.Commit(); for (int i = 0; i < threads.Length; i++) { sdp.Release(snapshots[i]); writer.DeleteUnusedFiles(); } Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count); writer.Dispose(); dir.Dispose(); }
public virtual void TestMissingCommits() { // Tests the behavior of SDP when commits that are given at ctor are missing // on onInit(). Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), DeletionPolicy)); SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy)writer.Config.DelPolicy; writer.AddDocument(new Document()); writer.Commit(); IndexCommit s1 = sdp.Snapshot(); // create another commit, not snapshotted. writer.AddDocument(new Document()); writer.Dispose(); // open a new writer w/ KeepOnlyLastCommit policy, so it will delete "s1" // commit. (new IndexWriter(dir, GetConfig(Random(), null))).Dispose(); Assert.IsFalse(SlowFileExists(dir, s1.SegmentsFileName), "snapshotted commit should not exist"); dir.Dispose(); }
/// <summary> /// Creates a <see cref="SnapshotCommitPoint"/> wrapping the provided /// <see cref="IndexCommit"/>. /// </summary> protected internal SnapshotCommitPoint(SnapshotDeletionPolicy outerInstance, IndexCommit cp) { this.outerInstance = outerInstance; this.m_cp = cp; }
private void CreateIndexWriter() { snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); IndexWriter.IndexReaderWarmer indexReaderWarmer = context.IndexReaderWarmers != null ? new IndexReaderWarmersWrapper(name, context.IndexReaderWarmers) : null; indexWriter = new RavenIndexWriter(directory, stopAnalyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED, context.Configuration.MaxIndexWritesBeforeRecreate, indexReaderWarmer); }
private void OpenIndexOnStartup() { analyzer = new LowerCaseKeywordAnalyzer(); snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); bool resetTried = false; bool recoveryTried = false; while (true) { LuceneDirectory luceneDirectory = null; try { luceneDirectory = OpenOrCreateLuceneDirectory(indexDirectory); // Skip sanity test if we are running in memory. Index will not exist anyways. if (!configuration.RunInMemory && !IsIndexStateValid(luceneDirectory)) throw new InvalidOperationException("Sanity check on the index failed."); directory = luceneDirectory; writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED); writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler()); currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true)); break; } catch (Exception e) { if (resetTried) throw new InvalidOperationException("Could not open / create index for file system '" + name + "', reset already tried", e); if (recoveryTried == false && luceneDirectory != null) { recoveryTried = true; StartupLog.WarnException("Could not open index for file system '" + name + "'. Trying to recover index", e); StartupLog.Info("Recover functionality is still not implemented. Skipping."); } else { resetTried = true; StartupLog.WarnException("Could not open index for file system '" + name + "'. Recovery operation failed, forcibly resetting index", e); TryResettingIndex(); } } } }
internal MyCommitPoint(SnapshotDeletionPolicy enclosingInstance, IndexCommit cp) { InitBlock(enclosingInstance); this.cp = cp; }
private void InitBlock(SnapshotDeletionPolicy enclosingInstance) { this.enclosingInstance = enclosingInstance; }
public LocalTempStorageIndexer() { IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy(); Snapshotter = new SnapshotDeletionPolicy(policy); }
public ThreadAnonymousInnerClassHelper2(TestSnapshotDeletionPolicy outerInstance, IndexWriter writer, SnapshotDeletionPolicy sdp, IndexCommit[] snapshots, int finalI) { this.OuterInstance = outerInstance; this.Writer = writer; this.Sdp = sdp; this.Snapshots = snapshots; this.FinalI = finalI; }
protected internal virtual void AssertSnapshotExists(Directory dir, SnapshotDeletionPolicy sdp, int numSnapshots, bool checkIndexCommitSame) { for (int i = 0; i < numSnapshots; i++) { IndexCommit snapshot = Snapshots[i]; CheckMaxDoc(snapshot, i + 1); CheckSnapshotExists(dir, snapshot); if (checkIndexCommitSame) { Assert.AreSame(snapshot, sdp.GetIndexCommit(snapshot.Generation)); } else { Assert.AreEqual(snapshot.Generation, sdp.GetIndexCommit(snapshot.Generation).Generation); } } }
private void CreateIndexWriter() { snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); indexWriter = new RavenIndexWriter(directory, stopAnalyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED, context.Configuration.MaxIndexWritesBeforeRecreate); }
private void CreateIndexWriter() { snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); indexWriter = new IndexWriter(directory, stopAnalyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED); using (indexWriter.MergeScheduler) { } indexWriter.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler()); // RavenDB already manages the memory for those, no need for Lucene to do this as well indexWriter.SetMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH); indexWriter.SetRAMBufferSizeMB(1024); }
public LuceneIndexCopy() { IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy(); _snapshotter = new SnapshotDeletionPolicy(policy); }
public ThreadAnonymousClass2(TestSnapshotDeletionPolicy outerInstance, IndexWriter writer, SnapshotDeletionPolicy sdp, IndexCommit[] snapshots, int finalI) { this.outerInstance = outerInstance; this.writer = writer; this.sdp = sdp; this.snapshots = snapshots; this.finalI = finalI; }