protected internal virtual void PrepareIndexAndSnapshots(SnapshotDeletionPolicy sdp, IndexWriter writer, int numSnapshots)
{
for (int i = 0; i < numSnapshots; i++)
{
// create dummy document to trigger commit.
writer.AddDocument(new Document());
writer.Commit();
Snapshots.Add(sdp.Snapshot());
}
}
public void Initialize()
{
if (System.IO.Directory.Exists(path) == false)
System.IO.Directory.CreateDirectory(path);
directory = FSDirectory.Open(new DirectoryInfo(path));
if (IndexWriter.IsLocked(directory))
IndexWriter.Unlock(directory);
analyzer = new LowerCaseKeywordAnalyzer();
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true));
}
/// <summary>
/// Example showing how to use the SnapshotDeletionPolicy to take a backup.
/// this method does not really do a backup; instead, it reads every byte of
/// every file just to test that the files indeed exist and are readable even
/// while the index is changing.
/// </summary>
public virtual void BackupIndex(Directory dir, SnapshotDeletionPolicy dp)
{
// To backup an index we first take a snapshot:
IndexCommit snapshot = dp.Snapshot();
try
{
CopyFiles(dir, snapshot);
}
finally
{
// Make sure to release the snapshot, otherwise these
// files will never be deleted during this IndexWriter
// session:
dp.Release(snapshot);
}
}
public virtual void TestReleaseSnapshot()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), DeletionPolicy));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
PrepareIndexAndSnapshots(sdp, writer, 1);
// Create another commit - we must do that, because otherwise the "snapshot"
// files will still remain in the index, since it's the last commit.
writer.AddDocument(new Document());
writer.Commit();
// Release
string segFileName = Snapshots[0].SegmentsFileName;
sdp.Release(Snapshots[0]);
writer.DeleteUnusedFiles();
writer.Dispose();
Assert.IsFalse(SlowFileExists(dir, segFileName), "segments file should not be found in dirctory: " + segFileName);
dir.Dispose();
}
public virtual void TestMultiThreadedSnapshotting()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), DeletionPolicy));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy)writer.Config.DelPolicy;
ThreadClass[] threads = new ThreadClass[10];
IndexCommit[] snapshots = new IndexCommit[threads.Length];
for (int i = 0; i < threads.Length; i++)
{
int finalI = i;
threads[i] = new ThreadAnonymousInnerClassHelper2(this, writer, sdp, snapshots, finalI);
threads[i].Name = "t" + i;
}
foreach (ThreadClass t in threads)
{
t.Start();
}
foreach (ThreadClass t in threads)
{
t.Join();
}
// Do one last commit, so that after we release all snapshots, we stay w/ one commit
writer.AddDocument(new Document());
writer.Commit();
for (int i = 0; i < threads.Length; i++)
{
sdp.Release(snapshots[i]);
writer.DeleteUnusedFiles();
}
Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count);
writer.Dispose();
dir.Dispose();
}
public virtual void TestMissingCommits()
{
// Tests the behavior of SDP when commits that are given at ctor are missing
// on onInit().
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), DeletionPolicy));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy)writer.Config.DelPolicy;
writer.AddDocument(new Document());
writer.Commit();
IndexCommit s1 = sdp.Snapshot();
// create another commit, not snapshotted.
writer.AddDocument(new Document());
writer.Dispose();
// open a new writer w/ KeepOnlyLastCommit policy, so it will delete "s1"
// commit.
(new IndexWriter(dir, GetConfig(Random(), null))).Dispose();
Assert.IsFalse(SlowFileExists(dir, s1.SegmentsFileName), "snapshotted commit should not exist");
dir.Dispose();
}
/// <summary>
/// Creates a <see cref="SnapshotCommitPoint"/> wrapping the provided
/// <see cref="IndexCommit"/>.
/// </summary>
protected internal SnapshotCommitPoint(SnapshotDeletionPolicy outerInstance, IndexCommit cp)
{
this.outerInstance = outerInstance;
this.m_cp = cp;
}
private void CreateIndexWriter()
{
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter.IndexReaderWarmer indexReaderWarmer = context.IndexReaderWarmers != null
? new IndexReaderWarmersWrapper(name, context.IndexReaderWarmers)
: null;
indexWriter = new RavenIndexWriter(directory, stopAnalyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED, context.Configuration.MaxIndexWritesBeforeRecreate, indexReaderWarmer);
}
private void OpenIndexOnStartup()
{
analyzer = new LowerCaseKeywordAnalyzer();
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
bool resetTried = false;
bool recoveryTried = false;
while (true)
{
LuceneDirectory luceneDirectory = null;
try
{
luceneDirectory = OpenOrCreateLuceneDirectory(indexDirectory);
// Skip sanity test if we are running in memory. Index will not exist anyways.
if (!configuration.RunInMemory && !IsIndexStateValid(luceneDirectory))
throw new InvalidOperationException("Sanity check on the index failed.");
directory = luceneDirectory;
writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true));
break;
}
catch (Exception e)
{
if (resetTried)
throw new InvalidOperationException("Could not open / create index for file system '" + name + "', reset already tried", e);
if (recoveryTried == false && luceneDirectory != null)
{
recoveryTried = true;
StartupLog.WarnException("Could not open index for file system '" + name + "'. Trying to recover index", e);
StartupLog.Info("Recover functionality is still not implemented. Skipping.");
}
else
{
resetTried = true;
StartupLog.WarnException("Could not open index for file system '" + name + "'. Recovery operation failed, forcibly resetting index", e);
TryResettingIndex();
}
}
}
}
internal MyCommitPoint(SnapshotDeletionPolicy enclosingInstance, IndexCommit cp)
{
InitBlock(enclosingInstance);
this.cp = cp;
}
private void InitBlock(SnapshotDeletionPolicy enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
}
public LocalTempStorageIndexer()
{
IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
Snapshotter = new SnapshotDeletionPolicy(policy);
}
public ThreadAnonymousInnerClassHelper2(TestSnapshotDeletionPolicy outerInstance, IndexWriter writer, SnapshotDeletionPolicy sdp, IndexCommit[] snapshots, int finalI)
{
this.OuterInstance = outerInstance;
this.Writer = writer;
this.Sdp = sdp;
this.Snapshots = snapshots;
this.FinalI = finalI;
}
protected internal virtual void AssertSnapshotExists(Directory dir, SnapshotDeletionPolicy sdp, int numSnapshots, bool checkIndexCommitSame)
{
for (int i = 0; i < numSnapshots; i++)
{
IndexCommit snapshot = Snapshots[i];
CheckMaxDoc(snapshot, i + 1);
CheckSnapshotExists(dir, snapshot);
if (checkIndexCommitSame)
{
Assert.AreSame(snapshot, sdp.GetIndexCommit(snapshot.Generation));
}
else
{
Assert.AreEqual(snapshot.Generation, sdp.GetIndexCommit(snapshot.Generation).Generation);
}
}
}
private void CreateIndexWriter()
{
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
indexWriter = new RavenIndexWriter(directory, stopAnalyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED, context.Configuration.MaxIndexWritesBeforeRecreate);
}
public ThreadAnonymousInnerClassHelper2(TestSnapshotDeletionPolicy outerInstance, IndexWriter writer, SnapshotDeletionPolicy sdp, IndexCommit[] snapshots, int finalI)
{
this.OuterInstance = outerInstance;
this.Writer = writer;
this.Sdp = sdp;
this.Snapshots = snapshots;
this.FinalI = finalI;
}
private void CreateIndexWriter()
{
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
indexWriter = new IndexWriter(directory, stopAnalyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
using (indexWriter.MergeScheduler) { }
indexWriter.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
// RavenDB already manages the memory for those, no need for Lucene to do this as well
indexWriter.SetMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH);
indexWriter.SetRAMBufferSizeMB(1024);
}
public LuceneIndexCopy()
{
IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
_snapshotter = new SnapshotDeletionPolicy(policy);
}
private void InitBlock(SnapshotDeletionPolicy enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
}
internal MyCommitPoint(SnapshotDeletionPolicy enclosingInstance, IndexCommit cp)
{
InitBlock(enclosingInstance);
this.cp = cp;
}
public ThreadAnonymousClass2(TestSnapshotDeletionPolicy outerInstance, IndexWriter writer, SnapshotDeletionPolicy sdp, IndexCommit[] snapshots, int finalI)
{
this.outerInstance = outerInstance;
this.writer = writer;
this.sdp = sdp;
this.snapshots = snapshots;
this.finalI = finalI;
}
