public void TestNoCommits()
{
// Tests that if there were no commits when snapshot() is called, then
// IllegalStateException is thrown rather than NPE.
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Assert.Throws <SystemException>(() => sdp.Snapshot(), "expected exception not hit");
}
/// <summary>Example showing how to use the SnapshotDeletionPolicy
/// to take a backup. This method does not really do a
/// backup; instead, it reads every byte of every file
/// just to test that the files indeed exist and are
/// readable even while the index is changing.
/// </summary>
public virtual void BackupIndex(Directory dir, SnapshotDeletionPolicy dp)
{
// To backup an index we first take a snapshot:
try
{
CopyFiles(dir, (IndexCommit)dp.Snapshot());
}
finally
{
// Make sure to release the snapshot, otherwise these
// files will never be deleted during this IndexWriter
// session:
dp.Release();
}
}
public void TestNoCommits()
{
// Tests that if there were no commits when snapshot() is called, then
// IllegalStateException is thrown rather than NPE.
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
try
{
sdp.Snapshot();
Assert.Fail("expected exception not hit");
}
catch (System.SystemException ise)
{
// expected
}
}
public virtual void TestReuseAcrossWriters()
{
Directory dir = new MockRAMDirectory();
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED, null);
// Force frequent flushes
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 7; i++)
{
writer.AddDocument(doc, null);
if (i % 2 == 0)
{
writer.Commit(null);
}
}
IndexCommit cp = dp.Snapshot();
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED, null);
CopyFiles(dir, cp);
for (int i = 0; i < 7; i++)
{
writer.AddDocument(doc, null);
if (i % 2 == 0)
{
writer.Commit(null);
}
}
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
dp.Release();
writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED, null);
writer.Close();
Assert.Throws <System.IO.FileNotFoundException>(() => CopyFiles(dir, cp), "did not hit expected IOException");
dir.Close();
}
public virtual void TestReuseAcrossWriters()
{
Directory dir = new MockRAMDirectory();
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
// Force frequent commits
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 7; i++)
{
writer.AddDocument(doc);
}
IndexCommit cp = (IndexCommit)dp.Snapshot();
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
CopyFiles(dir, cp);
for (int i = 0; i < 7; i++)
{
writer.AddDocument(doc);
}
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
dp.Release();
writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
writer.Close();
try
{
CopyFiles(dir, cp);
Assert.Fail("did not hit expected IOException");
}
catch (System.IO.IOException ioe)
{
// expected
}
dir.Close();
}
private void RunTest(Directory dir)
{
// Run for ~7 seconds
long stopTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) + 7000;
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
// Force frequent commits
writer.SetMaxBufferedDocs(2);
SupportClass.ThreadClass t = new AnonymousClassThread(stopTime, writer, this);
t.Start();
// While the above indexing thread is running, take many
// backups:
while ((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) < stopTime)
{
BackupIndex(dir, dp);
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 20));
if (!t.IsAlive)
{
break;
}
}
t.Join();
// Add one more document to force writer to commit a
// final segment, so deletion policy has a chance to
// delete again:
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
writer.AddDocument(doc);
// Make sure we don't have any leftover files in the
// directory:
writer.Close();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "some files were not deleted but should have been");
}
/// <summary>Example showing how to use the SnapshotDeletionPolicy
/// to take a backup. This method does not really do a
/// backup; instead, it reads every byte of every file
/// just to test that the files indeed exist and are
/// readable even while the index is changing.
/// </summary>
public virtual void BackupIndex(Directory dir, SnapshotDeletionPolicy dp)
{
// To backup an index we first take a snapshot:
try
{
CopyFiles(dir, (IndexCommit) dp.Snapshot());
}
finally
{
// Make sure to release the snapshot, otherwise these
// files will never be deleted during this IndexWriter
// session:
dp.Release();
}
}
private void RunTest(Directory dir)
{
// Run for ~7 seconds
long stopTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) + 7000;
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
// Force frequent commits
writer.SetMaxBufferedDocs(2);
SupportClass.ThreadClass t = new AnonymousClassThread(stopTime, writer, this);
t.Start();
// While the above indexing thread is running, take many
// backups:
while ((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) < stopTime)
{
BackupIndex(dir, dp);
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 20));
if (!t.IsAlive)
break;
}
t.Join();
// Add one more document to force writer to commit a
// final segment, so deletion policy has a chance to
// delete again:
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
writer.AddDocument(doc);
// Make sure we don't have any leftover files in the
// directory:
writer.Close();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "some files were not deleted but should have been");
}
public virtual void TestReuseAcrossWriters()
{
Directory dir = new MockRAMDirectory();
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
// Force frequent commits
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 7; i++)
writer.AddDocument(doc);
IndexCommit cp = (IndexCommit) dp.Snapshot();
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
CopyFiles(dir, cp);
for (int i = 0; i < 7; i++)
writer.AddDocument(doc);
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
dp.Release();
writer = new IndexWriter(dir, true, new StandardAnalyzer(), dp);
writer.Close();
try
{
CopyFiles(dir, cp);
Assert.Fail("did not hit expected IOException");
}
catch (System.IO.IOException ioe)
{
// expected
}
dir.Close();
}
public void TestNoCommits()
{
// Tests that if there were no commits when snapshot() is called, then
// IllegalStateException is thrown rather than NPE.
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Assert.Throws<SystemException>(() => sdp.Snapshot(), "expected exception not hit");
}
public virtual void TestReuseAcrossWriters()
{
Directory dir = new MockRAMDirectory();
SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
// Force frequent flushes
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 7; i++)
{
writer.AddDocument(doc);
if (i % 2 == 0)
{
writer.Commit();
}
}
IndexCommit cp = dp.Snapshot();
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
CopyFiles(dir, cp);
for (int i = 0; i < 7; i++)
{
writer.AddDocument(doc);
if (i % 2 == 0)
{
writer.Commit();
}
}
CopyFiles(dir, cp);
writer.Close();
CopyFiles(dir, cp);
dp.Release();
writer = new IndexWriter(dir, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
writer.Close();
Assert.Throws<System.IO.FileNotFoundException>(() => CopyFiles(dir, cp), "did not hit expected IOException");
dir.Close();
}
/// <summary>Example showing how to use the SnapshotDeletionPolicy
/// to take a backup. This method does not really do a
/// backup; instead, it reads every byte of every file
/// just to test that the files indeed exist and are
/// readable even while the index is changing.
/// </summary>
public virtual void BackupIndex(Directory dir, SnapshotDeletionPolicy dp)
{
// To backup an index we first take a snapshot:
IndexCommitPoint cp = dp.Snapshot();
try
{
// While we hold the snapshot, and nomatter how long
// we take to do the backup, the IndexWriter will
// never delete the files in the snapshot:
System.Collections.ICollection files = cp.GetFileNames();
System.Collections.IEnumerator it = files.GetEnumerator();
while (it.MoveNext())
{
System.String fileName = (System.String) it.Current;
// NOTE: in a real backup you would not use
// readFile; you would need to use something else
// that copies the file to a backup location. This
// could even be a spawned shell process (eg "tar",
// "zip") that takes the list of files and builds a
// backup.
ReadFile(dir, fileName);
}
}
finally
{
// Make sure to release the snapshot, otherwise these
// files will never be deleted during this IndexWriter
// session:
dp.Release();
}
}
public RunnableAnonymousInnerClassHelper(TestControlledRealTimeReopenThread outerInstance, SnapshotDeletionPolicy sdp, Directory dir, IndexWriter iw)
{
this.OuterInstance = outerInstance;
this.Sdp = sdp;
this.Dir = dir;
this.Iw = iw;
}
// LUCENE-5461
public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random().Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random()));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread<IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.SetDaemon(true);
controlledRealTimeReopenThread.Start();
IList<Thread> commitThreads = new List<Thread>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw));
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = DateTime.Now.Millisecond;
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = DateTime.Now.Millisecond - start;
Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait);
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
Assert.AreEqual(1, td.TotalHits);
}
foreach (Thread commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public void TestNoCommits()
{
// Tests that if there were no commits when snapshot() is called, then
// IllegalStateException is thrown rather than NPE.
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
try
{
sdp.Snapshot();
Assert.Fail("expected exception not hit");
}
catch (System.SystemException ise)
{
// expected
}
}
