| public static FileNameFromGeneration ( string @base, string ext, long gen ) : string | ||
| @base | string | |
| ext | string | extension of the filename |
| gen | long | generation |
| return | string | |
internal string GetDelFileName()
{
if (delGen == NO)
{
// In this case we know there is no deletion filename
// against this segment
return(null);
}
else
{
string retVal = null;
string current = IndexFileNames.FileNameFromGeneration(name, "." + IndexFileNames.DELETES_EXTENSION, delGen);
if (this.dir.FileExists(current))
{
retVal = current;
}
else
{
string backwards = (name + "_" + System.Convert.ToString(delGen, 16) + "." + IndexFileNames.DELETES_EXTENSION);
if (this.dir.FileExists(backwards))
{
// we are dealing with the old name
retVal = backwards;
}
else
{
// no file, creating one, so use the new name
retVal = current;
}
}
return(retVal);
}
}
internal void RollbackCommit(Directory dir)
{
if (pendingSegnOutput != null)
{
try
{
pendingSegnOutput.Close();
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
// in our caller
}
// Must carefully compute fileName from "generation"
// since lastGeneration isn't incremented:
try
{
System.String segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
dir.DeleteFile(segmentFileName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
// in our caller
}
pendingSegnOutput = null;
}
}
/// <summary> Get the file name for the norms file for this field.
///
/// </summary>
/// <param name="number">field index
/// </param>
public System.String GetNormFileName(int number)
{
System.String prefix;
long gen;
if (normGen == null)
{
gen = CHECK_DIR;
}
else
{
gen = normGen[number];
}
if (HasSeparateNorms(number))
{
// case 1: separate norm
prefix = ".s";
return(IndexFileNames.FileNameFromGeneration(name, prefix + number, gen));
}
if (hasSingleNormFile)
{
// case 2: lockless (or nrm file exists) - single file for all norms
prefix = "." + IndexFileNames.NORMS_EXTENSION;
return(IndexFileNames.FileNameFromGeneration(name, prefix, WITHOUT_GEN));
}
// case 3: norm file for each field
prefix = ".f";
return(IndexFileNames.FileNameFromGeneration(name, prefix + number, WITHOUT_GEN));
}
internal System.String GetDelFileName()
{
if (delGen == -1)
{
// In this case we know there is no deletion filename
// against this segment
return(null);
}
else
{
// If delGen is 0, it's the pre-lockless-commit file format
return(IndexFileNames.FileNameFromGeneration(name, ".del", delGen));
}
}
public System.String GetDelFileName()
{
if (delGen == NO)
{
// In this case we know there is no deletion filename
// against this segment
return(null);
}
else
{
// If delGen is CHECK_DIR, it's the pre-lockless-commit file format
return(IndexFileNames.FileNameFromGeneration(name, "." + IndexFileNames.DELETES_EXTENSION, delGen));
}
}
/// <summary> Get the next segments_N filename that will be written.</summary>
public System.String GetNextSegmentFileName()
{
long nextGeneration;
if (generation == -1)
{
nextGeneration = 1;
}
else
{
nextGeneration = generation + 1;
}
return(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", nextGeneration));
}
public virtual void TestKeepAllDeletionPolicy()
{
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
// Never deletes a commit
KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
Directory dir = new RAMDirectory();
policy.dir = dir;
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
writer.SetMergeScheduler(new SerialMergeScheduler());
for (int i = 0; i < 107; i++)
{
AddDoc(writer);
if (autoCommit && i % 10 == 0)
{
writer.Commit();
}
}
writer.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetUseCompoundFile(useCompoundFile);
writer.Optimize();
writer.Close();
Assert.AreEqual(2, policy.numOnInit);
if (!autoCommit)
{
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
Assert.AreEqual(2, policy.numOnCommit);
}
// Test listCommits
System.Collections.ICollection commits = IndexReader.ListCommits(dir);
if (!autoCommit)
{
// 1 from opening writer + 2 from closing writer
Assert.AreEqual(3, commits.Count);
}
// 1 from opening writer + 2 from closing writer +
// 11 from calling writer.commit() explicitly above
else
{
Assert.AreEqual(14, commits.Count);
}
System.Collections.IEnumerator it = commits.GetEnumerator();
// Make sure we can open a reader on each commit:
while (it.MoveNext())
{
IndexCommit commit = (IndexCommit)it.Current;
IndexReader r = IndexReader.Open(commit, null);
r.Close();
}
// Simplistic check: just verify all segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
while (gen > 0)
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0)
{
// Now that we've removed a commit point, which
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.ListAll().Length;
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.LIMITED);
writer.Close();
int postCount = dir.ListAll().Length;
Assert.IsTrue(postCount < preCount);
}
}
dir.Close();
}
}
public virtual void TestKeepLastNDeletionPolicyWithCreates()
{
const int N = 10;
for (int pass = 0; pass < 2; pass++)
{
bool useCompoundFile = (pass % 2) != 0;
Directory dir = NewDirectory();
IndexWriterConfig conf = (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetIndexDeletionPolicy(new KeepLastNDeletionPolicy(this, N)).SetMaxBufferedDocs(10);
MergePolicy mp = conf.MergePolicy;
mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0;
IndexWriter writer = new IndexWriter(dir, conf);
KeepLastNDeletionPolicy policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy;
writer.Dispose();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for (int i = 0; i < N + 1; i++)
{
conf = (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetIndexDeletionPolicy(policy).SetMaxBufferedDocs(10);
mp = conf.MergePolicy;
mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0;
writer = new IndexWriter(dir, conf);
policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy;
for (int j = 0; j < 17; j++)
{
AddDocWithID(writer, i * (N + 1) + j);
}
// this is a commit
writer.Dispose();
conf = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))).SetIndexDeletionPolicy(policy).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
writer = new IndexWriter(dir, conf);
policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy;
writer.DeleteDocuments(new Term("id", "" + (i * (N + 1) + 3)));
// this is a commit
writer.Dispose();
IndexReader reader = DirectoryReader.Open(dir);
IndexSearcher searcher = NewSearcher(reader);
ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs;
Assert.AreEqual(16, hits.Length);
reader.Dispose();
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetIndexDeletionPolicy(policy));
policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy;
// this will not commit: there are no changes
// pending because we opened for "create":
writer.Dispose();
}
Assert.AreEqual(3 * (N + 1) + 1, policy.NumOnInit);
Assert.AreEqual(3 * (N + 1) + 1, policy.NumOnCommit);
IndexReader rwReader = DirectoryReader.Open(dir);
IndexSearcher searcher_ = NewSearcher(rwReader);
ScoreDoc[] hits_ = searcher_.Search(query, null, 1000).ScoreDocs;
Assert.AreEqual(0, hits_.Length);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.GetLastCommitGeneration(dir);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
int expectedCount = 0;
rwReader.Dispose();
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = DirectoryReader.Open(dir);
// Work backwards in commits on what the expected
// count should be.
searcher_ = NewSearcher(reader);
hits_ = searcher_.Search(query, null, 1000).ScoreDocs;
Assert.AreEqual(expectedCount, hits_.Length);
if (expectedCount == 0)
{
expectedCount = 16;
}
else if (expectedCount == 16)
{
expectedCount = 17;
}
else if (expectedCount == 17)
{
expectedCount = 0;
}
reader.Dispose();
if (i == N)
{
Assert.Fail("should have failed on commits before last " + N);
}
}
catch (IOException /*e*/)
{
if (i != N)
{
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Dispose();
}
}
public virtual void TestKeepLastNDeletionPolicy()
{
const int N = 5;
for (int pass = 0; pass < 2; pass++)
{
bool useCompoundFile = (pass % 2) != 0;
Directory dir = NewDirectory();
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
for (int j = 0; j < N + 1; j++)
{
IndexWriterConfig conf = (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetIndexDeletionPolicy(policy).SetMaxBufferedDocs(10);
MergePolicy mp = conf.MergePolicy;
mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0;
IndexWriter writer = new IndexWriter(dir, conf);
policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy;
for (int i = 0; i < 17; i++)
{
AddDoc(writer);
}
writer.ForceMerge(1);
writer.Dispose();
}
Assert.IsTrue(policy.NumDelete > 0);
Assert.AreEqual(N + 1, policy.NumOnInit);
Assert.AreEqual(N + 1, policy.NumOnCommit);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetLastCommitGeneration(dir);
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = DirectoryReader.Open(dir);
reader.Dispose();
if (i == N)
{
Assert.Fail("should have failed on commits prior to last " + N);
}
}
catch (IOException /*e*/)
{
if (i != N)
{
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Dispose();
}
}
public virtual void TestKeepAllDeletionPolicy()
{
for (int pass = 0; pass < 2; pass++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: cycle pass="******"TEST: open writer for forceMerge");
}
writer = new IndexWriter(dir, conf);
policy = (KeepAllDeletionPolicy)writer.Config.IndexDeletionPolicy;
writer.ForceMerge(1);
writer.Dispose();
}
Assert.AreEqual(needsMerging ? 2 : 1, policy.NumOnInit);
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
Assert.AreEqual(1 + (needsMerging ? 1 : 0), policy.NumOnCommit);
// Test listCommits
ICollection <IndexCommit> commits = DirectoryReader.ListCommits(dir);
// 2 from closing writer
Assert.AreEqual(1 + (needsMerging ? 1 : 0), commits.Count);
// Make sure we can open a reader on each commit:
foreach (IndexCommit commit in commits)
{
IndexReader r = DirectoryReader.Open(commit);
r.Dispose();
}
// Simplistic check: just verify all segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetLastCommitGeneration(dir);
while (gen > 0)
{
IndexReader reader = DirectoryReader.Open(dir);
reader.Dispose();
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0)
{
// Now that we've removed a commit point, which
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.ListAll().Length;
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetIndexDeletionPolicy(policy));
writer.Dispose();
int postCount = dir.ListAll().Length;
Assert.IsTrue(postCount < preCount);
}
}
dir.Dispose();
}
}
/// <summary>
/// Get the segments_n file for the generation found in the
/// segments.gen file.
/// </summary>
/// <returns>The segments_n file to use.</returns>
public override string GetSegmentsFileName()
{
return(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, string.Empty, this.generation));
}
/// <summary> Get the filename of the current segments_N file
/// in the directory.
///
/// </summary>
/// <param name="directory">-- directory to search for the latest segments_N file
/// </param>
public static System.String GetCurrentSegmentFileName(Directory directory)
{
return(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", GetCurrentSegmentGeneration(directory)));
}
/*
* Return all files referenced by this SegmentInfo. The
* returns List is a locally cached List so you should not
* modify it.
*/
public IList <string> Files()
{
if (files != null)
{
// Already cached:
return(files);
}
var fileList = new System.Collections.Generic.List <string>();
bool useCompoundFile = GetUseCompoundFile();
if (useCompoundFile)
{
fileList.Add(name + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
}
else
{
System.String[] exts = IndexFileNames.NON_STORE_INDEX_EXTENSIONS;
for (int i = 0; i < exts.Length; i++)
{
AddIfExists(fileList, name + "." + exts[i]);
}
}
if (docStoreOffset != -1)
{
// We are sharing doc stores (stored fields, term
// vectors) with other segments
System.Diagnostics.Debug.Assert(docStoreSegment != null);
if (docStoreIsCompoundFile)
{
fileList.Add(docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION);
}
else
{
System.String[] exts = IndexFileNames.STORE_INDEX_EXTENSIONS;
for (int i = 0; i < exts.Length; i++)
{
AddIfExists(fileList, docStoreSegment + "." + exts[i]);
}
}
}
else if (!useCompoundFile)
{
// We are not sharing, and, these files were not
// included in the compound file
System.String[] exts = IndexFileNames.STORE_INDEX_EXTENSIONS;
for (int i = 0; i < exts.Length; i++)
{
AddIfExists(fileList, name + "." + exts[i]);
}
}
System.String delFileName = IndexFileNames.FileNameFromGeneration(name, "." + IndexFileNames.DELETES_EXTENSION, delGen);
if (delFileName != null && (delGen >= YES || dir.FileExists(delFileName)))
{
fileList.Add(delFileName);
}
// Careful logic for norms files
if (normGen != null)
{
for (int i = 0; i < normGen.Length; i++)
{
long gen = normGen[i];
if (gen >= YES)
{
// Definitely a separate norm file, with generation:
fileList.Add(IndexFileNames.FileNameFromGeneration(name, "." + IndexFileNames.SEPARATE_NORMS_EXTENSION + i, gen));
}
else if (NO == gen)
{
// No separate norms but maybe plain norms
// in the non compound file case:
if (!hasSingleNormFile && !useCompoundFile)
{
System.String fileName = name + "." + IndexFileNames.PLAIN_NORMS_EXTENSION + i;
if (dir.FileExists(fileName))
{
fileList.Add(fileName);
}
}
}
else if (CHECK_DIR == gen)
{
// Pre-2.1: we have to check file existence
System.String fileName = null;
if (useCompoundFile)
{
fileName = name + "." + IndexFileNames.SEPARATE_NORMS_EXTENSION + i;
}
else if (!hasSingleNormFile)
{
fileName = name + "." + IndexFileNames.PLAIN_NORMS_EXTENSION + i;
}
if (fileName != null && dir.FileExists(fileName))
{
fileList.Add(fileName);
}
}
}
}
else if (preLockless || (!hasSingleNormFile && !useCompoundFile))
{
// Pre-2.1: we have to scan the dir to find all
// matching _X.sN/_X.fN files for our segment:
System.String prefix;
if (useCompoundFile)
{
prefix = name + "." + IndexFileNames.SEPARATE_NORMS_EXTENSION;
}
else
{
prefix = name + "." + IndexFileNames.PLAIN_NORMS_EXTENSION;
}
int prefixLength = prefix.Length;
System.String[] allFiles = dir.ListAll();
IndexFileNameFilter filter = IndexFileNameFilter.Filter;
for (int i = 0; i < allFiles.Length; i++)
{
System.String fileName = allFiles[i];
if (filter.Accept(null, fileName) && fileName.Length > prefixLength && System.Char.IsDigit(fileName[prefixLength]) && fileName.StartsWith(prefix))
{
fileList.Add(fileName);
}
}
}
//System.Diagnostics.Debug.Assert();
files = fileList;
return(files);
}
public virtual void TestExpirationTimeDeletionPolicy()
{
double SECONDS = 2.0;
bool autoCommit = false;
bool useCompoundFile = true;
Directory dir = new RAMDirectory();
ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(this, dir, SECONDS);
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetUseCompoundFile(useCompoundFile);
writer.Close();
long lastDeleteTime = 0;
for (int i = 0; i < 7; i++)
{
// Record last time when writer performed deletes of
// past commits
lastDeleteTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetUseCompoundFile(useCompoundFile);
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
writer.Close();
// Make sure to sleep long enough so that some commit
// points will be deleted:
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * (int)(1000.0 * (SECONDS / 5.0))));
}
// First, make sure the policy in fact deleted something:
Assert.IsTrue(policy.numDelete > 0, "no commits were deleted");
// Then simplistic check: just verify that the
// segments_N's that still exist are in fact within SECONDS
// seconds of the last one's mod time, and, that I can
// open a reader on each:
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
while (gen > 0)
{
try
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
long modTime = dir.FileModified(fileName);
Assert.IsTrue(lastDeleteTime - modTime <= (SECONDS * 1000), "commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted");
}
catch (System.IO.IOException e)
{
// OK
break;
}
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
}
dir.Close();
}
internal void FinishCommit(Directory dir)
{
if (pendingSegnOutput == null)
{
throw new System.SystemException("prepareCommit was not called");
}
bool success = false;
try
{
pendingSegnOutput.FinishCommit();
pendingSegnOutput.Close();
pendingSegnOutput = null;
success = true;
}
finally
{
if (!success)
{
RollbackCommit(dir);
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception t)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
public virtual void TestKeepLastNDeletionPolicyWithReader()
{
int N = 10;
for (int pass = 0; pass < 2; pass++)
{
bool useCompoundFile = (pass % 2) != 0;
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
writer.UseCompoundFile = useCompoundFile;
writer.Close();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for (int i = 0; i < N + 1; i++)
{
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
writer.UseCompoundFile = useCompoundFile;
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
// this is a commit
writer.Close();
IndexReader reader = IndexReader.Open(dir, policy, false);
reader.DeleteDocument(3 * i + 1);
reader.SetNorm(4 * i + 1, "content", 2.0F);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs;
Assert.AreEqual(16 * (1 + i), hits.Length);
// this is a commit
reader.Close();
searcher.Close();
}
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
writer.UseCompoundFile = useCompoundFile;
writer.Optimize();
// this is a commit
writer.Close();
Assert.AreEqual(2 * (N + 2), policy.numOnInit);
Assert.AreEqual(2 * (N + 2) - 1, policy.numOnCommit);
IndexSearcher searcher2 = new IndexSearcher(dir, false);
ScoreDoc[] hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
Assert.AreEqual(176, hits2.Length);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
int expectedCount = 176;
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir, true);
// Work backwards in commits on what the expected
// count should be.
searcher2 = new IndexSearcher(reader);
hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
if (i > 1)
{
if (i % 2 == 0)
{
expectedCount += 1;
}
else
{
expectedCount -= 17;
}
}
Assert.AreEqual(expectedCount, hits2.Length);
searcher2.Close();
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits before last 5");
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
public System.Object Run(IndexCommit commit)
{
if (commit != null)
{
if (directory != commit.GetDirectory())
{
throw new System.IO.IOException("the specified commit does not match the specified Directory");
}
return(DoBody(commit.GetSegmentsFileName()));
}
System.String segmentFileName = null;
long lastGen = -1;
long gen = 0;
int genLookaheadCount = 0;
System.IO.IOException exc = null;
bool retry = false;
int method = 0;
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
// We have three methods for determining the current
// generation. We try the first two in parallel, and
// fall back to the third when necessary.
while (true)
{
if (0 == method)
{
// Method 1: list the directory and use the highest
// segments_N file. This method works well as long
// as there is no stale caching on the directory
// contents (NOTE: NFS clients often have such stale
// caching):
System.String[] files = null;
long genA = -1;
files = directory.ListAll();
if (files != null)
{
genA = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files);
}
Lucene.Net.Index.SegmentInfos.Message("directory listing genA=" + genA);
// Method 2: open segments.gen and read its
// contents. Then we take the larger of the two
// gen's. This way, if either approach is hitting
// a stale cache (NFS) we have a better chance of
// getting the right generation.
long genB = -1;
for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++)
{
IndexInput genInput = null;
try
{
genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN);
}
catch (System.IO.FileNotFoundException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: FileNotFoundException " + e);
break;
}
catch (System.IO.IOException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e);
}
if (genInput != null)
{
try
{
int version = genInput.ReadInt();
if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS)
{
long gen0 = genInput.ReadLong();
long gen1 = genInput.ReadLong();
Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1);
if (gen0 == gen1)
{
// The file is consistent.
genB = gen0;
break;
}
}
}
catch (System.IO.IOException err2)
{
// will retry
}
finally
{
genInput.Close();
}
}
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec));
}
catch (System.Threading.ThreadInterruptedException ie)
{
// In 3.0 we will change this to throw
// InterruptedException instead
SupportClass.ThreadClass.Current().Interrupt();
throw new System.SystemException(ie.Message, ie);
}
}
Lucene.Net.Index.SegmentInfos.Message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);
// Pick the larger of the two gen's:
if (genA > genB)
{
gen = genA;
}
else
{
gen = genB;
}
if (gen == -1)
{
// Neither approach found a generation
System.String s;
if (files != null)
{
s = "";
for (int i = 0; i < files.Length; i++)
{
s += (" " + files[i]);
}
}
else
{
s = " null";
}
throw new System.IO.FileNotFoundException("no segments* file found in " + directory + ": files:" + s);
}
}
// Third method (fallback if first & second methods
// are not reliable): since both directory cache and
// file contents cache seem to be stale, just
// advance the generation.
if (1 == method || (0 == method && lastGen == gen && retry))
{
method = 1;
if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount)
{
gen++;
genLookaheadCount++;
Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen);
}
}
if (lastGen == gen)
{
// This means we're about to try the same
// segments_N last tried. This is allowed,
// exactly once, because writer could have been in
// the process of writing segments_N last time.
if (retry)
{
// OK, we've tried the same segments_N file
// twice in a row, so this must be a real
// error. We throw the original exception we
// got.
throw exc;
}
else
{
retry = true;
}
}
else if (0 == method)
{
// Segment file has advanced since our last loop, so
// reset retry:
retry = false;
}
lastGen = gen;
segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try
{
System.Object v = DoBody(segmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
}
return(v);
}
catch (System.IO.IOException err)
{
// Save the original root cause:
if (exc == null)
{
exc = err;
}
Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);
if (!retry && gen > 1)
{
// This is our first time trying this segments
// file (because retry is false), and, there is
// possibly a segments_(N-1) (because gen > 1).
// So, check if the segments_(N-1) exists and
// try it if so:
System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);
bool prevExists;
prevExists = directory.FileExists(prevSegmentFileName);
if (prevExists)
{
Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'");
try
{
System.Object v = DoBody(prevSegmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName);
}
return(v);
}
catch (System.IO.IOException err2)
{
Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
}
}
}
}
}
}
/// <summary> Get the filename of the current segments_N file
/// from a list of files.
///
/// </summary>
/// <param name="files">-- array of file names to check
/// </param>
public static System.String GetCurrentSegmentFileName(System.String[] files)
{
return(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", GetCurrentSegmentGeneration(files)));
}
public virtual void TestKeepLastNDeletionPolicy()
{
int N = 5;
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
Directory dir = new RAMDirectory();
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
for (int j = 0; j < N + 1; j++)
{
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int i = 0; i < 17; i++)
{
AddDoc(writer);
}
writer.Optimize();
writer.Close();
}
Assert.IsTrue(policy.numDelete > 0);
Assert.AreEqual(N + 1, policy.numOnInit);
if (autoCommit)
{
Assert.IsTrue(policy.numOnCommit > 1);
}
else
{
Assert.AreEqual(N + 1, policy.numOnCommit);
}
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits prior to last " + N);
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
/// <summary> Get the segments_N filename in use by this segment infos.</summary>
public System.String GetCurrentSegmentFileName()
{
return(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", lastGeneration));
}
public virtual void TestKeepLastNDeletionPolicyWithCreates()
{
int N = 10;
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
writer.Close();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for (int i = 0; i < N + 1; i++)
{
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
// this is a commit when autoCommit=false:
writer.Close();
IndexReader reader = IndexReader.Open(dir, policy);
reader.DeleteDocument(3);
reader.SetNorm(5, "content", 2.0F);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(16, hits.Length);
// this is a commit when autoCommit=false:
reader.Close();
searcher.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
// This will not commit: there are no changes
// pending because we opened for "create":
writer.Close();
}
Assert.AreEqual(1 + 3 * (N + 1), policy.numOnInit);
if (!autoCommit)
{
Assert.AreEqual(3 * (N + 1), policy.numOnCommit);
}
IndexSearcher searcher2 = new IndexSearcher(dir);
ScoreDoc[] hits2 = searcher2.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(0, hits2.Length);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
int expectedCount = 0;
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir);
// Work backwards in commits on what the expected
// count should be. Only check this in the
// autoCommit false case:
if (!autoCommit)
{
searcher2 = new IndexSearcher(reader);
hits2 = searcher2.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(expectedCount, hits2.Length);
searcher2.Close();
if (expectedCount == 0)
{
expectedCount = 16;
}
else if (expectedCount == 16)
{
expectedCount = 17;
}
else if (expectedCount == 17)
{
expectedCount = 0;
}
}
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits before last " + N);
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
public System.Object run()
{
System.String segmentFileName = null;
long lastGen = -1;
long gen = 0;
int genLookaheadCount = 0;
System.IO.IOException exc = null;
bool retry = false;
int method = 0;
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
// We have three methods for determining the current
// generation. We try each in sequence.
while (true)
{
// Method 1: list the directory and use the highest
// segments_N file. This method works well as long
// as there is no stale caching on the directory
// contents:
System.String[] files = null;
if (0 == method)
{
if (directory != null)
{
files = directory.List();
}
else
{
files = System.IO.Directory.GetFileSystemEntries(fileDirectory.FullName);
for (int i = 0; i < files.Length; i++)
{
files[i] = System.IO.Path.GetFileName(files[i]);
}
}
gen = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files);
if (gen == -1)
{
System.String s = "";
for (int i = 0; i < files.Length; i++)
{
s += (" " + files[i]);
}
throw new System.IO.FileNotFoundException("no segments* file found: files:" + s);
}
}
// Method 2 (fallback if Method 1 isn't reliable):
// if the directory listing seems to be stale, then
// try loading the "segments.gen" file.
if (1 == method || (0 == method && lastGen == gen && retry))
{
method = 1;
for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++)
{
IndexInput genInput = null;
try
{
genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN);
}
catch (System.IO.IOException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e);
}
if (genInput != null)
{
try
{
int version = genInput.ReadInt();
if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS)
{
long gen0 = genInput.ReadLong();
long gen1 = genInput.ReadLong();
Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1);
if (gen0 == gen1)
{
// The file is consistent.
if (gen0 > gen)
{
Lucene.Net.Index.SegmentInfos.Message("fallback to '" + IndexFileNames.SEGMENTS_GEN + "' check: now try generation " + gen0 + " > " + gen);
gen = gen0;
}
break;
}
}
}
catch (System.IO.IOException err2)
{
// will retry
}
finally
{
genInput.Close();
}
}
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec));
}
catch (System.Threading.ThreadInterruptedException e)
{
// will retry
}
}
}
// Method 3 (fallback if Methods 2 & 3 are not
// reliable): since both directory cache and file
// contents cache seem to be stale, just advance the
// generation.
if (2 == method || (1 == method && lastGen == gen && retry))
{
method = 2;
if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount)
{
gen++;
genLookaheadCount++;
Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen);
}
}
if (lastGen == gen)
{
// This means we're about to try the same
// segments_N last tried. This is allowed,
// exactly once, because writer could have been in
// the process of writing segments_N last time.
if (retry)
{
// OK, we've tried the same segments_N file
// twice in a row, so this must be a real
// error. We throw the original exception we
// got.
throw exc;
}
else
{
retry = true;
}
}
else
{
// Segment file has advanced since our last loop, so
// reset retry:
retry = false;
}
lastGen = gen;
segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try
{
System.Object v = DoBody(segmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
}
return(v);
}
catch (System.IO.IOException err)
{
// Save the original root cause:
if (exc == null)
{
exc = err;
}
Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);
if (!retry && gen > 1)
{
// This is our first time trying this segments
// file (because retry is false), and, there is
// possibly a segments_(N-1) (because gen > 1).
// So, check if the segments_(N-1) exists and
// try it if so:
System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);
if (directory.FileExists(prevSegmentFileName))
{
Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'");
try
{
System.Object v = DoBody(prevSegmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName);
}
return(v);
}
catch (System.IO.IOException err2)
{
Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
}
}
}
}
}
}
public virtual void TestExpirationTimeDeletionPolicy()
{
const double SECONDS = 2.0;
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(this, dir, SECONDS));
MergePolicy mp = conf.MergePolicy;
mp.NoCFSRatio = 1.0;
IndexWriter writer = new IndexWriter(dir, conf);
ExpirationTimeDeletionPolicy policy = (ExpirationTimeDeletionPolicy)writer.Config.IndexDeletionPolicy;
IDictionary <string, string> commitData = new Dictionary <string, string>();
commitData["commitTime"] = Convert.ToString(Environment.TickCount);
writer.SetCommitData(commitData);
writer.Commit();
writer.Dispose();
long lastDeleteTime = 0;
int targetNumDelete = TestUtil.NextInt32(Random, 1, 5);
while (policy.NumDelete < targetNumDelete)
{
// Record last time when writer performed deletes of
// past commits
lastDeleteTime = Environment.TickCount;
conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetIndexDeletionPolicy(policy);
mp = conf.MergePolicy;
mp.NoCFSRatio = 1.0;
writer = new IndexWriter(dir, conf);
policy = (ExpirationTimeDeletionPolicy)writer.Config.IndexDeletionPolicy;
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
commitData = new Dictionary <string, string>();
commitData["commitTime"] = Convert.ToString(Environment.TickCount);
writer.SetCommitData(commitData);
writer.Commit();
writer.Dispose();
Thread.Sleep((int)(1000.0 * (SECONDS / 5.0)));
}
// Then simplistic check: just verify that the
// segments_N's that still exist are in fact within SECONDS
// seconds of the last one's mod time, and, that I can
// open a reader on each:
long gen = SegmentInfos.GetLastCommitGeneration(dir);
string fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
bool oneSecondResolution = true;
while (gen > 0)
{
try
{
IndexReader reader = DirectoryReader.Open(dir);
reader.Dispose();
fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
// if we are on a filesystem that seems to have only
// 1 second resolution, allow +1 second in commit
// age tolerance:
SegmentInfos sis = new SegmentInfos();
sis.Read(dir, fileName);
long modTime = Convert.ToInt64(sis.UserData["commitTime"]);
oneSecondResolution &= (modTime % 1000) == 0;
long leeway = (long)((SECONDS + (oneSecondResolution ? 1.0 : 0.0)) * 1000);
Assert.IsTrue(lastDeleteTime - modTime <= leeway, "commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted ");
}
#pragma warning disable 168
catch (IOException e)
#pragma warning restore 168
{
// OK
break;
}
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
}
dir.Dispose();
}
