private IndexSegmentsInfo GetCurrentSegmentsInfo()
{
var segmentInfos = new SegmentInfos();
var result = new IndexSegmentsInfo();
try
{
segmentInfos.Read(directory);
result.Generation = segmentInfos.Generation;
result.SegmentsFileName = segmentInfos.GetCurrentSegmentFileName();
result.ReferencedFiles = segmentInfos.Files(directory, false);
}
catch (CorruptIndexException ex)
{
logIndexing.WarnException(string.Format("Could not read segment information for an index '{0}'", indexId), ex);
result.IsIndexCorrupted = true;
}
return(result);
}
public override void Dispose()
{
lock (this)
{
// files that we tried to delete, but couldn't because readers were open.
// all that matters is that we tried! (they will eventually go away)
ISet<string> pendingDeletions = new HashSet<string>(OpenFilesDeleted);
MaybeYield();
if (OpenFiles == null)
{
OpenFiles = new Dictionary<string, int>();
OpenFilesDeleted = new HashSet<string>();
}
if (OpenFiles.Count > 0)
{
// print the first one as its very verbose otherwise
Exception cause = null;
IEnumerator<Exception> stacktraces = OpenFileHandles.Values.GetEnumerator();
if (stacktraces.MoveNext())
{
cause = stacktraces.Current;
}
// RuntimeException instead ofSystem.IO.IOException because
// super() does not throwSystem.IO.IOException currently:
throw new Exception("MockDirectoryWrapper: cannot close: there are still open files: "
+ String.Join(" ,", OpenFiles.ToArray().Select(x => x.Key)), cause);
}
if (OpenLocks.Count > 0)
{
throw new Exception("MockDirectoryWrapper: cannot close: there are still open locks: "
+ String.Join(" ,", OpenLocks.ToArray()));
}
IsOpen = false;
if (CheckIndexOnClose)
{
RandomIOExceptionRate_Renamed = 0.0;
RandomIOExceptionRateOnOpen_Renamed = 0.0;
if (DirectoryReader.IndexExists(this))
{
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("\nNOTE: MockDirectoryWrapper: now crush");
}
Crash(); // corrupt any unsynced-files
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
}
TestUtil.CheckIndex(this, CrossCheckTermVectorsOnClose);
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
if (AssertNoUnreferencedFilesOnClose)
{
// now look for unreferenced files: discount ones that we tried to delete but could not
HashSet<string> allFiles = new HashSet<string>(Arrays.AsList(ListAll()));
allFiles.RemoveAll(pendingDeletions);
string[] startFiles = allFiles.ToArray(/*new string[0]*/);
IndexWriterConfig iwc = new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null);
iwc.SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
(new IndexWriter(@in, iwc)).Rollback();
string[] endFiles = @in.ListAll();
ISet<string> startSet = new SortedSet<string>(Arrays.AsList(startFiles));
ISet<string> endSet = new SortedSet<string>(Arrays.AsList(endFiles));
if (pendingDeletions.Contains("segments.gen") && endSet.Contains("segments.gen"))
{
// this is possible if we hit an exception while writing segments.gen, we try to delete it
// and it ends out in pendingDeletions (but IFD wont remove this).
startSet.Add("segments.gen");
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MDW: Unreferenced check: Ignoring segments.gen that we could not delete.");
}
}
// its possible we cannot delete the segments_N on windows if someone has it open and
// maybe other files too, depending on timing. normally someone on windows wouldnt have
// an issue (IFD would nuke this stuff eventually), but we pass NoDeletionPolicy...
foreach (string file in pendingDeletions)
{
if (file.StartsWith("segments") && !file.Equals("segments.gen") && endSet.Contains(file))
{
startSet.Add(file);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MDW: Unreferenced check: Ignoring segments file: " + file + " that we could not delete.");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(@in, file);
}
catch (System.IO.IOException ioe)
{
// OK: likely some of the .si files were deleted
}
try
{
ISet<string> ghosts = new HashSet<string>(sis.Files(@in, false));
foreach (string s in ghosts)
{
if (endSet.Contains(s) && !startSet.Contains(s))
{
Debug.Assert(pendingDeletions.Contains(s));
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MDW: Unreferenced check: Ignoring referenced file: " + s + " " + "from " + file + " that we could not delete.");
}
startSet.Add(s);
}
}
}
catch (Exception t)
{
Console.Error.WriteLine("ERROR processing leftover segments file " + file + ":");
Console.WriteLine(t.ToString());
Console.Write(t.StackTrace);
}
}
}
startFiles = startSet.ToArray(/*new string[0]*/);
endFiles = endSet.ToArray(/*new string[0]*/);
if (!Arrays.Equals(startFiles, endFiles))
{
IList<string> removed = new List<string>();
foreach (string fileName in startFiles)
{
if (!endSet.Contains(fileName))
{
removed.Add(fileName);
}
}
IList<string> added = new List<string>();
foreach (string fileName in endFiles)
{
if (!startSet.Contains(fileName))
{
added.Add(fileName);
}
}
string extras;
if (removed.Count != 0)
{
extras = "\n\nThese files were removed: " + removed;
}
else
{
extras = "";
}
if (added.Count != 0)
{
extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
}
if (pendingDeletions.Count != 0)
{
extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
}
Debug.Assert(false, "unreferenced files: before delete:\n " + Arrays.ToString(startFiles) + "\n after delete:\n " + Arrays.ToString(endFiles) + extras);
}
DirectoryReader ir1 = DirectoryReader.Open(this);
int numDocs1 = ir1.NumDocs;
ir1.Dispose();
(new IndexWriter(this, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null))).Dispose();
DirectoryReader ir2 = DirectoryReader.Open(this);
int numDocs2 = ir2.NumDocs;
ir2.Dispose();
Debug.Assert(numDocs1 == numDocs2, "numDocs changed after opening/closing IW: before=" + numDocs1 + " after=" + numDocs2);
}
}
}
@in.Dispose();
}
}
/// <summary>This constructor is only used for <see cref="Reopen()" /> </summary>
internal DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
IEnumerable<KeyValuePair<string, byte[]>> oldNormsCache, bool readOnly, bool doClone, int termInfosIndexDivisor)
{
this.internalDirectory = directory;
this.readOnly = readOnly;
this.segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (!readOnly)
{
// We assume that this segments_N was previously
// properly sync'd:
synced.UnionWith(infos.Files(directory, true));
}
// we put the old SegmentReaders in a map, that allows us
// to lookup a reader using its segment name
IDictionary<string, int> segmentReaders = new HashMap<string, int>();
if (oldReaders != null)
{
// create a Map SegmentName->SegmentReader
for (int i = 0; i < oldReaders.Length; i++)
{
segmentReaders[oldReaders[i].SegmentName] = i;
}
}
var newReaders = new SegmentReader[infos.Count];
// remember which readers are shared between the old and the re-opened
// DirectoryReader - we have to incRef those readers
var readerShared = new bool[infos.Count];
for (int i = infos.Count - 1; i >= 0; i--)
{
// find SegmentReader for this segment
if (!segmentReaders.ContainsKey(infos.Info(i).name))
{
// this is a new segment, no old SegmentReader can be reused
newReaders[i] = null;
}
else
{
// there is an old reader for this segment - we'll try to reopen it
newReaders[i] = oldReaders[segmentReaders[infos.Info(i).name]];
}
bool success = false;
try
{
SegmentReader newReader;
if (newReaders[i] == null || infos.Info(i).GetUseCompoundFile() != newReaders[i].SegmentInfo.GetUseCompoundFile())
{
// We should never see a totally new segment during cloning
System.Diagnostics.Debug.Assert(!doClone);
// this is a new reader; in case we hit an exception we can close it safely
newReader = SegmentReader.Get(readOnly, infos.Info(i), termInfosIndexDivisor);
}
else
{
newReader = newReaders[i].ReopenSegment(infos.Info(i), doClone, readOnly);
}
if (newReader == newReaders[i])
{
// this reader will be shared between the old and the new one,
// so we must incRef it
readerShared[i] = true;
newReader.IncRef();
}
else
{
readerShared[i] = false;
newReaders[i] = newReader;
}
success = true;
}
finally
{
if (!success)
{
for (i++; i < infos.Count; i++)
{
if (newReaders[i] != null)
{
try
{
if (!readerShared[i])
{
// this is a new subReader that is not used by the old one,
// we can close it
newReaders[i].Close();
}
else
{
// this subReader is also used by the old reader, so instead
// closing we must decRef it
newReaders[i].DecRef();
}
}
catch (System.IO.IOException)
{
// keep going - we want to clean up as much as possible
}
}
}
}
}
}
// initialize the readers to calculate maxDoc before we try to reuse the old normsCache
Initialize(newReaders);
// try to copy unchanged norms from the old normsCache to the new one
if (oldNormsCache != null)
{
foreach(var entry in oldNormsCache)
{
String field = entry.Key;
if (!HasNorms(field))
{
continue;
}
byte[] oldBytes = entry.Value;
var bytes = new byte[MaxDoc];
for (int i = 0; i < subReaders.Length; i++)
{
int oldReaderIndex = segmentReaders[subReaders[i].SegmentName];
// this SegmentReader was not re-opened, we can copy all of its norms
if (segmentReaders.ContainsKey(subReaders[i].SegmentName) &&
(oldReaders[oldReaderIndex] == subReaders[i]
|| oldReaders[oldReaderIndex].norms[field] == subReaders[i].norms[field]))
{
// we don't have to synchronize here: either this constructor is called from a SegmentReader,
// in which case no old norms cache is present, or it is called from MultiReader.reopen(),
// which is synchronized
Array.Copy(oldBytes, oldStarts[oldReaderIndex], bytes, starts[i], starts[i + 1] - starts[i]);
}
else
{
subReaders[i].Norms(field, bytes, starts[i]);
}
}
normsCache[field] = bytes; // update cache
}
}
}
// Used by near real-time search
internal DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor)
{
this.internalDirectory = writer.Directory;
this.readOnly = true;
segmentInfos = infos;
segmentInfosStart = (SegmentInfos) infos.Clone();
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (!readOnly)
{
// We assume that this segments_N was previously
// properly sync'd:
synced.UnionWith(infos.Files(internalDirectory, true));
}
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
// no need to process segments in reverse order
int numSegments = infos.Count;
var readers = new SegmentReader[numSegments];
Directory dir = writer.Directory;
int upto = 0;
for (int i = 0; i < numSegments; i++)
{
bool success = false;
try
{
SegmentInfo info = infos.Info(i);
if (info.dir == dir)
{
readers[upto++] = writer.readerPool.GetReadOnlyClone(info, true, termInfosIndexDivisor);
}
success = true;
}
finally
{
if (!success)
{
// Close all readers we had opened:
for (upto--; upto >= 0; upto--)
{
try
{
readers[upto].Close();
}
catch (System.Exception)
{
// keep going - we want to clean up as much as possible
}
}
}
}
}
this.writer = writer;
if (upto < readers.Length)
{
// This means some segments were in a foreign Directory
var newReaders = new SegmentReader[upto];
Array.Copy(readers, 0, newReaders, 0, upto);
readers = newReaders;
}
Initialize(readers);
}
/// <summary>Construct reading the named set of readers. </summary>
internal DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, bool readOnly, int termInfosIndexDivisor)
{
internalDirectory = directory;
this.readOnly = readOnly;
this.segmentInfos = sis;
this.deletionPolicy = deletionPolicy;
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (!readOnly)
{
// We assume that this segments_N was previously
// properly sync'd:
synced.UnionWith(sis.Files(directory, true));
}
// To reduce the chance of hitting FileNotFound
// (and having to retry), we open segments in
// reverse because IndexWriter merges & deletes
// the newest segments first.
var readers = new SegmentReader[sis.Count];
for (int i = sis.Count - 1; i >= 0; i--)
{
bool success = false;
try
{
readers[i] = SegmentReader.Get(readOnly, sis.Info(i), termInfosIndexDivisor);
success = true;
}
finally
{
if (!success)
{
// Close all readers we had opened:
for (i++; i < sis.Count; i++)
{
try
{
readers[i].Close();
}
catch (System.Exception)
{
// keep going - we want to clean up as much as possible
}
}
}
}
}
Initialize(readers);
}
internal ReaderCommit(SegmentInfos infos, Directory dir)
{
segmentsFileName = infos.GetCurrentSegmentFileName();
this.dir = dir;
userData = infos.UserData;
files = infos.Files(dir, true);
version = infos.Version;
generation = infos.Generation;
isOptimized = infos.Count == 1 && !infos.Info(0).HasDeletions();
}