/// <summary> Construct a FieldInfos object using the directory and the name of the file
/// IndexInput
/// </summary>
/// <param name="d">The directory to open the IndexInput from
/// </param>
/// <param name="name">The name of the file to open the IndexInput from in the Directory
/// </param>
/// <throws> IOException </throws>
public /*internal*/ FieldInfos(Directory d, string name)
{
IndexInput input = d.OpenInput(name);
try
{
try
{
Read(input, name);
}
catch (System.IO.IOException)
{
if (format == FORMAT_PRE)
{
// LUCENE-1623: FORMAT_PRE (before there was a
// format) may be 2.3.2 (pre-utf8) or 2.4.x (utf8)
// encoding; retry with input set to pre-utf8
input.Seek(0);
input.SetModifiedUTF8StringsMode();
byNumber.Clear();
byName.Clear();
bool rethrow = false;
try
{
Read(input, name);
}
catch (Exception)
{
// Ignore any new exception & set to throw original IOE
rethrow = true;
}
if (rethrow)
{
// Preserve stack trace
throw;
}
}
else
{
// The IOException cannot be caused by
// LUCENE-1623, so re-throw it
throw;
}
}
}
finally
{
input.Close();
}
}
public void Read(Directory directory)
{
IndexInput input = directory.OpenInput(IndexFileNames.SEGMENTS);
try
{
int format = input.ReadInt();
if (format < 0)
{
// file contains explicit format info
// check that it is a format we can understand
if (format < FORMAT)
{
throw new System.IO.IOException("Unknown format version: " + format);
}
version = input.ReadLong(); // read version
counter = input.ReadInt(); // read counter
}
else
{
// file is in old format without explicit format info
counter = format;
}
for (int i = input.ReadInt(); i > 0; i--)
{
// read segmentInfos
SegmentInfo si = new SegmentInfo(input.ReadString(), input.ReadInt(), directory);
Add(si);
}
if (format >= 0)
{
// in old format the version number may be at the end of the file
if (input.GetFilePointer() >= input.Length())
{
version = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
}
// old file format without version number
else
{
version = input.ReadLong(); // read version
}
}
}
finally
{
input.Close();
}
}
/// <summary>Copy the contents of the file with specified extension into the
/// provided output stream. Use the provided buffer for moving data
/// to reduce memory allocation.
/// </summary>
private void CopyFile(FileEntry source, IndexOutput os, byte[] buffer)
{
IndexInput is_Renamed = null;
try
{
long startPtr = os.GetFilePointer();
is_Renamed = directory.OpenInput(source.file);
long length = is_Renamed.Length();
long remainder = length;
int chunk = buffer.Length;
while (remainder > 0)
{
int len = (int)System.Math.Min(chunk, remainder);
is_Renamed.ReadBytes(buffer, 0, len, false);
os.WriteBytes(buffer, len);
remainder -= len;
if (checkAbort != null)
{
// Roughly every 2 MB we will check if
// it's time to abort
checkAbort.Work(80);
}
}
// Verify that remainder is 0
if (remainder != 0)
{
throw new System.IO.IOException("Non-zero remainder length after copying: " + remainder + " (id: " + source.file + ", length: " + length + ", buffer size: " + chunk + ")");
}
// Verify that the output length diff is equal to original file
long endPtr = os.GetFilePointer();
long diff = endPtr - startPtr;
if (diff != length)
{
throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length);
}
}
finally
{
if (is_Renamed != null)
{
is_Renamed.Close();
}
}
}
/// <summary>Constructs a bit vector from the file <code>name</code> in Directory
/// <code>d</code>, as written by the {@link #write} method.
/// </summary>
public BitVector(Directory d, System.String name)
{
IndexInput input = d.OpenInput(name);
try
{
size = input.ReadInt(); // read size
count = input.ReadInt(); // read count
bits = new byte[(size >> 3) + 1]; // allocate bits
input.ReadBytes(bits, 0, bits.Length); // read bits
}
finally
{
input.Close();
}
}
protected override void Dispose(bool disposing)
{
if (isDisposed)
{
return;
}
if (disposing)
{
if (base_Renamed != null)
{
base_Renamed.Close();
}
}
isDisposed = true;
}
public virtual void TestRandomFiles()
{
// Setup the test segment
System.String segment = "test";
int chunk = 1024; // internal buffer size used by the stream
CreateRandomFile(dir, segment + ".zero", 0);
CreateRandomFile(dir, segment + ".one", 1);
CreateRandomFile(dir, segment + ".ten", 10);
CreateRandomFile(dir, segment + ".hundred", 100);
CreateRandomFile(dir, segment + ".big1", chunk);
CreateRandomFile(dir, segment + ".big2", chunk - 1);
CreateRandomFile(dir, segment + ".big3", chunk + 1);
CreateRandomFile(dir, segment + ".big4", 3 * chunk);
CreateRandomFile(dir, segment + ".big5", 3 * chunk - 1);
CreateRandomFile(dir, segment + ".big6", 3 * chunk + 1);
CreateRandomFile(dir, segment + ".big7", 1000 * chunk);
// Setup extraneous files
CreateRandomFile(dir, "onetwothree", 100);
CreateRandomFile(dir, segment + ".notIn", 50);
CreateRandomFile(dir, segment + ".notIn2", 51);
// Now test
CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
System.String[] data = new System.String[] { ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3", ".big4", ".big5", ".big6", ".big7" };
for (int i = 0; i < data.Length; i++)
{
csw.AddFile(segment + data[i]);
}
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
for (int i = 0; i < data.Length; i++)
{
IndexInput check = dir.OpenInput(segment + data[i]);
IndexInput test = csr.OpenInput(segment + data[i]);
AssertSameStreams(data[i], check, test);
AssertSameSeekBehavior(data[i], check, test);
test.Close();
check.Close();
}
csr.Close();
}
protected override void Dispose(bool disposing)
{
if (isDisposed)
{
return;
}
if (disposing)
{
if (delegate_Renamed != null)
{
delegate_Renamed.Close();
}
}
delegate_Renamed = null;
isDisposed = true;
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp", null);
IndexInput is_Renamed = cr.OpenInput("f2", null);
is_Renamed.Seek(is_Renamed.Length(null) - 10, null);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10, null);
Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadByte(null), "Single byte read past end of file");
is_Renamed.Seek(is_Renamed.Length(null) - 10, null);
Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadBytes(b, 0, 50, null), "Block read past end of file");
is_Renamed.Close();
cr.Close();
}
public virtual void CopyFile(Directory dir, System.String src, System.String dest)
{
IndexInput in_Renamed = dir.OpenInput(src);
IndexOutput out_Renamed = dir.CreateOutput(dest);
byte[] b = new byte[1024];
long remainder = in_Renamed.Length();
while (remainder > 0)
{
int len = (int)System.Math.Min(b.Length, remainder);
in_Renamed.ReadBytes(b, 0, len);
out_Renamed.WriteBytes(b, len);
remainder -= len;
}
in_Renamed.Close();
out_Renamed.Close();
}
/// <summary>Constructs a bit vector from the file <code>name</code> in Directory
/// <code>d</code>, as written by the {@link #write} method.
/// </summary>
public BitVector(Directory d, System.String name)
{
IndexInput input = d.OpenInput(name);
try
{
size = input.ReadInt(); // read size
if (size == -1)
{
ReadDgaps(input);
}
else
{
ReadBits(input);
}
}
finally
{
input.Close();
}
}
public virtual void TestSingleFile()
{
int[] data = new int[] { 0, 1, 10, 100 };
for (int i = 0; i < data.Length; i++)
{
System.String name = "t" + data[i];
CreateSequenceFile(dir, name, (byte)0, data[i]);
CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
csw.AddFile(name);
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
IndexInput expected = dir.OpenInput(name);
IndexInput actual = csr.OpenInput(name);
AssertSameStreams(name, expected, actual);
AssertSameSeekBehavior(name, expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
}
private void Demo_FSIndexInputBug(Directory fsdir, System.String file)
{
// Setup the test file - we need more than 1024 bytes
IndexOutput os = fsdir.CreateOutput(file);
for (int i = 0; i < 2000; i++)
{
os.WriteByte((byte)i);
}
os.Close();
IndexInput in_Renamed = fsdir.OpenInput(file);
// This read primes the buffer in IndexInput
byte b = in_Renamed.ReadByte();
// Close the file
in_Renamed.Close();
// ERROR: this call should fail, but succeeds because the buffer
// is still filled
b = in_Renamed.ReadByte();
// ERROR: this call should fail, but succeeds for some reason as well
in_Renamed.Seek(1099);
try
{
// OK: this call correctly fails. We are now past the 1024 internal
// buffer, so an actual IO is attempted, which fails
b = in_Renamed.ReadByte();
Assert.Fail("expected readByte() to throw exception");
}
catch (System.Exception e)
{
// expected exception
}
}
/// <summary> Current version number from segments file.</summary>
public static long ReadCurrentVersion(Directory directory)
{
IndexInput input = directory.OpenInput(IndexFileNames.SEGMENTS);
int format = 0;
long version = 0;
try
{
format = input.ReadInt();
if (format < 0)
{
if (format < FORMAT)
{
throw new System.IO.IOException("Unknown format version: " + format);
}
version = input.ReadLong(); // read version
}
}
finally
{
input.Close();
}
if (format < 0)
{
return(version);
}
// We cannot be sure about the format of the file.
// Therefore we have to read the whole file and cannot simply seek to the version entry.
SegmentInfos sis = new SegmentInfos();
sis.Read(directory);
return(sis.GetVersion());
}
private System.Collections.ArrayList ReadDeleteableFiles()
{
System.Collections.ArrayList result = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10));
if (!directory.FileExists(IndexFileNames.DELETABLE))
{
return(result);
}
IndexInput input = directory.OpenInput(IndexFileNames.DELETABLE);
try
{
for (int i = input.ReadInt(); i > 0; i--)
{
// read file names
result.Add(input.ReadString());
}
}
finally
{
input.Close();
}
return(result);
}
public override System.Object DoBody(System.String segmentFileName)
{
IndexInput input = directory.OpenInput(segmentFileName);
int format = 0;
long version = 0;
try
{
format = input.ReadInt();
if (format < 0)
{
if (format < Lucene.Net.Index.SegmentInfos.FORMAT_SINGLE_NORM_FILE)
{
throw new System.IO.IOException("Unknown format version: " + format);
}
version = input.ReadLong(); // read version
}
}
finally
{
input.Close();
}
if (format < 0)
{
return((long)version);
}
// We cannot be sure about the format of the file.
// Therefore we have to read the whole file and cannot simply seek to the version entry.
SegmentInfos sis = new SegmentInfos();
sis.Read(directory, segmentFileName);
return((long)sis.GetVersion());
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
IndexInput is_Renamed = cr.OpenInput("f2");
is_Renamed.Seek(is_Renamed.Length() - 10);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10);
try
{
byte test = is_Renamed.ReadByte();
Assert.Fail("Single byte read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: single byte read past end of file: " + e);
}
is_Renamed.Seek(is_Renamed.Length() - 10);
try
{
is_Renamed.ReadBytes(b, 0, 50);
Assert.Fail("Block read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: block read past end of file: " + e);
}
is_Renamed.Close();
cr.Close();
}
/// <summary>Returns true if index is clean, else false.</summary>
public static bool Check(Directory dir, bool doFix)
{
System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(dir);
}
catch (System.Exception t)
{
out_Renamed.WriteLine("ERROR: could not read any segments file in directory");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
int numSegments = sis.Count;
System.String segmentsFileName = sis.GetCurrentSegmentFileName();
IndexInput input = null;
try
{
input = dir.OpenInput(segmentsFileName);
}
catch (System.Exception t)
{
out_Renamed.WriteLine("ERROR: could not open segments file in directory");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
int format = 0;
try
{
format = input.ReadInt();
}
catch (System.Exception t)
{
out_Renamed.WriteLine("ERROR: could not read segment file version in directory");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
finally
{
if (input != null)
{
input.Close();
}
}
System.String sFormat = "";
bool skip = false;
if (format == SegmentInfos.FORMAT)
{
sFormat = "FORMAT [Lucene Pre-2.1]";
}
if (format == SegmentInfos.FORMAT_LOCKLESS)
{
sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
}
else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
{
sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
}
else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
}
else if (format < SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
sFormat = "int=" + format + " [newer version of Lucene than this tool]";
skip = true;
}
else
{
sFormat = format + " [Lucene 1.3 or prior]";
}
out_Renamed.WriteLine("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat);
if (skip)
{
out_Renamed.WriteLine("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
return(false);
}
SegmentInfos newSIS = (SegmentInfos)sis.Clone();
newSIS.Clear();
bool changed = false;
int totLoseDocCount = 0;
int numBadSegments = 0;
for (int i = 0; i < numSegments; i++)
{
SegmentInfo info = sis.Info(i);
out_Renamed.WriteLine(" " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
int toLoseDocCount = info.docCount;
SegmentReader reader = null;
try
{
out_Renamed.WriteLine(" compound=" + info.GetUseCompoundFile());
out_Renamed.WriteLine(" numFiles=" + info.Files().Count);
out_Renamed.WriteLine(String.Format(nf, " size (MB)={0:f}", new Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) }));
int docStoreOffset = info.GetDocStoreOffset();
if (docStoreOffset != -1)
{
out_Renamed.WriteLine(" docStoreOffset=" + docStoreOffset);
out_Renamed.WriteLine(" docStoreSegment=" + info.GetDocStoreSegment());
out_Renamed.WriteLine(" docStoreIsCompoundFile=" + info.GetDocStoreIsCompoundFile());
}
System.String delFileName = info.GetDelFileName();
if (delFileName == null)
{
out_Renamed.WriteLine(" no deletions");
}
else
{
out_Renamed.WriteLine(" has deletions [delFileName=" + delFileName + "]");
}
out_Renamed.Write(" test: open reader.........");
reader = SegmentReader.Get(info);
int numDocs = reader.NumDocs();
toLoseDocCount = numDocs;
if (reader.HasDeletions())
{
out_Renamed.WriteLine("OK [" + (info.docCount - numDocs) + " deleted docs]");
}
else
{
out_Renamed.WriteLine("OK");
}
out_Renamed.Write(" test: fields, norms.......");
System.Collections.IDictionary fieldNames = (System.Collections.IDictionary)reader.GetFieldNames(IndexReader.FieldOption.ALL);
System.Collections.IEnumerator it = fieldNames.Keys.GetEnumerator();
while (it.MoveNext())
{
System.String fieldName = (System.String)it.Current;
byte[] b = reader.Norms(fieldName);
if (b.Length != info.docCount)
{
throw new System.SystemException("norms for field \"" + fieldName + "\" is length " + b.Length + " != maxDoc " + info.docCount);
}
}
out_Renamed.WriteLine("OK [" + fieldNames.Count + " fields]");
out_Renamed.Write(" test: terms, freq, prox...");
TermEnum termEnum = reader.Terms();
TermPositions termPositions = reader.TermPositions();
// Used only to count up # deleted docs for this
// term
MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);
long termCount = 0;
long totFreq = 0;
long totPos = 0;
while (termEnum.Next())
{
termCount++;
Term term = termEnum.Term();
int docFreq = termEnum.DocFreq();
termPositions.Seek(term);
int lastDoc = -1;
int freq0 = 0;
totFreq += docFreq;
while (termPositions.Next())
{
freq0++;
int doc = termPositions.Doc();
int freq = termPositions.Freq();
if (doc <= lastDoc)
{
throw new System.SystemException("term " + term + ": doc " + doc + " < lastDoc " + lastDoc);
}
lastDoc = doc;
if (freq <= 0)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
}
int lastPos = -1;
totPos += freq;
for (int j = 0; j < freq; j++)
{
int pos = termPositions.NextPosition();
if (pos < 0)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
}
if (pos <= lastPos)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
}
}
}
// Now count how many deleted docs occurred in
// this term:
int delCount;
if (reader.HasDeletions())
{
myTermDocs.Seek(term);
while (myTermDocs.Next())
{
}
delCount = myTermDocs.delCount;
}
else
{
delCount = 0;
}
if (freq0 + delCount != docFreq)
{
throw new System.SystemException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
}
}
out_Renamed.WriteLine("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]");
out_Renamed.Write(" test: stored fields.......");
int docCount = 0;
long totFields = 0;
for (int j = 0; j < info.docCount; j++)
{
if (!reader.IsDeleted(j))
{
docCount++;
Document doc = reader.Document(j);
totFields += doc.GetFields().Count;
}
}
if (docCount != reader.NumDocs())
{
throw new System.SystemException("docCount=" + docCount + " but saw " + docCount + " undeleted docs");
}
out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total field count; avg {1:f} fields per doc]", new Object[] { totFields, (((float)totFields) / docCount) }));
out_Renamed.Write(" test: term vectors........");
int totVectors = 0;
for (int j = 0; j < info.docCount; j++)
{
if (!reader.IsDeleted(j))
{
TermFreqVector[] tfv = reader.GetTermFreqVectors(j);
if (tfv != null)
{
totVectors += tfv.Length;
}
}
}
out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total vector count; avg {1:f} term/freq vector fields per doc]", new Object[] { totVectors, (((float)totVectors) / docCount) }));
out_Renamed.WriteLine("");
}
catch (System.Exception t)
{
out_Renamed.WriteLine("FAILED");
System.String comment;
if (doFix)
{
comment = "will remove reference to this segment (-fix is specified)";
}
else
{
comment = "would remove reference to this segment (-fix was not specified)";
}
out_Renamed.WriteLine(" WARNING: " + comment + "; full exception:");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
out_Renamed.WriteLine("");
totLoseDocCount += toLoseDocCount;
numBadSegments++;
changed = true;
continue;
}
finally
{
if (reader != null)
{
reader.Close();
}
}
// Keeper
newSIS.Add(info.Clone());
}
if (!changed)
{
out_Renamed.WriteLine("No problems were detected with this index.\n");
return(true);
}
else
{
out_Renamed.WriteLine("WARNING: " + numBadSegments + " broken segments detected");
if (doFix)
{
out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents will be lost");
}
else
{
out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents would be lost if -fix were specified");
}
out_Renamed.WriteLine();
}
if (doFix)
{
out_Renamed.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
for (int i = 0; i < 5; i++)
{
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
}
catch (System.Threading.ThreadInterruptedException)
{
SupportClass.ThreadClass.Current().Interrupt();
i--;
continue;
}
out_Renamed.WriteLine(" " + (5 - i) + "...");
}
out_Renamed.Write("Writing...");
try
{
newSIS.Write(dir);
}
catch (System.Exception t)
{
out_Renamed.WriteLine("FAILED; exiting");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
out_Renamed.WriteLine("OK");
out_Renamed.WriteLine("Wrote new segments file \"" + newSIS.GetCurrentSegmentFileName() + "\"");
}
else
{
out_Renamed.WriteLine("NOTE: would write new segments file [-fix was not specified]");
}
out_Renamed.WriteLine("");
return(false);
}
/// <summary>Closes the stream to further operations. </summary>
public override void Close()
{
base_Renamed.Close();
}
/// <summary>Returns a {@link Status} instance detailing
/// the state of the index.
///
/// </summary>
/// <param name="onlySegments">list of specific segment names to check
///
/// <p/>As this method checks every byte in the specified
/// segments, on a large index it can take quite a long
/// time to run.
///
/// <p/><b>WARNING</b>: make sure
/// you only call this when the index is not opened by any
/// writer.
/// </param>
public virtual Status CheckIndex_Renamed_Method(System.Collections.IList onlySegments)
{
System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
SegmentInfos sis = new SegmentInfos();
Status result = new Status();
result.dir = dir;
try
{
sis.Read(dir);
}
catch (System.Exception t)
{
Msg("ERROR: could not read any segments file in directory");
result.missingSegments = true;
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
return(result);
}
int numSegments = sis.Count;
System.String segmentsFileName = sis.GetCurrentSegmentFileName();
IndexInput input = null;
try
{
input = dir.OpenInput(segmentsFileName);
}
catch (System.Exception t)
{
Msg("ERROR: could not open segments file in directory");
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
result.cantOpenSegments = true;
return(result);
}
int format = 0;
try
{
format = input.ReadInt();
}
catch (System.Exception t)
{
Msg("ERROR: could not read segment file version in directory");
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
result.missingSegmentVersion = true;
return(result);
}
finally
{
if (input != null)
{
input.Close();
}
}
System.String sFormat = "";
bool skip = false;
if (format == SegmentInfos.FORMAT)
{
sFormat = "FORMAT [Lucene Pre-2.1]";
}
if (format == SegmentInfos.FORMAT_LOCKLESS)
{
sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
}
else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
{
sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
}
else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
}
else
{
if (format == SegmentInfos.FORMAT_CHECKSUM)
{
sFormat = "FORMAT_CHECKSUM [Lucene 2.4]";
}
else if (format == SegmentInfos.FORMAT_DEL_COUNT)
{
sFormat = "FORMAT_DEL_COUNT [Lucene 2.4]";
}
else if (format == SegmentInfos.FORMAT_HAS_PROX)
{
sFormat = "FORMAT_HAS_PROX [Lucene 2.4]";
}
else if (format == SegmentInfos.FORMAT_USER_DATA)
{
sFormat = "FORMAT_USER_DATA [Lucene 2.9]";
}
else if (format == SegmentInfos.FORMAT_DIAGNOSTICS)
{
sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
}
else if (format < SegmentInfos.CURRENT_FORMAT)
{
sFormat = "int=" + format + " [newer version of Lucene than this tool]";
skip = true;
}
else
{
sFormat = format + " [Lucene 1.3 or prior]";
}
}
result.segmentsFileName = segmentsFileName;
result.numSegments = numSegments;
result.segmentFormat = sFormat;
result.userData = sis.GetUserData();
System.String userDataString;
if (sis.GetUserData().Count > 0)
{
userDataString = " userData=" + SupportClass.CollectionsHelper.CollectionToString(sis.GetUserData());
}
else
{
userDataString = "";
}
Msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString);
if (onlySegments != null)
{
result.partial = true;
if (infoStream != null)
{
infoStream.Write("\nChecking only these segments:");
}
System.Collections.IEnumerator it = onlySegments.GetEnumerator();
while (it.MoveNext())
{
if (infoStream != null)
{
infoStream.Write(" " + it.Current);
}
}
System.Collections.IEnumerator e = onlySegments.GetEnumerator();
while (e.MoveNext() == true)
{
result.segmentsChecked.Add(e.Current);
}
Msg(":");
}
if (skip)
{
Msg("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
result.toolOutOfDate = true;
return(result);
}
result.newSegments = (SegmentInfos)sis.Clone();
result.newSegments.Clear();
for (int i = 0; i < numSegments; i++)
{
SegmentInfo info = sis.Info(i);
if (onlySegments != null && !onlySegments.Contains(info.name))
{
continue;
}
Status.SegmentInfoStatus segInfoStat = new Status.SegmentInfoStatus();
result.segmentInfos.Add(segInfoStat);
Msg(" " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
segInfoStat.name = info.name;
segInfoStat.docCount = info.docCount;
int toLoseDocCount = info.docCount;
SegmentReader reader = null;
try
{
Msg(" compound=" + info.GetUseCompoundFile());
segInfoStat.compound = info.GetUseCompoundFile();
Msg(" hasProx=" + info.GetHasProx());
segInfoStat.hasProx = info.GetHasProx();
Msg(" numFiles=" + info.Files().Count);
segInfoStat.numFiles = info.Files().Count;
Msg(System.String.Format(nf, " size (MB)={0:f}", new System.Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) }));
segInfoStat.sizeMB = info.SizeInBytes() / (1024.0 * 1024.0);
System.Collections.Generic.IDictionary <string, string> diagnostics = info.GetDiagnostics();
segInfoStat.diagnostics = diagnostics;
if (diagnostics.Count > 0)
{
Msg(" diagnostics = " + SupportClass.CollectionsHelper.CollectionToString(diagnostics));
}
int docStoreOffset = info.GetDocStoreOffset();
if (docStoreOffset != -1)
{
Msg(" docStoreOffset=" + docStoreOffset);
segInfoStat.docStoreOffset = docStoreOffset;
Msg(" docStoreSegment=" + info.GetDocStoreSegment());
segInfoStat.docStoreSegment = info.GetDocStoreSegment();
Msg(" docStoreIsCompoundFile=" + info.GetDocStoreIsCompoundFile());
segInfoStat.docStoreCompoundFile = info.GetDocStoreIsCompoundFile();
}
System.String delFileName = info.GetDelFileName();
if (delFileName == null)
{
Msg(" no deletions");
segInfoStat.hasDeletions = false;
}
else
{
Msg(" has deletions [delFileName=" + delFileName + "]");
segInfoStat.hasDeletions = true;
segInfoStat.deletionsFileName = delFileName;
}
if (infoStream != null)
{
infoStream.Write(" test: open reader.........");
}
reader = SegmentReader.Get(info);
segInfoStat.openReaderPassed = true;
int numDocs = reader.NumDocs();
toLoseDocCount = numDocs;
if (reader.HasDeletions())
{
if (reader.deletedDocs.Count() != info.GetDelCount())
{
throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
}
if (reader.deletedDocs.Count() > reader.MaxDoc())
{
throw new System.SystemException("too many deleted docs: maxDoc()=" + reader.MaxDoc() + " vs deletedDocs.count()=" + reader.deletedDocs.Count());
}
if (info.docCount - numDocs != info.GetDelCount())
{
throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs reader=" + (info.docCount - numDocs));
}
segInfoStat.numDeleted = info.docCount - numDocs;
Msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]");
}
else
{
if (info.GetDelCount() != 0)
{
throw new System.SystemException("delete count mismatch: info=" + info.GetDelCount() + " vs reader=" + (info.docCount - numDocs));
}
Msg("OK");
}
if (reader.MaxDoc() != info.docCount)
{
throw new System.SystemException("SegmentReader.maxDoc() " + reader.MaxDoc() + " != SegmentInfos.docCount " + info.docCount);
}
// Test getFieldNames()
if (infoStream != null)
{
infoStream.Write(" test: fields..............");
}
System.Collections.Generic.ICollection <string> fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL);
Msg("OK [" + fieldNames.Count + " fields]");
segInfoStat.numFields = fieldNames.Count;
// Test Field Norms
segInfoStat.fieldNormStatus = TestFieldNorms(fieldNames, reader);
// Test the Term Index
segInfoStat.termIndexStatus = TestTermIndex(info, reader);
// Test Stored Fields
segInfoStat.storedFieldStatus = TestStoredFields(info, reader, nf);
// Test Term Vectors
segInfoStat.termVectorStatus = TestTermVectors(info, reader, nf);
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented properly
if (segInfoStat.fieldNormStatus.error != null)
{
throw new System.SystemException("Field Norm test failed");
}
else if (segInfoStat.termIndexStatus.error != null)
{
throw new System.SystemException("Term Index test failed");
}
else if (segInfoStat.storedFieldStatus.error != null)
{
throw new System.SystemException("Stored Field test failed");
}
else if (segInfoStat.termVectorStatus.error != null)
{
throw new System.SystemException("Term Vector test failed");
}
Msg("");
}
catch (System.Exception t)
{
Msg("FAILED");
System.String comment;
comment = "fixIndex() would remove reference to this segment";
Msg(" WARNING: " + comment + "; full exception:");
if (infoStream != null)
{
infoStream.WriteLine(t.StackTrace);
}
Msg("");
result.totLoseDocCount += toLoseDocCount;
result.numBadSegments++;
continue;
}
finally
{
if (reader != null)
{
reader.Close();
}
}
// Keeper
result.newSegments.Add(info.Clone());
}
if (0 == result.numBadSegments)
{
result.clean = true;
Msg("No problems were detected with this index.\n");
}
else
{
Msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected");
}
return(result);
}
/// <summary> Read a particular segmentFileName. Note that this may
/// throw an IOException if a commit is in process.
///
/// </summary>
/// <param name="directory">-- directory containing the segments file
/// </param>
/// <param name="segmentFileName">-- segment file to load
/// </param>
public void Read(Directory directory, System.String segmentFileName)
{
bool success = false;
IndexInput input = directory.OpenInput(segmentFileName);
if (segmentFileName.Equals(IndexFileNames.SEGMENTS))
{
generation = 0;
}
else
{
#if !PRE_LUCENE_NET_2_0_0_COMPATIBLE
generation = Lucene.Net.Documents.NumberTools.ToLong(segmentFileName.Substring(1 + IndexFileNames.SEGMENTS.Length));
#else
generation = System.Convert.ToInt64(segmentFileName.Substring(1 + IndexFileNames.SEGMENTS.Length), 16);
#endif
}
lastGeneration = generation;
try
{
int format = input.ReadInt();
if (format < 0)
{
// file contains explicit format info
// check that it is a format we can understand
if (format < FORMAT_SINGLE_NORM_FILE)
{
throw new System.IO.IOException("Unknown format version: " + format);
}
version = input.ReadLong(); // read version
counter = input.ReadInt(); // read counter
}
else
{
// file is in old format without explicit format info
counter = format;
}
for (int i = input.ReadInt(); i > 0; i--)
{
// read segmentInfos
Add(new SegmentInfo(directory, format, input));
}
if (format >= 0)
{
// in old format the version number may be at the end of the file
if (input.GetFilePointer() >= input.Length())
{
version = System.DateTime.Now.Millisecond;
}
// old file format without version number
else
{
version = input.ReadLong(); // read version
}
}
success = true;
}
finally
{
input.Close();
if (!success)
{
// Clear any segment infos we had loaded so we
// have a clean slate on retry:
Clear();
}
}
}
public virtual void TestRandomAccessClones()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
IndexInput e1 = cr.OpenInput("f11");
IndexInput e2 = cr.OpenInput("f3");
IndexInput a1 = (IndexInput)e1.Clone();
IndexInput a2 = (IndexInput)e2.Clone();
// Seek the first pair
e1.Seek(100);
a1.Seek(100);
Assert.AreEqual(100, e1.GetFilePointer());
Assert.AreEqual(100, a1.GetFilePointer());
byte be1 = e1.ReadByte();
byte ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now seek the second pair
e2.Seek(1027);
a2.Seek(1027);
Assert.AreEqual(1027, e2.GetFilePointer());
Assert.AreEqual(1027, a2.GetFilePointer());
byte be2 = e2.ReadByte();
byte ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Now make sure the first one didn't move
Assert.AreEqual(101, e1.GetFilePointer());
Assert.AreEqual(101, a1.GetFilePointer());
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now more the first one again, past the buffer length
e1.Seek(1910);
a1.Seek(1910);
Assert.AreEqual(1910, e1.GetFilePointer());
Assert.AreEqual(1910, a1.GetFilePointer());
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now make sure the second set didn't move
Assert.AreEqual(1028, e2.GetFilePointer());
Assert.AreEqual(1028, a2.GetFilePointer());
be2 = e2.ReadByte();
ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Move the second set back, again cross the buffer size
e2.Seek(17);
a2.Seek(17);
Assert.AreEqual(17, e2.GetFilePointer());
Assert.AreEqual(17, a2.GetFilePointer());
be2 = e2.ReadByte();
ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Finally, make sure the first set didn't move
// Now make sure the first one didn't move
Assert.AreEqual(1911, e1.GetFilePointer());
Assert.AreEqual(1911, a1.GetFilePointer());
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
e1.Close();
e2.Close();
a1.Close();
a2.Close();
cr.Close();
}
public static void Main(System.String[] args)
{
System.String filename = null;
bool extract = false;
for (int i = 0; i < args.Length; ++i)
{
if (args[i].Equals("-extract"))
{
extract = true;
}
else if (filename == null)
{
filename = args[i];
}
}
if (filename == null)
{
System.Console.Out.WriteLine("Usage: Lucene.Net.index.IndexReader [-extract] <cfsfile>");
return;
}
Directory dir = null;
CompoundFileReader cfr = null;
try
{
System.IO.FileInfo file = new System.IO.FileInfo(filename);
System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName;
filename = file.Name;
dir = FSDirectory.GetDirectory(dirname, false);
cfr = new CompoundFileReader(dir, filename);
System.String[] files = cfr.List();
System.Array.Sort(files); // sort the array of filename so that the output is more readable
for (int i = 0; i < files.Length; ++i)
{
long len = cfr.FileLength(files[i]);
if (extract)
{
System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory...");
IndexInput ii = cfr.OpenInput(files[i]);
System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create);
// read and write with a small buffer, which is more effectiv than reading byte by byte
byte[] buffer = new byte[1024];
int chunk = buffer.Length;
while (len > 0)
{
int bufLen = (int)System.Math.Min(chunk, len);
ii.ReadBytes(buffer, 0, bufLen);
byte[] byteArray = new byte[buffer.Length];
for (int index = 0; index < buffer.Length; index++)
{
byteArray[index] = (byte)buffer[index];
}
f.Write(byteArray, 0, bufLen);
len -= bufLen;
}
f.Close();
ii.Close();
}
else
{
System.Console.Out.WriteLine(files[i] + ": " + len + " bytes");
}
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
finally
{
try
{
if (dir != null)
{
dir.Close();
}
if (cfr != null)
{
cfr.Close();
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
}
}
public override void Close()
{
delegate_Renamed.Close();
}
public System.Object Run(IndexCommit commit)
{
if (commit != null)
{
if (directory != commit.GetDirectory())
{
throw new System.IO.IOException("the specified commit does not match the specified Directory");
}
return(DoBody(commit.GetSegmentsFileName()));
}
System.String segmentFileName = null;
long lastGen = -1;
long gen = 0;
int genLookaheadCount = 0;
System.IO.IOException exc = null;
bool retry = false;
int method = 0;
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
// We have three methods for determining the current
// generation. We try the first two in parallel, and
// fall back to the third when necessary.
while (true)
{
if (0 == method)
{
// Method 1: list the directory and use the highest
// segments_N file. This method works well as long
// as there is no stale caching on the directory
// contents (NOTE: NFS clients often have such stale
// caching):
System.String[] files = null;
long genA = -1;
files = directory.ListAll();
if (files != null)
{
genA = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files);
}
Lucene.Net.Index.SegmentInfos.Message("directory listing genA=" + genA);
// Method 2: open segments.gen and read its
// contents. Then we take the larger of the two
// gen's. This way, if either approach is hitting
// a stale cache (NFS) we have a better chance of
// getting the right generation.
long genB = -1;
for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++)
{
IndexInput genInput = null;
try
{
genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN);
}
catch (System.IO.FileNotFoundException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: FileNotFoundException " + e);
break;
}
catch (System.IO.IOException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e);
}
if (genInput != null)
{
try
{
int version = genInput.ReadInt();
if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS)
{
long gen0 = genInput.ReadLong();
long gen1 = genInput.ReadLong();
Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1);
if (gen0 == gen1)
{
// The file is consistent.
genB = gen0;
break;
}
}
}
catch (System.IO.IOException err2)
{
// will retry
}
finally
{
genInput.Close();
}
}
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec));
}
catch (System.Threading.ThreadInterruptedException ie)
{
// In 3.0 we will change this to throw
// InterruptedException instead
SupportClass.ThreadClass.Current().Interrupt();
throw new System.SystemException(ie.Message, ie);
}
}
Lucene.Net.Index.SegmentInfos.Message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);
// Pick the larger of the two gen's:
if (genA > genB)
{
gen = genA;
}
else
{
gen = genB;
}
if (gen == -1)
{
// Neither approach found a generation
System.String s;
if (files != null)
{
s = "";
for (int i = 0; i < files.Length; i++)
{
s += (" " + files[i]);
}
}
else
{
s = " null";
}
throw new System.IO.FileNotFoundException("no segments* file found in " + directory + ": files:" + s);
}
}
// Third method (fallback if first & second methods
// are not reliable): since both directory cache and
// file contents cache seem to be stale, just
// advance the generation.
if (1 == method || (0 == method && lastGen == gen && retry))
{
method = 1;
if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount)
{
gen++;
genLookaheadCount++;
Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen);
}
}
if (lastGen == gen)
{
// This means we're about to try the same
// segments_N last tried. This is allowed,
// exactly once, because writer could have been in
// the process of writing segments_N last time.
if (retry)
{
// OK, we've tried the same segments_N file
// twice in a row, so this must be a real
// error. We throw the original exception we
// got.
throw exc;
}
else
{
retry = true;
}
}
else if (0 == method)
{
// Segment file has advanced since our last loop, so
// reset retry:
retry = false;
}
lastGen = gen;
segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try
{
System.Object v = DoBody(segmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
}
return(v);
}
catch (System.IO.IOException err)
{
// Save the original root cause:
if (exc == null)
{
exc = err;
}
Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);
if (!retry && gen > 1)
{
// This is our first time trying this segments
// file (because retry is false), and, there is
// possibly a segments_(N-1) (because gen > 1).
// So, check if the segments_(N-1) exists and
// try it if so:
System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);
bool prevExists;
prevExists = directory.FileExists(prevSegmentFileName);
if (prevExists)
{
Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'");
try
{
System.Object v = DoBody(prevSegmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName);
}
return(v);
}
catch (System.IO.IOException err2)
{
Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
}
}
}
}
}
}
/// <summary> Read a particular segmentFileName. Note that this may
/// throw an IOException if a commit is in process.
///
/// </summary>
/// <param name="directory">-- directory containing the segments file
/// </param>
/// <param name="segmentFileName">-- segment file to load
/// </param>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public void Read(Directory directory, System.String segmentFileName)
{
bool success = false;
// Clear any previous segments:
Clear();
IndexInput input = directory.OpenInput(segmentFileName);
generation = GenerationFromSegmentsFileName(segmentFileName);
lastGeneration = generation;
try
{
int format = input.ReadInt();
if (format < 0)
{
// file contains explicit format info
// check that it is a format we can understand
if (format < CURRENT_FORMAT)
{
throw new CorruptIndexException("Unknown format version: " + format);
}
version = input.ReadLong(); // read version
counter = input.ReadInt(); // read counter
}
else
{
// file is in old format without explicit format info
counter = format;
}
for (int i = input.ReadInt(); i > 0; i--)
{
// read segmentInfos
Add(new SegmentInfo(directory, format, input));
}
if (format >= 0)
{
// in old format the version number may be at the end of the file
if (input.GetFilePointer() >= input.Length())
{
version = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
}
// old file format without version number
else
{
version = input.ReadLong(); // read version
}
}
success = true;
}
finally
{
input.Close();
if (!success)
{
// Clear any segment infos we had loaded so we
// have a clean slate on retry:
Clear();
}
}
}
/// <summary>Closes the enumeration to further activity, freeing resources. </summary>
public override void Close()
{
input.Close();
}
public /*internal*/ void Close()
{
fieldsStream.Close();
indexStream.Close();
}
public System.Object run()
{
System.String segmentFileName = null;
long lastGen = -1;
long gen = 0;
int genLookaheadCount = 0;
System.IO.IOException exc = null;
bool retry = false;
int method = 0;
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
// We have three methods for determining the current
// generation. We try each in sequence.
while (true)
{
// Method 1: list the directory and use the highest
// segments_N file. This method works well as long
// as there is no stale caching on the directory
// contents:
System.String[] files = null;
if (0 == method)
{
if (directory != null)
{
files = directory.List();
}
else
{
files = System.IO.Directory.GetFileSystemEntries(fileDirectory.FullName);
for (int i = 0; i < files.Length; i++)
{
files[i] = System.IO.Path.GetFileName(files[i]);
}
}
gen = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files);
if (gen == -1)
{
System.String s = "";
for (int i = 0; i < files.Length; i++)
{
s += (" " + files[i]);
}
throw new System.IO.FileNotFoundException("no segments* file found: files:" + s);
}
}
// Method 2 (fallback if Method 1 isn't reliable):
// if the directory listing seems to be stale, then
// try loading the "segments.gen" file.
if (1 == method || (0 == method && lastGen == gen && retry))
{
method = 1;
for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++)
{
IndexInput genInput = null;
try
{
genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN);
}
catch (System.IO.IOException e)
{
Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e);
}
if (genInput != null)
{
try
{
int version = genInput.ReadInt();
if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS)
{
long gen0 = genInput.ReadLong();
long gen1 = genInput.ReadLong();
Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1);
if (gen0 == gen1)
{
// The file is consistent.
if (gen0 > gen)
{
Lucene.Net.Index.SegmentInfos.Message("fallback to '" + IndexFileNames.SEGMENTS_GEN + "' check: now try generation " + gen0 + " > " + gen);
gen = gen0;
}
break;
}
}
}
catch (System.IO.IOException err2)
{
// will retry
}
finally
{
genInput.Close();
}
}
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec));
}
catch (System.Threading.ThreadInterruptedException e)
{
// will retry
}
}
}
// Method 3 (fallback if Methods 2 & 3 are not
// reliable): since both directory cache and file
// contents cache seem to be stale, just advance the
// generation.
if (2 == method || (1 == method && lastGen == gen && retry))
{
method = 2;
if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount)
{
gen++;
genLookaheadCount++;
Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen);
}
}
if (lastGen == gen)
{
// This means we're about to try the same
// segments_N last tried. This is allowed,
// exactly once, because writer could have been in
// the process of writing segments_N last time.
if (retry)
{
// OK, we've tried the same segments_N file
// twice in a row, so this must be a real
// error. We throw the original exception we
// got.
throw exc;
}
else
{
retry = true;
}
}
else
{
// Segment file has advanced since our last loop, so
// reset retry:
retry = false;
}
lastGen = gen;
segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try
{
System.Object v = DoBody(segmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
}
return(v);
}
catch (System.IO.IOException err)
{
// Save the original root cause:
if (exc == null)
{
exc = err;
}
Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);
if (!retry && gen > 1)
{
// This is our first time trying this segments
// file (because retry is false), and, there is
// possibly a segments_(N-1) (because gen > 1).
// So, check if the segments_(N-1) exists and
// try it if so:
System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);
if (directory.FileExists(prevSegmentFileName))
{
Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'");
try
{
System.Object v = DoBody(prevSegmentFileName);
if (exc != null)
{
Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName);
}
return(v);
}
catch (System.IO.IOException err2)
{
Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
}
}
}
}
}
}
public override void Close()
{
base.Close();
proxStream.Close();
}
public CompoundFileReader(Directory dir, System.String name)
{
directory = dir;
fileName = name;
bool success = false;
try
{
stream = dir.OpenInput(name);
// read the directory and init files
int count = stream.ReadVInt();
FileEntry entry = null;
for (int i = 0; i < count; i++)
{
long offset = stream.ReadLong();
System.String id = stream.ReadString();
if (entry != null)
{
// set length of the previous entry
entry.length = offset - entry.offset;
}
entry = new FileEntry();
entry.offset = offset;
entries[id] = entry;
}
// set the length of the final entry
if (entry != null)
{
entry.length = stream.Length() - entry.offset;
}
success = true;
}
finally
{
if (!success && (stream != null))
{
try
{
stream.Close();
}
catch (System.IO.IOException)
{
}
}
}
}
protected internal override void DoClose()
{
bool hasReferencedReader = (referencedSegmentReader != null);
termVectorsLocal.Close();
//termVectorsLocal.Clear();
if (hasReferencedReader)
{
referencedSegmentReader.DecRefReaderNotNorms();
referencedSegmentReader = null;
}
deletedDocs = null;
// close the single norms stream
if (singleNormStream != null)
{
// we can close this stream, even if the norms
// are shared, because every reader has it's own
// singleNormStream
singleNormStream.Close();
singleNormStream = null;
}
// re-opened SegmentReaders have their own instance of FieldsReader
if (fieldsReader != null)
{
fieldsReader.Close();
}
if (!hasReferencedReader)
{
// close everything, nothing is shared anymore with other readers
if (tis != null)
{
tis.Close();
}
if (freqStream != null)
{
freqStream.Close();
}
if (proxStream != null)
{
proxStream.Close();
}
if (termVectorsReaderOrig != null)
{
termVectorsReaderOrig.Close();
}
if (cfsReader != null)
{
cfsReader.Close();
}
if (storeCFSReader != null)
{
storeCFSReader.Close();
}
// maybe close directory
base.DoClose();
}
}
