public virtual void TestTwoFiles()
{
CreateSequenceFile(dir, "d1", (byte)0, 15);
CreateSequenceFile(dir, "d2", (byte)0, 114);
CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
csw.AddFile("d1");
csw.AddFile("d2");
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
IndexInput expected = dir.OpenInput("d1");
IndexInput actual = csr.OpenInput("d1");
AssertSameStreams("d1", expected, actual);
AssertSameSeekBehavior("d1", expected, actual);
expected.Close();
actual.Close();
expected = dir.OpenInput("d2");
actual = csr.OpenInput("d2");
AssertSameStreams("d2", expected, actual);
AssertSameSeekBehavior("d2", expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
private void Initialize(SegmentInfo si)
{
segment = si.name;
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (Directory().FileExists(segment + ".cfs"))
{
cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
cfsDir = cfsReader;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
tis = new TermInfosReader(cfsDir, segment, fieldInfos);
// NOTE: the bitvector is stored using the regular directory, not cfs
if (HasDeletions(si))
deletedDocs = new BitVector(Directory(), segment + ".del");
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq");
proxStream = cfsDir.OpenInput(segment + ".prx");
OpenNorms(cfsDir);
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
}
}
public virtual void TestFileNotFound()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp", null);
// Open two files
Assert.Throws <System.IO.IOException>(() => cr.OpenInput("bogus", null), "File not found");
cr.Close();
}
public virtual void TestClonedStreamsClosing()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// basic clone
IndexInput expected = dir.OpenInput("f11");
// this test only works for FSIndexInput
Assert.IsTrue(_TestHelper.IsSimpleFSIndexInput(expected));
Assert.IsTrue(_TestHelper.IsSimpleFSIndexInputOpen(expected));
IndexInput one = cr.OpenInput("f11");
Assert.IsTrue(IsCSIndexInputOpen(one));
IndexInput two = (IndexInput)one.Clone();
Assert.IsTrue(IsCSIndexInputOpen(two));
AssertSameStreams("basic clone one", expected, one);
expected.Seek(0);
AssertSameStreams("basic clone two", expected, two);
// Now close the first stream
one.Close();
Assert.IsTrue(IsCSIndexInputOpen(one), "Only close when cr is closed");
// The following should really fail since we couldn't expect to
// access a file once close has been called on it (regardless of
// buffering and/or clone magic)
expected.Seek(0);
two.Seek(0);
AssertSameStreams("basic clone two/2", expected, two);
// Now close the compound reader
cr.Close();
Assert.IsFalse(IsCSIndexInputOpen(one), "Now closed one");
Assert.IsFalse(IsCSIndexInputOpen(two), "Now closed two");
// The following may also fail since the compound stream is closed
expected.Seek(0);
two.Seek(0);
//assertSameStreams("basic clone two/3", expected, two);
// Now close the second clone
two.Close();
expected.Seek(0);
two.Seek(0);
//assertSameStreams("basic clone two/4", expected, two);
expected.Close();
}
public virtual void TestRandomFiles()
{
// Setup the test segment
System.String segment = "test";
int chunk = 1024; // internal buffer size used by the stream
CreateRandomFile(dir, segment + ".zero", 0);
CreateRandomFile(dir, segment + ".one", 1);
CreateRandomFile(dir, segment + ".ten", 10);
CreateRandomFile(dir, segment + ".hundred", 100);
CreateRandomFile(dir, segment + ".big1", chunk);
CreateRandomFile(dir, segment + ".big2", chunk - 1);
CreateRandomFile(dir, segment + ".big3", chunk + 1);
CreateRandomFile(dir, segment + ".big4", 3 * chunk);
CreateRandomFile(dir, segment + ".big5", 3 * chunk - 1);
CreateRandomFile(dir, segment + ".big6", 3 * chunk + 1);
CreateRandomFile(dir, segment + ".big7", 1000 * chunk);
// Setup extraneous files
CreateRandomFile(dir, "onetwothree", 100);
CreateRandomFile(dir, segment + ".notIn", 50);
CreateRandomFile(dir, segment + ".notIn2", 51);
// Now test
CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
System.String[] data = new System.String[] { ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3", ".big4", ".big5", ".big6", ".big7" };
for (int i = 0; i < data.Length; i++)
{
csw.AddFile(segment + data[i]);
}
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
for (int i = 0; i < data.Length; i++)
{
IndexInput check = dir.OpenInput(segment + data[i]);
IndexInput test = csr.OpenInput(segment + data[i]);
AssertSameStreams(data[i], check, test);
AssertSameSeekBehavior(data[i], check, test);
test.Close();
check.Close();
}
csr.Close();
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp", null);
IndexInput is_Renamed = cr.OpenInput("f2", null);
is_Renamed.Seek(is_Renamed.Length(null) - 10, null);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10, null);
Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadByte(null), "Single byte read past end of file");
is_Renamed.Seek(is_Renamed.Length(null) - 10, null);
Assert.Throws <System.IO.IOException>(() => is_Renamed.ReadBytes(b, 0, 50, null), "Block read past end of file");
is_Renamed.Close();
cr.Close();
}
public virtual void TestFileNotFound()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
try
{
IndexInput e1 = cr.OpenInput("bogus");
Assert.Fail("File not found");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: File Not Found: " + e);
}
cr.Close();
}
public virtual void TestSingleFile()
{
int[] data = new int[] { 0, 1, 10, 100 };
for (int i = 0; i < data.Length; i++)
{
System.String name = "t" + data[i];
CreateSequenceFile(dir, name, (byte)0, data[i]);
CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
csw.AddFile(name);
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
IndexInput expected = dir.OpenInput(name);
IndexInput actual = csr.OpenInput(name);
AssertSameStreams(name, expected, actual);
AssertSameSeekBehavior(name, expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
IndexInput is_Renamed = cr.OpenInput("f2");
is_Renamed.Seek(is_Renamed.Length() - 10);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10);
try
{
byte test = is_Renamed.ReadByte();
Assert.Fail("Single byte read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: single byte read past end of file: " + e);
}
is_Renamed.Seek(is_Renamed.Length() - 10);
try
{
is_Renamed.ReadBytes(b, 0, 50);
Assert.Fail("Block read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: block read past end of file: " + e);
}
is_Renamed.Close();
cr.Close();
}
public static void Main(System.String[] args)
{
System.String filename = null;
bool extract = false;
for (int i = 0; i < args.Length; ++i)
{
if (args[i].Equals("-extract"))
{
extract = true;
}
else if (filename == null)
{
filename = args[i];
}
}
if (filename == null)
{
System.Console.Out.WriteLine("Usage: Lucene.Net.index.IndexReader [-extract] <cfsfile>");
return;
}
Directory dir = null;
CompoundFileReader cfr = null;
try
{
System.IO.FileInfo file = new System.IO.FileInfo(filename);
System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName;
filename = file.Name;
dir = FSDirectory.GetDirectory(dirname, false);
cfr = new CompoundFileReader(dir, filename);
System.String[] files = cfr.List();
System.Array.Sort(files); // sort the array of filename so that the output is more readable
for (int i = 0; i < files.Length; ++i)
{
long len = cfr.FileLength(files[i]);
if (extract)
{
System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory...");
IndexInput ii = cfr.OpenInput(files[i]);
System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create);
// read and write with a small buffer, which is more effectiv than reading byte by byte
byte[] buffer = new byte[1024];
int chunk = buffer.Length;
while (len > 0)
{
int bufLen = (int)System.Math.Min(chunk, len);
ii.ReadBytes(buffer, 0, bufLen);
byte[] byteArray = new byte[buffer.Length];
for (int index = 0; index < buffer.Length; index++)
{
byteArray[index] = (byte)buffer[index];
}
f.Write(byteArray, 0, bufLen);
len -= bufLen;
}
f.Close();
ii.Close();
}
else
{
System.Console.Out.WriteLine(files[i] + ": " + len + " bytes");
}
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
finally
{
try
{
if (dir != null)
{
dir.Close();
}
if (cfr != null)
{
cfr.Close();
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
}
}
internal CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor)
{
segment = si.name;
this.readBufferSize = readBufferSize;
this.dir = dir;
bool success = false;
try
{
Directory dir0 = dir;
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
dir0 = cfsReader;
}
cfsDir = dir0;
fieldInfos = new FieldInfos(cfsDir, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
this.termsIndexDivisor = termsIndexDivisor;
TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor);
if (termsIndexDivisor == - 1)
{
tisNoIndex = reader;
}
else
{
tis = reader;
tisNoIndex = null;
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + "." + IndexFileNames.FREQ_EXTENSION, readBufferSize);
if (fieldInfos.HasProx())
{
proxStream = cfsDir.OpenInput(segment + "." + IndexFileNames.PROX_EXTENSION, readBufferSize);
}
else
{
proxStream = null;
}
success = true;
}
finally
{
if (!success)
{
DecRef();
}
}
// Must assign this at the end -- if we hit an
// exception above core, we don't want to attempt to
// purge the FieldCache (will hit NPE because core is
// not assigned yet).
this.origInstance = origInstance;
}
private void Initialize(SegmentInfo si)
{
segment = si.name;
this.si = si;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
cfsDir = cfsReader;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
// Verify two sources of "maxDoc" agree:
if (fieldsReader.Size() != si.docCount)
{
throw new System.SystemException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos);
// NOTE: the bitvector is stored using the regular directory, not cfs
if (HasDeletions(si))
{
deletedDocs = new BitVector(Directory(), si.GetDelFileName());
// Verify # deletes does not exceed maxDoc for this segment:
if (deletedDocs.Count() > MaxDoc())
{
throw new System.SystemException("number of deletes (" + deletedDocs.Count() + ") exceeds max doc (" + MaxDoc() + ") for segment " + si.name);
}
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq");
proxStream = cfsDir.OpenInput(segment + ".prx");
OpenNorms(cfsDir);
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
public virtual void TestFileNotFound()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
try
{
IndexInput e1 = cr.OpenInput("bogus");
Assert.Fail("File not found");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: File Not Found: " + e);
}
cr.Close();
}
public static void Main(System.String[] args)
{
System.String filename = null;
bool extract = false;
for (int i = 0; i < args.Length; ++i)
{
if (args[i].Equals("-extract"))
{
extract = true;
}
else if (filename == null)
{
filename = args[i];
}
}
if (filename == null)
{
System.Console.Out.WriteLine("Usage: Lucene.Net.Index.IndexReader [-extract] <cfsfile>");
return ;
}
Directory dir = null;
CompoundFileReader cfr = null;
try
{
System.IO.FileInfo file = new System.IO.FileInfo(filename);
System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName;
filename = file.Name;
dir = FSDirectory.Open(new System.IO.FileInfo(dirname));
cfr = new CompoundFileReader(dir, filename);
System.String[] files = cfr.List();
System.Array.Sort(files); // sort the array of filename so that the output is more readable
for (int i = 0; i < files.Length; ++i)
{
long len = cfr.FileLength(files[i]);
if (extract)
{
System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory...");
IndexInput ii = cfr.OpenInput(files[i]);
System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create);
// read and write with a small buffer, which is more effectiv than reading byte by byte
byte[] buffer = new byte[1024];
int chunk = buffer.Length;
while (len > 0)
{
int bufLen = (int) System.Math.Min(chunk, len);
ii.ReadBytes(buffer, 0, bufLen);
f.Write(buffer, 0, bufLen);
len -= bufLen;
}
f.Close();
ii.Close();
}
else
System.Console.Out.WriteLine(files[i] + ": " + len + " bytes");
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
finally
{
try
{
if (dir != null)
dir.Close();
if (cfr != null)
cfr.Close();
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
}
}
public virtual void TestFileNotFound()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
Assert.Throws<System.IO.IOException>(() => cr.OpenInput("bogus"), "File not found");
cr.Close();
}
public virtual void TestClonedStreamsClosing()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// basic clone
IndexInput expected = dir.OpenInput("f11");
// this test only works for FSIndexInput
Assert.IsTrue(_TestHelper.IsSimpleFSIndexInput(expected));
Assert.IsTrue(_TestHelper.IsSimpleFSIndexInputOpen(expected));
IndexInput one = cr.OpenInput("f11");
Assert.IsTrue(IsCSIndexInputOpen(one));
IndexInput two = (IndexInput) one.Clone();
Assert.IsTrue(IsCSIndexInputOpen(two));
AssertSameStreams("basic clone one", expected, one);
expected.Seek(0);
AssertSameStreams("basic clone two", expected, two);
// Now close the first stream
one.Close();
Assert.IsTrue(IsCSIndexInputOpen(one), "Only close when cr is closed");
// The following should really fail since we couldn't expect to
// access a file once close has been called on it (regardless of
// buffering and/or clone magic)
expected.Seek(0);
two.Seek(0);
AssertSameStreams("basic clone two/2", expected, two);
// Now close the compound reader
cr.Close();
Assert.IsFalse(IsCSIndexInputOpen(one), "Now closed one");
Assert.IsFalse(IsCSIndexInputOpen(two), "Now closed two");
// The following may also fail since the compound stream is closed
expected.Seek(0);
two.Seek(0);
//assertSameStreams("basic clone two/3", expected, two);
// Now close the second clone
two.Close();
expected.Seek(0);
two.Seek(0);
//assertSameStreams("basic clone two/4", expected, two);
expected.Close();
}
public virtual void TestTwoFiles()
{
CreateSequenceFile(dir, "d1", (byte) 0, 15);
CreateSequenceFile(dir, "d2", (byte) 0, 114);
CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
csw.AddFile("d1");
csw.AddFile("d2");
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
IndexInput expected = dir.OpenInput("d1");
IndexInput actual = csr.OpenInput("d1");
AssertSameStreams("d1", expected, actual);
AssertSameSeekBehavior("d1", expected, actual);
expected.Close();
actual.Close();
expected = dir.OpenInput("d2");
actual = csr.OpenInput("d2");
AssertSameStreams("d2", expected, actual);
AssertSameSeekBehavior("d2", expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
public virtual void TestRandomAccessClones()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
IndexInput e1 = cr.OpenInput("f11");
IndexInput e2 = cr.OpenInput("f3");
IndexInput a1 = (IndexInput)e1.Clone();
IndexInput a2 = (IndexInput)e2.Clone();
// Seek the first pair
e1.Seek(100);
a1.Seek(100);
Assert.AreEqual(100, e1.GetFilePointer());
Assert.AreEqual(100, a1.GetFilePointer());
byte be1 = e1.ReadByte();
byte ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now seek the second pair
e2.Seek(1027);
a2.Seek(1027);
Assert.AreEqual(1027, e2.GetFilePointer());
Assert.AreEqual(1027, a2.GetFilePointer());
byte be2 = e2.ReadByte();
byte ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Now make sure the first one didn't move
Assert.AreEqual(101, e1.GetFilePointer());
Assert.AreEqual(101, a1.GetFilePointer());
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now more the first one again, past the buffer length
e1.Seek(1910);
a1.Seek(1910);
Assert.AreEqual(1910, e1.GetFilePointer());
Assert.AreEqual(1910, a1.GetFilePointer());
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now make sure the second set didn't move
Assert.AreEqual(1028, e2.GetFilePointer());
Assert.AreEqual(1028, a2.GetFilePointer());
be2 = e2.ReadByte();
ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Move the second set back, again cross the buffer size
e2.Seek(17);
a2.Seek(17);
Assert.AreEqual(17, e2.GetFilePointer());
Assert.AreEqual(17, a2.GetFilePointer());
be2 = e2.ReadByte();
ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Finally, make sure the first set didn't move
// Now make sure the first one didn't move
Assert.AreEqual(1911, e1.GetFilePointer());
Assert.AreEqual(1911, a1.GetFilePointer());
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
e1.Close();
e2.Close();
a1.Close();
a2.Close();
cr.Close();
}
private void Initialize(SegmentInfo si)
{
segment = si.name;
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (Directory().FileExists(segment + ".cfs"))
{
cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
cfsDir = cfsReader;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
tis = new TermInfosReader(cfsDir, segment, fieldInfos);
// NOTE: the bitvector is stored using the regular directory, not cfs
if (HasDeletions(si))
deletedDocs = new BitVector(Directory(), segment + ".del");
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq");
proxStream = cfsDir.OpenInput(segment + ".prx");
OpenNorms(cfsDir);
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
}
}
public virtual void TestDeleteLeftoverFiles()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
writer.SetMaxBufferedDocs(10);
int i;
for (i = 0; i < 35; i++)
{
AddDoc(writer, i);
}
writer.SetUseCompoundFile(false);
for (; i < 45; i++)
{
AddDoc(writer, i);
}
writer.Close();
// Delete one doc so we get a .del file:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "7");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "didn't delete the right number of documents");
// Set one norm so we get a .s0 file:
reader.SetNorm(21, "content", (float) 1.5);
reader.Close();
// Now, artificially create an extra .del file & extra
// .s0 file:
System.String[] files = dir.List();
/*
for(int i=0;i<files.length;i++) {
System.out.println(i + ": " + files[i]);
}
*/
// The numbering of fields can vary depending on which
// JRE is in use. On some JREs we see content bound to
// field 0; on others, field 1. So, here we have to
// figure out which field number corresponds to
// "content", and then set our expected file names below
// accordingly:
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
int contentFieldIndex = - 1;
for (i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.Name_ForNUnitTest.Equals("content"))
{
contentFieldIndex = i;
break;
}
}
cfsReader.Close();
Assert.IsTrue(contentFieldIndex != - 1, "could not locate the 'content' field number in the _2.cfs segment");
System.String normSuffix = "s" + contentFieldIndex;
// Create a bogus separate norms file for a
// segment/field that actually has a separate norms file
// already:
CopyFile(dir, "_2_1." + normSuffix, "_2_2." + normSuffix);
// Create a bogus separate norms file for a
// segment/field that actually has a separate norms file
// already, using the "not compound file" extension:
CopyFile(dir, "_2_1." + normSuffix, "_2_2.f" + contentFieldIndex);
// Create a bogus separate norms file for a
// segment/field that does not have a separate norms
// file already:
CopyFile(dir, "_2_1." + normSuffix, "_1_1." + normSuffix);
// Create a bogus separate norms file for a
// segment/field that does not have a separate norms
// file already using the "not compound file" extension:
CopyFile(dir, "_2_1." + normSuffix, "_1_1.f" + contentFieldIndex);
// Create a bogus separate del file for a
// segment that already has a separate del file:
CopyFile(dir, "_0_1.del", "_0_2.del");
// Create a bogus separate del file for a
// segment that does not yet have a separate del file:
CopyFile(dir, "_0_1.del", "_1_1.del");
// Create a bogus separate del file for a
// non-existent segment:
CopyFile(dir, "_0_1.del", "_188_1.del");
// Create a bogus segment file:
CopyFile(dir, "_0.cfs", "_188.cfs");
// Create a bogus fnm file when the CFS already exists:
CopyFile(dir, "_0.cfs", "_0.fnm");
// Create a deletable file:
CopyFile(dir, "_0.cfs", "deletable");
// Create some old segments file:
CopyFile(dir, "segments_a", "segments");
CopyFile(dir, "segments_a", "segments_2");
// Create a bogus cfs file shadowing a non-cfs segment:
CopyFile(dir, "_2.cfs", "_3.cfs");
System.String[] filesPre = dir.List();
// Open & close a writer: it should delete the above 4
// files and nothing more:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
writer.Close();
System.String[] files2 = dir.List();
dir.Close();
System.Array.Sort(files);
System.Array.Sort(files2);
if (!ArrayEquals(files, files2))
{
Assert.Fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.Length - files.Length) + " files but only deleted " + (filesPre.Length - files2.Length) + "; expected files:\n " + AsString(files) + "\n actual files:\n " + AsString(files2));
}
}
public virtual void TestExactFileNames()
{
for (int pass = 0; pass < 2; pass++)
{
System.String outputDir = "lucene.backwardscompat0.index";
RmDir(outputDir);
try
{
Directory dir = FSDirectory.Open(new System.IO.FileInfo(FullDir(outputDir)));
bool autoCommit = 0 == pass;
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
writer.SetRAMBufferSizeMB(16.0);
for (int i = 0; i < 35; i++)
{
AddDoc(writer, i);
}
Assert.AreEqual(35, writer.DocCount(), "wrong doc count");
writer.Close();
// Delete one doc so we get a .del file:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "7");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "didn't delete the right number of documents");
// Set one norm so we get a .s0 file:
reader.SetNorm(21, "content", (float) 1.5);
reader.Close();
// The numbering of fields can vary depending on which
// JRE is in use. On some JREs we see content bound to
// field 0; on others, field 1. So, here we have to
// figure out which field number corresponds to
// "content", and then set our expected file names below
// accordingly:
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
int contentFieldIndex = - 1;
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.name_ForNUnit.Equals("content"))
{
contentFieldIndex = i;
break;
}
}
cfsReader.Close();
Assert.IsTrue(contentFieldIndex != - 1, "could not locate the 'content' field number in the _2.cfs segment");
// Now verify file names:
System.String[] expected;
expected = new System.String[]{"_0.cfs", "_0_1.del", "_0_1.s" + contentFieldIndex, "segments_3", "segments.gen"};
System.String[] actual = dir.ListAll();
System.Array.Sort(expected);
System.Array.Sort(actual);
if (!SupportClass.CollectionsHelper.Equals(expected, actual))
{
Assert.Fail("incorrect filenames in index: expected:\n " + AsString(expected) + "\n actual:\n " + AsString(actual));
}
dir.Close();
}
finally
{
RmDir(outputDir);
}
}
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
IndexInput is_Renamed = cr.OpenInput("f2");
is_Renamed.Seek(is_Renamed.Length() - 10);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10);
try
{
byte test = is_Renamed.ReadByte();
Assert.Fail("Single byte read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: single byte read past end of file: " + e);
}
is_Renamed.Seek(is_Renamed.Length() - 10);
try
{
is_Renamed.ReadBytes(b, 0, 50);
Assert.Fail("Block read past end of file");
}
catch (System.IO.IOException e)
{
/* success */
//System.out.println("SUCCESS: block read past end of file: " + e);
}
is_Renamed.Close();
cr.Close();
}
private void Initialize(SegmentInfo si, int readBufferSize, bool doOpenStores)
{
segment = si.name;
this.si = si;
this.readBufferSize = readBufferSize;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
cfsDir = cfsReader;
}
Directory storeDir;
if (doOpenStores)
{
if (si.GetDocStoreOffset() != -1)
{
if (si.GetDocStoreIsCompoundFile())
{
storeCFSReader = new CompoundFileReader(Directory(), si.GetDocStoreSegment() + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION, readBufferSize);
storeDir = storeCFSReader;
}
else
{
storeDir = Directory();
}
}
else
{
storeDir = cfsDir;
}
}
else
{
storeDir = null;
}
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
bool anyProx = false;
int numFields = fieldInfos.Size();
for (int i = 0; !anyProx && i < numFields; i++)
{
if (!fieldInfos.FieldInfo(i).omitTf)
{
anyProx = true;
}
}
System.String fieldsSegment;
if (si.GetDocStoreOffset() != -1)
{
fieldsSegment = si.GetDocStoreSegment();
}
else
{
fieldsSegment = segment;
}
if (doOpenStores)
{
fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
// Verify two sources of "maxDoc" agree:
if (si.GetDocStoreOffset() == -1 && fieldsReader.Size() != si.docCount)
{
throw new CorruptIndexException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize);
LoadDeletedDocs();
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq", readBufferSize);
if (anyProx)
{
proxStream = cfsDir.OpenInput(segment + ".prx", readBufferSize);
}
OpenNorms(cfsDir, readBufferSize);
if (doOpenStores && fieldInfos.HasVectors())
{
// open term vector files only as needed
System.String vectorsSegment;
if (si.GetDocStoreOffset() != -1)
{
vectorsSegment = si.GetDocStoreSegment();
}
else
{
vectorsSegment = segment;
}
termVectorsReaderOrig = new TermVectorsReader(storeDir, vectorsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
public virtual void TestDeleteLeftoverFiles()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
int i;
for (i = 0; i < 35; i++)
{
AddDoc(writer, i);
}
writer.SetUseCompoundFile(false);
for (; i < 45; i++)
{
AddDoc(writer, i);
}
writer.Close();
// Delete one doc so we get a .del file:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "7");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "didn't delete the right number of documents");
// Set one norm so we get a .s0 file:
reader.SetNorm(21, "content", (float)1.5);
reader.Close();
// Now, artificially create an extra .del file & extra
// .s0 file:
System.String[] files = dir.ListAll();
/*
* for(int j=0;j<files.length;j++) {
* System.out.println(j + ": " + files[j]);
* }
*/
// The numbering of fields can vary depending on which
// JRE is in use. On some JREs we see content bound to
// field 0; on others, field 1. So, here we have to
// figure out which field number corresponds to
// "content", and then set our expected file names below
// accordingly:
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
int contentFieldIndex = -1;
for (i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.name_ForNUnit.Equals("content"))
{
contentFieldIndex = i;
break;
}
}
cfsReader.Close();
Assert.IsTrue(contentFieldIndex != -1, "could not locate the 'content' field number in the _2.cfs segment");
System.String normSuffix = "s" + contentFieldIndex;
// Create a bogus separate norms file for a
// segment/field that actually has a separate norms file
// already:
CopyFile(dir, "_2_1." + normSuffix, "_2_2." + normSuffix);
// Create a bogus separate norms file for a
// segment/field that actually has a separate norms file
// already, using the "not compound file" extension:
CopyFile(dir, "_2_1." + normSuffix, "_2_2.f" + contentFieldIndex);
// Create a bogus separate norms file for a
// segment/field that does not have a separate norms
// file already:
CopyFile(dir, "_2_1." + normSuffix, "_1_1." + normSuffix);
// Create a bogus separate norms file for a
// segment/field that does not have a separate norms
// file already using the "not compound file" extension:
CopyFile(dir, "_2_1." + normSuffix, "_1_1.f" + contentFieldIndex);
// Create a bogus separate del file for a
// segment that already has a separate del file:
CopyFile(dir, "_0_1.del", "_0_2.del");
// Create a bogus separate del file for a
// segment that does not yet have a separate del file:
CopyFile(dir, "_0_1.del", "_1_1.del");
// Create a bogus separate del file for a
// non-existent segment:
CopyFile(dir, "_0_1.del", "_188_1.del");
// Create a bogus segment file:
CopyFile(dir, "_0.cfs", "_188.cfs");
// Create a bogus fnm file when the CFS already exists:
CopyFile(dir, "_0.cfs", "_0.fnm");
// Create a deletable file:
CopyFile(dir, "_0.cfs", "deletable");
// Create some old segments file:
CopyFile(dir, "segments_3", "segments");
CopyFile(dir, "segments_3", "segments_2");
// Create a bogus cfs file shadowing a non-cfs segment:
CopyFile(dir, "_2.cfs", "_3.cfs");
System.String[] filesPre = dir.ListAll();
// Open & close a writer: it should delete the above 4
// files and nothing more:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
writer.Close();
System.String[] files2 = dir.ListAll();
dir.Close();
System.Array.Sort(files);
System.Array.Sort(files2);
System.Collections.Hashtable dif = DifFiles(files, files2);
if (!SupportClass.CollectionsHelper.Equals(files, files2))
{
Assert.Fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.Length - files.Length) + " files but only deleted " + (filesPre.Length - files2.Length) + "; expected files:\n " + AsString(files) + "\n actual files:\n " + AsString(files2) + "\ndif: " + SupportClass.CollectionsHelper.CollectionToString(dif));
}
}
public virtual void TestSingleFile()
{
int[] data = new int[]{0, 1, 10, 100};
for (int i = 0; i < data.Length; i++)
{
System.String name = "t" + data[i];
CreateSequenceFile(dir, name, (byte) 0, data[i]);
CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
csw.AddFile(name);
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
IndexInput expected = dir.OpenInput(name);
IndexInput actual = csr.OpenInput(name);
AssertSameStreams(name, expected, actual);
AssertSameSeekBehavior(name, expected, actual);
expected.Close();
actual.Close();
csr.Close();
}
}
internal CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor)
{
segment = si.name;
this.readBufferSize = readBufferSize;
this.dir = dir;
bool success = false;
try
{
Directory dir0 = dir;
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
dir0 = cfsReader;
}
cfsDir = dir0;
fieldInfos = new FieldInfos(cfsDir, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
this.termsIndexDivisor = termsIndexDivisor;
TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor);
if (termsIndexDivisor == - 1)
{
tisNoIndex = reader;
}
else
{
tis = reader;
tisNoIndex = null;
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + "." + IndexFileNames.FREQ_EXTENSION, readBufferSize);
if (fieldInfos.HasProx())
{
proxStream = cfsDir.OpenInput(segment + "." + IndexFileNames.PROX_EXTENSION, readBufferSize);
}
else
{
proxStream = null;
}
success = true;
}
finally
{
if (!success)
{
DecRef();
}
}
}
public virtual void TestRandomFiles()
{
// Setup the test segment
System.String segment = "test";
int chunk = 1024; // internal buffer size used by the stream
CreateRandomFile(dir, segment + ".zero", 0);
CreateRandomFile(dir, segment + ".one", 1);
CreateRandomFile(dir, segment + ".ten", 10);
CreateRandomFile(dir, segment + ".hundred", 100);
CreateRandomFile(dir, segment + ".big1", chunk);
CreateRandomFile(dir, segment + ".big2", chunk - 1);
CreateRandomFile(dir, segment + ".big3", chunk + 1);
CreateRandomFile(dir, segment + ".big4", 3 * chunk);
CreateRandomFile(dir, segment + ".big5", 3 * chunk - 1);
CreateRandomFile(dir, segment + ".big6", 3 * chunk + 1);
CreateRandomFile(dir, segment + ".big7", 1000 * chunk);
// Setup extraneous files
CreateRandomFile(dir, "onetwothree", 100);
CreateRandomFile(dir, segment + ".notIn", 50);
CreateRandomFile(dir, segment + ".notIn2", 51);
// Now test
CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
System.String[] data = new System.String[]{".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3", ".big4", ".big5", ".big6", ".big7"};
for (int i = 0; i < data.Length; i++)
{
csw.AddFile(segment + data[i]);
}
csw.Close();
CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
for (int i = 0; i < data.Length; i++)
{
IndexInput check = dir.OpenInput(segment + data[i]);
IndexInput test = csr.OpenInput(segment + data[i]);
AssertSameStreams(data[i], check, test);
AssertSameSeekBehavior(data[i], check, test);
test.Close();
check.Close();
}
csr.Close();
}
// NOTE: only called from IndexWriter when a near
// real-time reader is opened, or applyDeletes is run,
// sharing a segment that's still being merged. This
// method is not fully thread safe, and relies on the
// synchronization in IndexWriter
internal void LoadTermsIndex(SegmentInfo si, int termsIndexDivisor)
{
lock (this)
{
if (tis == null)
{
Directory dir0;
if (si.GetUseCompoundFile())
{
// In some cases, we were originally opened when CFS
// was not used, but then we are asked to open the
// terms reader with index, the segment has switched
// to CFS
if (cfsReader == null)
{
cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
}
dir0 = cfsReader;
}
else
{
dir0 = dir;
}
tis = new TermInfosReader(dir0, segment, fieldInfos, readBufferSize, termsIndexDivisor);
}
}
}
public virtual void TestRandomAccessClones()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
IndexInput e1 = cr.OpenInput("f11");
IndexInput e2 = cr.OpenInput("f3");
IndexInput a1 = (IndexInput) e1.Clone();
IndexInput a2 = (IndexInput) e2.Clone();
// Seek the first pair
e1.Seek(100);
a1.Seek(100);
Assert.AreEqual(100, e1.FilePointer);
Assert.AreEqual(100, a1.FilePointer);
byte be1 = e1.ReadByte();
byte ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now seek the second pair
e2.Seek(1027);
a2.Seek(1027);
Assert.AreEqual(1027, e2.FilePointer);
Assert.AreEqual(1027, a2.FilePointer);
byte be2 = e2.ReadByte();
byte ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Now make sure the first one didn't move
Assert.AreEqual(101, e1.FilePointer);
Assert.AreEqual(101, a1.FilePointer);
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now more the first one again, past the buffer length
e1.Seek(1910);
a1.Seek(1910);
Assert.AreEqual(1910, e1.FilePointer);
Assert.AreEqual(1910, a1.FilePointer);
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
// Now make sure the second set didn't move
Assert.AreEqual(1028, e2.FilePointer);
Assert.AreEqual(1028, a2.FilePointer);
be2 = e2.ReadByte();
ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Move the second set back, again cross the buffer size
e2.Seek(17);
a2.Seek(17);
Assert.AreEqual(17, e2.FilePointer);
Assert.AreEqual(17, a2.FilePointer);
be2 = e2.ReadByte();
ba2 = a2.ReadByte();
Assert.AreEqual(be2, ba2);
// Finally, make sure the first set didn't move
// Now make sure the first one didn't move
Assert.AreEqual(1911, e1.FilePointer);
Assert.AreEqual(1911, a1.FilePointer);
be1 = e1.ReadByte();
ba1 = a1.ReadByte();
Assert.AreEqual(be1, ba1);
e1.Close();
e2.Close();
a1.Close();
a2.Close();
cr.Close();
}
internal void OpenDocStores(SegmentInfo si)
{
lock (this)
{
System.Diagnostics.Debug.Assert(si.name.Equals(segment));
if (fieldsReaderOrig == null)
{
Directory storeDir;
if (si.GetDocStoreOffset() != - 1)
{
if (si.GetDocStoreIsCompoundFile())
{
System.Diagnostics.Debug.Assert(storeCFSReader == null);
storeCFSReader = new CompoundFileReader(dir, si.GetDocStoreSegment() + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION, readBufferSize);
storeDir = storeCFSReader;
System.Diagnostics.Debug.Assert(storeDir != null);
}
else
{
storeDir = dir;
System.Diagnostics.Debug.Assert(storeDir != null);
}
}
else if (si.GetUseCompoundFile())
{
// In some cases, we were originally opened when CFS
// was not used, but then we are asked to open doc
// stores after the segment has switched to CFS
if (cfsReader == null)
{
cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
}
storeDir = cfsReader;
System.Diagnostics.Debug.Assert(storeDir != null);
}
else
{
storeDir = dir;
System.Diagnostics.Debug.Assert(storeDir != null);
}
System.String storesSegment;
if (si.GetDocStoreOffset() != - 1)
{
storesSegment = si.GetDocStoreSegment();
}
else
{
storesSegment = segment;
}
fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
// Verify two sources of "maxDoc" agree:
if (si.GetDocStoreOffset() == - 1 && fieldsReaderOrig.Size() != si.docCount)
{
throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + fieldsReaderOrig.Size() + " but segmentInfo shows " + si.docCount);
}
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
}
}
}
}
public virtual void TestReadPastEOF()
{
SetUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
IndexInput is_Renamed = cr.OpenInput("f2");
is_Renamed.Seek(is_Renamed.Length() - 10);
byte[] b = new byte[100];
is_Renamed.ReadBytes(b, 0, 10);
Assert.Throws<System.IO.IOException>(() => is_Renamed.ReadByte(), "Single byte read past end of file");
is_Renamed.Seek(is_Renamed.Length() - 10);
Assert.Throws<System.IO.IOException>(() => is_Renamed.ReadBytes(b, 0, 50), "Block read past end of file");
is_Renamed.Close();
cr.Close();
}
public virtual void TestExactFileNames()
{
System.String outputDir = "lucene.backwardscompat0.index";
RmDir(outputDir);
try
{
Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo(FullDir(outputDir)));
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true,
IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetRAMBufferSizeMB(16.0);
for (int i = 0; i < 35; i++)
{
AddDoc(writer, i);
}
Assert.AreEqual(35, writer.MaxDoc(), "wrong doc count");
writer.Close();
// Delete one doc so we get a .del file:
IndexReader reader = IndexReader.Open(dir, false);
Term searchTerm = new Term("id", "7");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "didn't delete the right number of documents");
// Set one norm so we get a .s0 file:
reader.SetNorm(21, "content", (float)1.5);
reader.Close();
// The numbering of fields can vary depending on which
// JRE is in use. On some JREs we see content bound to
// field 0; on others, field 1. So, here we have to
// figure out which field number corresponds to
// "content", and then set our expected file names below
// accordingly:
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
int contentFieldIndex = -1;
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.name_ForNUnit.Equals("content"))
{
contentFieldIndex = i;
break;
}
}
cfsReader.Close();
Assert.IsTrue(contentFieldIndex != -1,
"could not locate the 'content' field number in the _2.cfs segment");
// Now verify file names:
System.String[] expected;
expected = new System.String[]
{ "_0.cfs", "_0_1.del", "_0_1.s" + contentFieldIndex, "segments_3", "segments.gen" };
System.String[] actual = dir.ListAll();
System.Array.Sort(expected);
System.Array.Sort(actual);
if (!CollectionsHelper.Equals(expected, actual))
{
Assert.Fail("incorrect filenames in index: expected:\n " + AsString(expected) +
"\n actual:\n " + AsString(actual));
}
dir.Close();
}
finally
{
RmDir(outputDir);
}
}
private void Initialize(SegmentInfo si)
{
segment = si.name;
this.si = si;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
cfsDir = cfsReader;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
// Verify two sources of "maxDoc" agree:
if (fieldsReader.Size() != si.docCount)
{
throw new System.SystemException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos);
// NOTE: the bitvector is stored using the regular directory, not cfs
if (HasDeletions(si))
{
deletedDocs = new BitVector(Directory(), si.GetDelFileName());
// Verify # deletes does not exceed maxDoc for this segment:
if (deletedDocs.Count() > MaxDoc())
{
throw new System.SystemException("number of deletes (" + deletedDocs.Count() + ") exceeds max doc (" + MaxDoc() + ") for segment " + si.name);
}
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq");
proxStream = cfsDir.OpenInput(segment + ".prx");
OpenNorms(cfsDir);
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
private void Initialize(SegmentInfo si, int readBufferSize, bool doOpenStores)
{
segment = si.name;
this.si = si;
this.readBufferSize = readBufferSize;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
cfsDir = cfsReader;
}
Directory storeDir;
if (doOpenStores)
{
if (si.GetDocStoreOffset() != - 1)
{
if (si.GetDocStoreIsCompoundFile())
{
storeCFSReader = new CompoundFileReader(Directory(), si.GetDocStoreSegment() + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION, readBufferSize);
storeDir = storeCFSReader;
}
else
{
storeDir = Directory();
}
}
else
{
storeDir = cfsDir;
}
}
else
storeDir = null;
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
bool anyProx = false;
int numFields = fieldInfos.Size();
for (int i = 0; !anyProx && i < numFields; i++)
if (!fieldInfos.FieldInfo(i).omitTf)
anyProx = true;
System.String fieldsSegment;
if (si.GetDocStoreOffset() != - 1)
fieldsSegment = si.GetDocStoreSegment();
else
fieldsSegment = segment;
if (doOpenStores)
{
fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
// Verify two sources of "maxDoc" agree:
if (si.GetDocStoreOffset() == - 1 && fieldsReader.Size() != si.docCount)
{
throw new CorruptIndexException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize);
LoadDeletedDocs();
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq", readBufferSize);
if (anyProx)
proxStream = cfsDir.OpenInput(segment + ".prx", readBufferSize);
OpenNorms(cfsDir, readBufferSize);
if (doOpenStores && fieldInfos.HasVectors())
{
// open term vector files only as needed
System.String vectorsSegment;
if (si.GetDocStoreOffset() != - 1)
vectorsSegment = si.GetDocStoreSegment();
else
vectorsSegment = segment;
termVectorsReaderOrig = new TermVectorsReader(storeDir, vectorsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
