internal void DecRef()
{
lock (this)
{
if (ref_Renamed.DecRef() == 0)
{
// close everything, nothing is shared anymore with other readers
if (tis != null)
{
tis.Close();
// null so if an app hangs on to us we still free most ram
tis = null;
}
if (tisNoIndex != null)
{
tisNoIndex.Close();
}
if (freqStream != null)
{
freqStream.Close();
}
if (proxStream != null)
{
proxStream.Close();
}
if (termVectorsReaderOrig != null)
{
termVectorsReaderOrig.Close();
}
if (fieldsReaderOrig != null)
{
fieldsReaderOrig.Close();
}
if (cfsReader != null)
{
cfsReader.Close();
}
if (storeCFSReader != null)
{
storeCFSReader.Close();
}
}
}
}
private void Initialize(SegmentInfo si)
{
segment = si.name;
this.si = si;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
cfsDir = cfsReader;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
// Verify two sources of "maxDoc" agree:
if (fieldsReader.Size() != si.docCount)
{
throw new System.SystemException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos);
// NOTE: the bitvector is stored using the regular directory, not cfs
if (HasDeletions(si))
{
deletedDocs = new BitVector(Directory(), si.GetDelFileName());
// Verify # deletes does not exceed maxDoc for this segment:
if (deletedDocs.Count() > MaxDoc())
{
throw new System.SystemException("number of deletes (" + deletedDocs.Count() + ") exceeds max doc (" + MaxDoc() + ") for segment " + si.name);
}
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq");
proxStream = cfsDir.OpenInput(segment + ".prx");
OpenNorms(cfsDir);
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
internal CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor)
{
segment = si.name;
this.readBufferSize = readBufferSize;
this.dir = dir;
bool success = false;
try
{
Directory dir0 = dir;
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
dir0 = cfsReader;
}
cfsDir = dir0;
fieldInfos = new FieldInfos(cfsDir, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
this.termsIndexDivisor = termsIndexDivisor;
TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor);
if (termsIndexDivisor == - 1)
{
tisNoIndex = reader;
}
else
{
tis = reader;
tisNoIndex = null;
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + "." + IndexFileNames.FREQ_EXTENSION, readBufferSize);
if (fieldInfos.HasProx())
{
proxStream = cfsDir.OpenInput(segment + "." + IndexFileNames.PROX_EXTENSION, readBufferSize);
}
else
{
proxStream = null;
}
success = true;
}
finally
{
if (!success)
{
DecRef();
}
}
}
// NOTE: only called from IndexWriter when a near
// real-time reader is opened, or applyDeletes is run,
// sharing a segment that's still being merged. This
// method is not fully thread safe, and relies on the
// synchronization in IndexWriter
internal void LoadTermsIndex(SegmentInfo si, int termsIndexDivisor)
{
lock (this)
{
if (tis == null)
{
Directory dir0;
if (si.GetUseCompoundFile())
{
// In some cases, we were originally opened when CFS
// was not used, but then we are asked to open the
// terms reader with index, the segment has switched
// to CFS
if (cfsReader == null)
{
cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
}
dir0 = cfsReader;
}
else
{
dir0 = dir;
}
tis = new TermInfosReader(dir0, segment, fieldInfos, readBufferSize, termsIndexDivisor);
}
}
}
private void Initialize(SegmentInfo si)
{
segment = si.name;
this.si = si;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
cfsDir = cfsReader;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
// Verify two sources of "maxDoc" agree:
if (fieldsReader.Size() != si.docCount)
{
throw new System.SystemException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos);
// NOTE: the bitvector is stored using the regular directory, not cfs
if (HasDeletions(si))
{
deletedDocs = new BitVector(Directory(), si.GetDelFileName());
// Verify # deletes does not exceed maxDoc for this segment:
if (deletedDocs.Count() > MaxDoc())
{
throw new System.SystemException("number of deletes (" + deletedDocs.Count() + ") exceeds max doc (" + MaxDoc() + ") for segment " + si.name);
}
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq");
proxStream = cfsDir.OpenInput(segment + ".prx");
OpenNorms(cfsDir);
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
private void Initialize(SegmentInfo si)
{
segment = si.name;
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (Directory().FileExists(segment + ".cfs"))
{
cfsReader = new CompoundFileReader(Directory(), segment + ".cfs");
cfsDir = cfsReader;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos);
tis = new TermInfosReader(cfsDir, segment, fieldInfos);
// NOTE: the bitvector is stored using the regular directory, not cfs
if (HasDeletions(si))
deletedDocs = new BitVector(Directory(), segment + ".del");
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq");
proxStream = cfsDir.OpenInput(segment + ".prx");
OpenNorms(cfsDir);
if (fieldInfos.HasVectors())
{
// open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(cfsDir, segment, fieldInfos);
}
}
internal void DecRef()
{
lock (this)
{
if (ref_Renamed.DecRef() == 0)
{
// close everything, nothing is shared anymore with other readers
if (tis != null)
{
tis.Close();
// null so if an app hangs on to us we still free most ram
tis = null;
}
if (tisNoIndex != null)
{
tisNoIndex.Close();
}
if (freqStream != null)
{
freqStream.Close();
}
if (proxStream != null)
{
proxStream.Close();
}
if (termVectorsReaderOrig != null)
{
termVectorsReaderOrig.Close();
}
if (fieldsReaderOrig != null)
{
fieldsReaderOrig.Close();
}
if (cfsReader != null)
{
cfsReader.Close();
}
if (storeCFSReader != null)
{
storeCFSReader.Close();
}
// Force FieldCache to evict our entries at this point
if (origInstance != null)
{
Lucene.Net.Search.FieldCache_Fields.DEFAULT.Purge(origInstance);
}
}
}
}
internal CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor)
{
segment = si.name;
this.readBufferSize = readBufferSize;
this.dir = dir;
bool success = false;
try
{
Directory dir0 = dir;
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(dir, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
dir0 = cfsReader;
}
cfsDir = dir0;
fieldInfos = new FieldInfos(cfsDir, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
this.termsIndexDivisor = termsIndexDivisor;
TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor);
if (termsIndexDivisor == - 1)
{
tisNoIndex = reader;
}
else
{
tis = reader;
tisNoIndex = null;
}
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + "." + IndexFileNames.FREQ_EXTENSION, readBufferSize);
if (fieldInfos.HasProx())
{
proxStream = cfsDir.OpenInput(segment + "." + IndexFileNames.PROX_EXTENSION, readBufferSize);
}
else
{
proxStream = null;
}
success = true;
}
finally
{
if (!success)
{
DecRef();
}
}
// Must assign this at the end -- if we hit an
// exception above core, we don't want to attempt to
// purge the FieldCache (will hit NPE because core is
// not assigned yet).
this.origInstance = origInstance;
}
// FIXME: OG: remove hard-coded file names
public static void Test()
{
System.IO.FileInfo file = new System.IO.FileInfo("words.txt");
System.Console.Out.WriteLine(" reading word file containing " + file.Length + " bytes");
System.DateTime start = System.DateTime.Now;
System.Collections.ArrayList keys = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10));
System.IO.FileStream ws = new System.IO.FileStream(file.FullName, System.IO.FileMode.Open, System.IO.FileAccess.Read);
System.IO.StreamReader wr = new System.IO.StreamReader(new System.IO.StreamReader(ws, System.Text.Encoding.Default).BaseStream, new System.IO.StreamReader(ws, System.Text.Encoding.Default).CurrentEncoding);
for (System.String key = wr.ReadLine(); key != null; key = wr.ReadLine())
keys.Add(new Term("word", key));
wr.Close();
System.DateTime end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to read " + keys.Count + " words");
start = System.DateTime.Now;
System.Random gen = new System.Random((System.Int32) 1251971);
long fp = (gen.Next() & 0xF) + 1;
long pp = (gen.Next() & 0xF) + 1;
int[] docFreqs = new int[keys.Count];
long[] freqPointers = new long[keys.Count];
long[] proxPointers = new long[keys.Count];
for (int i = 0; i < keys.Count; i++)
{
docFreqs[i] = (gen.Next() & 0xF) + 1;
freqPointers[i] = fp;
proxPointers[i] = pp;
fp += (gen.Next() & 0xF) + 1;
;
pp += (gen.Next() & 0xF) + 1;
;
}
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to generate values");
start = System.DateTime.Now;
Directory store = FSDirectory.GetDirectory("test.store", true);
FieldInfos fis = new FieldInfos();
TermInfosWriter writer = new TermInfosWriter(store, "words", fis);
fis.Add("word", false);
for (int i = 0; i < keys.Count; i++)
writer.Add((Term) keys[i], new TermInfo(docFreqs[i], freqPointers[i], proxPointers[i]));
writer.Close();
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to write table");
System.Console.Out.WriteLine(" table occupies " + store.FileLength("words.tis") + " bytes");
start = System.DateTime.Now;
TermInfosReader reader = new TermInfosReader(store, "words", fis);
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to open table");
start = System.DateTime.Now;
SegmentTermEnum enumerator = reader.Terms();
for (int i = 0; i < keys.Count; i++)
{
enumerator.Next();
Term key = (Term) keys[i];
if (!key.Equals(enumerator.Term()))
{
throw new System.Exception("wrong term: " + enumerator.Term() + ", expected: " + key + " at " + i);
}
TermInfo ti = enumerator.TermInfo();
if (ti.docFreq != docFreqs[i])
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.docFreq, 16) + ", expected: " + System.Convert.ToString(docFreqs[i], 16) + " at " + i);
if (ti.freqPointer != freqPointers[i])
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.freqPointer, 16) + ", expected: " + System.Convert.ToString(freqPointers[i], 16) + " at " + i);
if (ti.proxPointer != proxPointers[i])
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.proxPointer, 16) + ", expected: " + System.Convert.ToString(proxPointers[i], 16) + " at " + i);
}
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to iterate over " + keys.Count + " words");
start = System.DateTime.Now;
for (int i = 0; i < keys.Count; i++)
{
Term key = (Term) keys[i];
TermInfo ti = reader.Get(key);
if (ti.docFreq != docFreqs[i])
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.docFreq, 16) + ", expected: " + System.Convert.ToString(docFreqs[i], 16) + " at " + i);
if (ti.freqPointer != freqPointers[i])
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.freqPointer, 16) + ", expected: " + System.Convert.ToString(freqPointers[i], 16) + " at " + i);
if (ti.proxPointer != proxPointers[i])
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.proxPointer, 16) + ", expected: " + System.Convert.ToString(proxPointers[i], 16) + " at " + i);
}
end = System.DateTime.Now;
System.Console.Out.Write((end.Ticks - start.Ticks) / (float) keys.Count);
System.Console.Out.WriteLine(" average milliseconds per lookup");
TermEnum e = reader.Terms(new Term("word", "azz"));
System.Console.Out.WriteLine("Word after azz is " + e.Term().text);
reader.Close();
store.Close();
}
private void Initialize(SegmentInfo si, int readBufferSize, bool doOpenStores)
{
segment = si.name;
this.si = si;
this.readBufferSize = readBufferSize;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
cfsDir = cfsReader;
}
Directory storeDir;
if (doOpenStores)
{
if (si.GetDocStoreOffset() != - 1)
{
if (si.GetDocStoreIsCompoundFile())
{
storeCFSReader = new CompoundFileReader(Directory(), si.GetDocStoreSegment() + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION, readBufferSize);
storeDir = storeCFSReader;
}
else
{
storeDir = Directory();
}
}
else
{
storeDir = cfsDir;
}
}
else
storeDir = null;
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
bool anyProx = false;
int numFields = fieldInfos.Size();
for (int i = 0; !anyProx && i < numFields; i++)
if (!fieldInfos.FieldInfo(i).omitTf)
anyProx = true;
System.String fieldsSegment;
if (si.GetDocStoreOffset() != - 1)
fieldsSegment = si.GetDocStoreSegment();
else
fieldsSegment = segment;
if (doOpenStores)
{
fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
// Verify two sources of "maxDoc" agree:
if (si.GetDocStoreOffset() == - 1 && fieldsReader.Size() != si.docCount)
{
throw new CorruptIndexException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize);
LoadDeletedDocs();
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq", readBufferSize);
if (anyProx)
proxStream = cfsDir.OpenInput(segment + ".prx", readBufferSize);
OpenNorms(cfsDir, readBufferSize);
if (doOpenStores && fieldInfos.HasVectors())
{
// open term vector files only as needed
System.String vectorsSegment;
if (si.GetDocStoreOffset() != - 1)
vectorsSegment = si.GetDocStoreSegment();
else
vectorsSegment = segment;
termVectorsReaderOrig = new TermVectorsReader(storeDir, vectorsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
private void Initialize(SegmentInfo si, int readBufferSize, bool doOpenStores)
{
segment = si.name;
this.si = si;
this.readBufferSize = readBufferSize;
bool success = false;
try
{
// Use compound file directory for some files, if it exists
Directory cfsDir = Directory();
if (si.GetUseCompoundFile())
{
cfsReader = new CompoundFileReader(Directory(), segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION, readBufferSize);
cfsDir = cfsReader;
}
Directory storeDir;
if (doOpenStores)
{
if (si.GetDocStoreOffset() != -1)
{
if (si.GetDocStoreIsCompoundFile())
{
storeCFSReader = new CompoundFileReader(Directory(), si.GetDocStoreSegment() + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION, readBufferSize);
storeDir = storeCFSReader;
}
else
{
storeDir = Directory();
}
}
else
{
storeDir = cfsDir;
}
}
else
{
storeDir = null;
}
// No compound file exists - use the multi-file format
fieldInfos = new FieldInfos(cfsDir, segment + ".fnm");
System.String fieldsSegment;
if (si.GetDocStoreOffset() != -1)
{
fieldsSegment = si.GetDocStoreSegment();
}
else
{
fieldsSegment = segment;
}
if (doOpenStores)
{
fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
// Verify two sources of "maxDoc" agree:
if (si.GetDocStoreOffset() == -1 && fieldsReader.Size() != si.docCount)
{
throw new CorruptIndexException("doc counts differ for segment " + si.name + ": fieldsReader shows " + fieldsReader.Size() + " but segmentInfo shows " + si.docCount);
}
}
tis = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize);
LoadDeletedDocs();
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.OpenInput(segment + ".frq", readBufferSize);
proxStream = cfsDir.OpenInput(segment + ".prx", readBufferSize);
OpenNorms(cfsDir, readBufferSize);
if (doOpenStores && fieldInfos.HasVectors())
{
// open term vector files only as needed
System.String vectorsSegment;
if (si.GetDocStoreOffset() != -1)
{
vectorsSegment = si.GetDocStoreSegment();
}
else
{
vectorsSegment = segment;
}
termVectorsReaderOrig = new TermVectorsReader(storeDir, vectorsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
}
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
DoClose();
}
}
}
