internal override void CloseDocStore(SegmentWriteState state, IState s)
{
lock (this)
{
if (tvx != null)
{
// At least one doc in this run had term vectors
// enabled
Fill(state.numDocsInStore - docWriter.DocStoreOffset);
tvx.Close();
tvf.Close();
tvd.Close();
tvx = null;
System.Diagnostics.Debug.Assert(state.docStoreSegmentName != null);
System.String fileName = state.docStoreSegmentName + "." + IndexFileNames.VECTORS_INDEX_EXTENSION;
if (4 + ((long)state.numDocsInStore) * 16 != state.directory.FileLength(fileName, s))
{
throw new System.SystemException("after flush: tvx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.FileLength(fileName, s) + " length in bytes of " + fileName + " file exists?=" + state.directory.FileExists(fileName, s));
}
state.flushedFiles.Add(state.docStoreSegmentName + "." + IndexFileNames.VECTORS_INDEX_EXTENSION);
state.flushedFiles.Add(state.docStoreSegmentName + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION);
state.flushedFiles.Add(state.docStoreSegmentName + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION);
docWriter.RemoveOpenFile(state.docStoreSegmentName + "." + IndexFileNames.VECTORS_INDEX_EXTENSION);
docWriter.RemoveOpenFile(state.docStoreSegmentName + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION);
docWriter.RemoveOpenFile(state.docStoreSegmentName + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION);
lastDocID = 0;
}
}
}
public void Dispose()
{
// Move to protected method if class becomes unsealed
if (doClose)
{
try
{
if (fieldsStream != null)
{
try
{
fieldsStream.Close();
}
finally
{
fieldsStream = null;
}
}
}
catch (System.IO.IOException)
{
try
{
if (indexStream != null)
{
try
{
indexStream.Close();
}
finally
{
indexStream = null;
}
}
}
catch (System.IO.IOException)
{
// Ignore so we throw only first IOException hit
}
throw;
}
finally
{
if (indexStream != null)
{
try
{
indexStream.Close();
}
finally
{
indexStream = null;
}
}
}
}
}
private RAMDirectory(Directory dir, bool closeDir)
{
System.String[] files = dir.List();
byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE];
for (int i = 0; i < files.Length; i++)
{
// make place on ram disk
IndexOutput os = CreateOutput(System.IO.Path.GetFileName(files[i]));
// read current file
IndexInput is_Renamed = dir.OpenInput(files[i]);
// and copy to ram disk
long len = (int)is_Renamed.Length();
long readCount = 0;
while (readCount < len)
{
int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int)(len - readCount) : BufferedIndexOutput.BUFFER_SIZE;
is_Renamed.ReadBytes(buf, 0, toRead);
os.WriteBytes(buf, toRead);
readCount += toRead;
}
// graceful cleanup
is_Renamed.Close();
os.Close();
}
if (closeDir)
{
dir.Close();
}
}
public virtual void TestNotDirectory()
{
System.IO.FileInfo path = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), "testnotdir"));
Directory fsDir = new SimpleFSDirectory(path, null);
try
{
IndexOutput out_Renamed = fsDir.CreateOutput("afile");
out_Renamed.Close();
Assert.IsTrue(fsDir.FileExists("afile"));
try
{
new SimpleFSDirectory(new System.IO.FileInfo(System.IO.Path.Combine(path.FullName, "afile")), null);
Assert.Fail("did not hit expected exception");
}
catch (NoSuchDirectoryException nsde)
{
// Expected
}
}
finally
{
fsDir.Close();
_TestUtil.RmDir(path);
}
}
public void Dispose()
{
// Move to protected method if class becomes unsealed
if (out_Renamed != null)
{
out_Renamed.Close();
}
}
/// <summary> Copy contents of a directory src to a directory dest.
/// If a file in src already exists in dest then the
/// one in dest will be blindly overwritten.
///
/// <p/><b>NOTE:</b> the source directory cannot change
/// while this method is running. Otherwise the results
/// are undefined and you could easily hit a
/// FileNotFoundException.
///
/// <p/><b>NOTE:</b> this method only copies files that look
/// like index files (ie, have extensions matching the
/// known extensions of index files).
///
/// </summary>
/// <param name="src">source directory
/// </param>
/// <param name="dest">destination directory
/// </param>
/// <param name="closeDirSrc">if <code>true</code>, call {@link #Close()} method on source directory
/// </param>
/// <throws> IOException </throws>
public static void Copy(Directory src, Directory dest, bool closeDirSrc)
{
System.String[] files = src.ListAll();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE];
for (int i = 0; i < files.Length; i++)
{
if (!filter.Accept(null, files[i]))
{
continue;
}
IndexOutput os = null;
IndexInput is_Renamed = null;
try
{
// create file in dest directory
os = dest.CreateOutput(files[i]);
// read current file
is_Renamed = src.OpenInput(files[i]);
// and copy to dest directory
long len = is_Renamed.Length();
long readCount = 0;
while (readCount < len)
{
int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len?(int)(len - readCount):BufferedIndexOutput.BUFFER_SIZE;
is_Renamed.ReadBytes(buf, 0, toRead);
os.WriteBytes(buf, toRead);
readCount += toRead;
}
}
finally
{
// graceful cleanup
try
{
if (os != null)
{
os.Close();
}
}
finally
{
if (is_Renamed != null)
{
is_Renamed.Close();
}
}
}
}
if (closeDirSrc)
{
src.Close();
}
}
private void MergeNorms(IState state)
{
byte[] normBuffer = null;
IndexOutput output = null;
try
{
int numFieldInfos = fieldInfos.Size();
for (int i = 0; i < numFieldInfos; i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.isIndexed && !fi.omitNorms)
{
if (output == null)
{
output = directory.CreateOutput(segment + "." + IndexFileNames.NORMS_EXTENSION, state);
output.WriteBytes(NORMS_HEADER, NORMS_HEADER.Length);
}
foreach (IndexReader reader in readers)
{
int maxDoc = reader.MaxDoc;
if (normBuffer == null || normBuffer.Length < maxDoc)
{
// the buffer is too small for the current segment
normBuffer = new byte[maxDoc];
}
reader.Norms(fi.name, normBuffer, 0, state);
if (!reader.HasDeletions)
{
//optimized case for segments without deleted docs
output.WriteBytes(normBuffer, maxDoc);
}
else
{
// this segment has deleted docs, so we have to
// check for every doc if it is deleted or not
for (int k = 0; k < maxDoc; k++)
{
if (!reader.IsDeleted(k))
{
output.WriteByte(normBuffer[k]);
}
}
}
checkAbort.Work(maxDoc, state);
}
}
}
}
finally
{
if (output != null)
{
output.Close();
}
}
}
/// <summary> Copy contents of a directory src to a directory dest.
/// If a file in src already exists in dest then the
/// one in dest will be blindly overwritten.
///
/// </summary>
/// <param name="src">source directory
/// </param>
/// <param name="dest">destination directory
/// </param>
/// <param name="closeDirSrc">if <code>true</code>, call {@link #close()} method on source directory
/// </param>
/// <throws> IOException </throws>
public static void Copy(Directory src, Directory dest, bool closeDirSrc)
{
System.String[] files = src.List();
if (files == null)
{
throw new System.IO.IOException("cannot read directory " + src + ": list() returned null");
}
byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE];
for (int i = 0; i < files.Length; i++)
{
IndexOutput os = null;
IndexInput is_Renamed = null;
try
{
// create file in dest directory
os = dest.CreateOutput(files[i]);
// read current file
is_Renamed = src.OpenInput(files[i]);
// and copy to dest directory
long len = is_Renamed.Length();
long readCount = 0;
while (readCount < len)
{
int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int)(len - readCount) : BufferedIndexOutput.BUFFER_SIZE;
is_Renamed.ReadBytes(buf, 0, toRead);
os.WriteBytes(buf, toRead);
readCount += toRead;
}
}
finally
{
// graceful cleanup
try
{
if (os != null)
{
os.Close();
}
}
finally
{
if (is_Renamed != null)
{
is_Renamed.Close();
}
}
}
}
if (closeDirSrc)
{
src.Close();
}
}
public void Write(Directory d, System.String name, IState state)
{
IndexOutput output = d.CreateOutput(name, state);
try
{
Write(output);
}
finally
{
output.Close();
}
}
/// <summary>Close all streams. </summary>
public void Dispose()
{
// Move to a protected method if class becomes unsealed
// make an effort to close all streams we can but remember and re-throw
// the first exception encountered in this process
System.IO.IOException keep = null;
if (tvx != null)
{
try
{
tvx.Close();
}
catch (System.IO.IOException e)
{
keep = e;
}
}
if (tvd != null)
{
try
{
tvd.Close();
}
catch (System.IO.IOException e)
{
if (keep == null)
{
keep = e;
}
}
}
if (tvf != null)
{
try
{
tvf.Close();
}
catch (System.IO.IOException e)
{
if (keep == null)
{
keep = e;
}
}
}
if (keep != null)
{
throw new System.IO.IOException(keep.StackTrace);
}
}
public virtual void TestIllegalEOF()
{
RAMDirectory dir = new RAMDirectory();
IndexOutput o = dir.CreateOutput("out");
byte[] b = new byte[1024];
o.WriteBytes(b, 0, 1024);
o.Close();
IndexInput i = dir.OpenInput("out");
i.Seek(1024);
i.Close();
dir.Close();
}
protected override void Dispose(bool disposing)
{
if (isDisposed)
{
return;
}
if (disposing)
{
main.Close();
}
isDisposed = true;
}
public void TestFSDirectorySync()
{
System.IO.FileInfo path = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), "testsync"));
Lucene.Net.Store.Directory directory = new Lucene.Net.Store.SimpleFSDirectory(path, null);
try
{
Lucene.Net.Store.IndexOutput io = directory.CreateOutput("syncfile");
io.Close();
directory.Sync("syncfile");
}
finally
{
directory.Close();
Lucene.Net.Util._TestUtil.RmDir(path);
}
}
/// <summary>Writes this vector to the file <c>name</c> in Directory
/// <c>d</c>, in a format that can be read by the constructor
/// <see cref="BitVector(Directory, String)" />.
/// </summary>
public void Write(Directory d, System.String name, IState state)
{
IndexOutput output = d.CreateOutput(name, state);
try
{
if (IsSparse())
{
WriteDgaps(output); // sparse bit-set more efficiently saved as d-gaps.
}
else
{
WriteBits(output);
}
}
finally
{
output.Close();
}
}
public virtual void TestNotDirectory()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testnotdir"));
Directory fsDir = new SimpleFSDirectory(path, null);
try
{
IndexOutput out_Renamed = fsDir.CreateOutput("afile", null);
out_Renamed.Close();
Assert.IsTrue(fsDir.FileExists("afile", null));
Assert.Throws <NoSuchDirectoryException>(
() =>
new SimpleFSDirectory(new System.IO.DirectoryInfo(System.IO.Path.Combine(path.FullName, "afile")), null),
"did not hit expected exception");
}
finally
{
fsDir.Close();
_TestUtil.RmDir(path);
}
}
public virtual void TestDirectInstantiation()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(AppSettings.Get("tempDir", System.IO.Path.GetTempPath()));
int sz = 2;
Directory[] dirs = new Directory[sz];
dirs[0] = new SimpleFSDirectory(path, null);
// dirs[1] = new NIOFSDirectory(path, null);
System.Console.WriteLine("Skipping NIOFSDirectory() test under Lucene.Net");
dirs[1] = new MMapDirectory(path, null);
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
System.String fname = "foo." + i;
System.String lockname = "foo" + i + ".lck";
IndexOutput out_Renamed = dir.CreateOutput(fname, null);
out_Renamed.WriteByte((byte)i);
out_Renamed.Close();
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
d2.EnsureOpen();
Assert.IsTrue(d2.FileExists(fname, null));
Assert.AreEqual(1, d2.FileLength(fname, null));
// don't test read on MMapDirectory, since it can't really be
// closed and will cause a failure to delete the file.
if (d2 is MMapDirectory)
{
continue;
}
IndexInput input = d2.OpenInput(fname, null);
Assert.AreEqual((byte)i, input.ReadByte(null));
input.Close();
}
// delete with a different dir
dirs[(i + 1) % sz].DeleteFile(fname, null);
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Assert.IsFalse(d2.FileExists(fname, null));
}
Lock lock_Renamed = dir.MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Lock lock2 = d2.MakeLock(lockname);
try
{
Assert.IsFalse(lock2.Obtain(1));
}
catch (LockObtainFailedException)
{
// OK
}
}
lock_Renamed.Release();
// now lock with different dir
lock_Renamed = dirs[(i + 1) % sz].MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
lock_Renamed.Release();
}
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
dir.Close();
Assert.IsFalse(dir.isOpen_ForNUnit);
}
}
/// <summary>Produce _X.nrm if any document had a field with norms
/// not disabled
/// </summary>
public override void Flush(IDictionary <InvertedDocEndConsumerPerThread, ICollection <InvertedDocEndConsumerPerField> > threadsAndFields, SegmentWriteState state, IState s)
{
var byField = new HashMap <FieldInfo, IList <NormsWriterPerField> >();
// Typically, each thread will have encountered the same
// field. So first we collate by field, ie, all
// per-thread field instances that correspond to the
// same FieldInfo
foreach (var entry in threadsAndFields)
{
var fields = entry.Value;
var fieldsIt = fields.GetEnumerator();
var fieldsToRemove = new HashSet <NormsWriterPerField>();
while (fieldsIt.MoveNext())
{
NormsWriterPerField perField = (NormsWriterPerField)fieldsIt.Current;
if (perField.upto > 0)
{
// It has some norms
var l = byField[perField.fieldInfo];
if (l == null)
{
l = new List <NormsWriterPerField>();
byField[perField.fieldInfo] = l;
}
l.Add(perField);
}
// Remove this field since we haven't seen it
// since the previous flush
else
{
fieldsToRemove.Add(perField);
}
}
foreach (var field in fieldsToRemove)
{
fields.Remove(field);
}
}
System.String normsFileName = state.segmentName + "." + IndexFileNames.NORMS_EXTENSION;
state.flushedFiles.Add(normsFileName);
IndexOutput normsOut = state.directory.CreateOutput(normsFileName, s);
try
{
normsOut.WriteBytes(SegmentMerger.NORMS_HEADER, 0, SegmentMerger.NORMS_HEADER.Length);
int numField = fieldInfos.Size();
int normCount = 0;
for (int fieldNumber = 0; fieldNumber < numField; fieldNumber++)
{
FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber);
IList <NormsWriterPerField> toMerge = byField[fieldInfo];
int upto = 0;
if (toMerge != null)
{
int numFields = toMerge.Count;
normCount++;
NormsWriterPerField[] fields = new NormsWriterPerField[numFields];
int[] uptos = new int[numFields];
for (int j = 0; j < numFields; j++)
{
fields[j] = toMerge[j];
}
int numLeft = numFields;
while (numLeft > 0)
{
System.Diagnostics.Debug.Assert(uptos [0] < fields [0].docIDs.Length, " uptos[0]=" + uptos [0] + " len=" + (fields [0].docIDs.Length));
int minLoc = 0;
int minDocID = fields[0].docIDs[uptos[0]];
for (int j = 1; j < numLeft; j++)
{
int docID = fields[j].docIDs[uptos[j]];
if (docID < minDocID)
{
minDocID = docID;
minLoc = j;
}
}
System.Diagnostics.Debug.Assert(minDocID < state.numDocs);
// Fill hole
for (; upto < minDocID; upto++)
{
normsOut.WriteByte(defaultNorm);
}
normsOut.WriteByte(fields[minLoc].norms[uptos[minLoc]]);
(uptos[minLoc])++;
upto++;
if (uptos[minLoc] == fields[minLoc].upto)
{
fields[minLoc].Reset();
if (minLoc != numLeft - 1)
{
fields[minLoc] = fields[numLeft - 1];
uptos[minLoc] = uptos[numLeft - 1];
}
numLeft--;
}
}
// Fill final hole with defaultNorm
for (; upto < state.numDocs; upto++)
{
normsOut.WriteByte(defaultNorm);
}
}
else if (fieldInfo.isIndexed && !fieldInfo.omitNorms)
{
normCount++;
// Fill entire field with default norm:
for (; upto < state.numDocs; upto++)
{
normsOut.WriteByte(defaultNorm);
}
}
System.Diagnostics.Debug.Assert(4 + normCount * state.numDocs == normsOut.FilePointer, ".nrm file size mismatch: expected=" + (4 + normCount * state.numDocs) + " actual=" + normsOut.FilePointer);
}
}
finally
{
normsOut.Close();
}
}
internal void FinishCommit(Directory dir, IState state)
{
if (pendingSegnOutput == null)
{
throw new System.SystemException("prepareCommit was not called");
}
bool success = false;
try
{
pendingSegnOutput.FinishCommit();
pendingSegnOutput.Close();
pendingSegnOutput = null;
success = true;
}
finally
{
if (!success)
{
RollbackCommit(dir, state);
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName, state);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN, state);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
public override void Close()
{
main.Close();
}
/// <summary>Merge files with the extensions added up to now.
/// All files with these extensions are combined sequentially into the
/// compound stream. After successful merge, the source files
/// are deleted.
/// </summary>
/// <throws> IllegalStateException if close() had been called before or </throws>
/// <summary> if no file has been added to this object
/// </summary>
public void Dispose()
{
// Extract into protected method if class ever becomes unsealed
// TODO: Dispose shouldn't throw exceptions!
if (merged)
{
throw new SystemException("Merge already performed");
}
if ((entries.Count == 0))
{
throw new SystemException("No entries to merge have been defined");
}
merged = true;
// open the compound stream
IndexOutput os = null;
try
{
var state = StateHolder.Current.Value;
os = directory.CreateOutput(fileName, state);
// Write the number of entries
os.WriteVInt(entries.Count);
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
long totalSize = 0;
foreach (FileEntry fe in entries)
{
fe.directoryOffset = os.FilePointer;
os.WriteLong(0); // for now
os.WriteString(fe.file);
totalSize += directory.FileLength(fe.file, state);
}
// Pre-allocate size of file as optimization --
// this can potentially help IO performance as
// we write the file and also later during
// searching. It also uncovers a disk-full
// situation earlier and hopefully without
// actually filling disk to 100%:
long finalLength = totalSize + os.FilePointer;
os.SetLength(finalLength);
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
var buffer = new byte[16384];
foreach (FileEntry fe in entries)
{
fe.dataOffset = os.FilePointer;
CopyFile(fe, os, buffer, state);
}
// Write the data offsets into the directory of the compound stream
foreach (FileEntry fe in entries)
{
os.Seek(fe.directoryOffset);
os.WriteLong(fe.dataOffset);
}
System.Diagnostics.Debug.Assert(finalLength == os.Length);
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
IndexOutput tmp = os;
os = null;
tmp.Close();
}
finally
{
if (os != null)
{
try
{
os.Close();
}
catch (System.IO.IOException)
{
}
}
}
}
