protected internal FixedIntBlockIndexOutput(IndexOutput output, int fixedBlockSize)
{
blockSize = fixedBlockSize;
this.output = output;
output.WriteVInt(blockSize);
buffer = new int[blockSize];
}
public MockSingleIntIndexOutput(Directory dir, string fileName, IOContext context)
{
@out = dir.CreateOutput(fileName, context);
bool success = false;
try
{
CodecUtil.WriteHeader(@out, CODEC, VERSION_CURRENT);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(@out);
}
}
}
public SecureStoreIndexOutput(Directory cache, string cachePath, Func<DataWithMetadata, Task> saveTask)
{
_cache = cache;
_name = cachePath;
_saveTask = saveTask;
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
// create the local cache one we will operate against...
_indexOutput = _cache.CreateOutput(_name);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob)
{
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azureDirectory;
_blobContainer = _azureDirectory.BlobContainer;
_blob = blob;
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
// create the local cache one we will operate against...
_indexOutput = CacheDirectory.CreateOutput(_name);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
/// <summary>Copy the current contents of this buffer to the named output. </summary>
public virtual void WriteTo(IndexOutput out_Renamed)
{
Flush();
long end = file.length;
long pos = 0;
int buffer = 0;
while (pos < end)
{
int length = BUFFER_SIZE;
long nextPos = pos + length;
if (nextPos > end)
{
// at the last buffer
length = (int) (end - pos);
}
out_Renamed.WriteBytes((byte[]) file.GetBuffer(buffer++), length);
pos = nextPos;
}
}
public SyncIndexOutput(SyncDirectory syncDirectory, string name)
{
if (syncDirectory == null) throw new ArgumentNullException(nameof(syncDirectory));
//TODO: _name was null here, is this intended? https://github.com/azure-contrib/AzureDirectory/issues/19
// I have changed this to be correct now
_name = name;
_syncDirectory = syncDirectory;
_fileMutex = SyncMutexManager.GrabMutex(_syncDirectory, _name);
_fileMutex.WaitOne();
try
{
// create the local cache one we will operate against...
_indexOutput = CacheDirectory.CreateOutput(_name);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob)
{
if (azureDirectory == null) throw new ArgumentNullException(nameof(azureDirectory));
_azureDirectory = azureDirectory;
_fileMutex = SyncMutexManager.GrabMutex(_azureDirectory, _name);
_fileMutex.WaitOne();
try
{
_blobContainer = _azureDirectory.BlobContainer;
_blob = blob;
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
// create the local cache one we will operate against...
_indexOutput = CacheDirectory.CreateOutput(_name);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
/// <summary>
/// Write a block of data (<code>For</code> format).
/// </summary>
/// <param name="data"> the data to write </param>
/// <param name="encoded"> a buffer to use to encode data </param>
/// <param name="out"> the destination output </param>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
public void WriteBlock(int[] data, sbyte[] encoded, IndexOutput @out)
{
if (IsAllEqual(data))
{
@out.WriteByte((sbyte)ALL_VALUES_EQUAL);
@out.WriteVInt(data[0]);
return;
}
int numBits = BitsRequired(data);
Debug.Assert(numBits > 0 && numBits <= 32, numBits.ToString());
PackedInts.Encoder encoder = Encoders[numBits];
int iters = Iterations[numBits];
Debug.Assert(iters * encoder.ByteValueCount() >= Lucene41PostingsFormat.BLOCK_SIZE);
int encodedSize = EncodedSizes[numBits];
Debug.Assert(iters * encoder.ByteBlockCount() >= encodedSize);
@out.WriteByte((sbyte)numBits);
encoder.Encode(data, 0, encoded, 0, iters);
@out.WriteBytes(encoded, encodedSize);
}
private IndexOutput GetOutput()
{
lock (this)
{
if (DataOut == null)
{
bool success = false;
try
{
DataOut = Directory.CreateOutput(DataFileName, IOContext.DEFAULT);
CodecUtil.WriteHeader(DataOut, DATA_CODEC, VERSION_CURRENT);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException((IDisposable)DataOut);
}
}
}
return DataOut;
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
try
{
IOUtils.Close(FieldsStream, IndexWriter);
}
finally
{
FieldsStream = null;
IndexWriter = null;
}
}
}
internal CopyThread(IndexInput src, IndexOutput dst)
{
this.Src = src;
this.Dst = dst;
}
internal void FinishCommit(Directory dir, IState state)
{
if (pendingSegnOutput == null)
{
throw new System.SystemException("prepareCommit was not called");
}
bool success = false;
try
{
pendingSegnOutput.FinishCommit();
pendingSegnOutput.Close();
pendingSegnOutput = null;
success = true;
}
finally
{
if (!success)
{
RollbackCommit(dir, state);
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName, state);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN, state);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
protected override void Dispose(bool disposing)
{
_fileMutex.WaitOne();
try
{
string fileName = _name;
// make sure it's all written out
_indexOutput.Flush();
long originalLength = _indexOutput.Length;
_indexOutput.Dispose();
Stream blobStream;
#if COMPRESSBLOBS
// optionally put a compressor around the blob stream
if (_azureDirectory.ShouldCompressFile(_name))
{
// unfortunately, deflate stream doesn't allow seek, and we need a seekable stream
// to pass to the blob storage stuff, so we compress into a memory stream
MemoryStream compressedStream = new MemoryStream();
try
{
IndexInput indexInput = CacheDirectory.OpenInput(fileName);
using (DeflateStream compressor = new DeflateStream(compressedStream, CompressionMode.Compress, true))
{
// compress to compressedOutputStream
byte[] bytes = new byte[indexInput.Length()];
indexInput.ReadBytes(bytes, 0, (int)bytes.Length);
compressor.Write(bytes, 0, (int)bytes.Length);
}
indexInput.Close();
// seek back to beginning of comrpessed stream
compressedStream.Seek(0, SeekOrigin.Begin);
Debug.WriteLine(string.Format("COMPRESSED {0} -> {1} {2}% to {3}",
originalLength,
compressedStream.Length,
((float)compressedStream.Length / (float)originalLength) * 100,
_name));
}
catch
{
// release the compressed stream resources if an error occurs
compressedStream.Dispose();
throw;
}
blobStream = compressedStream;
}
else
#endif
{
blobStream = new StreamInput(CacheDirectory.OpenInput(fileName));
}
try
{
// push the blobStream up to the cloud
_blob.UploadFromStream(blobStream);
// set the metadata with the original index file properties
_blob.Metadata["CachedLength"] = originalLength.ToString();
_blob.Metadata["CachedLastModified"] = CacheDirectory.FileModified(fileName).ToString();
_blob.SetMetadata();
Debug.WriteLine(string.Format("PUT {1} bytes to {0} in cloud", _name, blobStream.Length));
}
finally
{
blobStream.Dispose();
}
#if FULLDEBUG
Debug.WriteLine(string.Format("CLOSED WRITESTREAM {0}", _name));
#endif
// clean up
_indexOutput = null;
_blobContainer = null;
_blob = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public override void Init(IndexOutput termsOut)
{
this.termsOut = termsOut;
CodecUtil.WriteHeader(termsOut, CODEC, VERSION_CURRENT);
termsOut.WriteVInt(pending.Length); // encode maxPositions in header
_wrappedPostingsWriter.Init(termsOut);
}
// TODO what Var-Var codecs exist in practice... and what are there blocksizes like?
// if its less than 128 we should set that as max and use byte?
/// <summary>
/// NOTE: maxBlockSize must be the maximum block size
/// plus the max non-causal lookahead of your codec. EG Simple9
/// requires lookahead=1 because on seeing the Nth value
/// it knows it must now encode the N-1 values before it.
/// </summary>
protected internal VariableIntBlockIndexOutput(IndexOutput output, int maxBlockSize)
{
this.output = output;
this.output.WriteInt(maxBlockSize);
}
public virtual void RemoveIndexOutput(IndexOutput @out, string name)
{
lock (this)
{
OpenFilesForWrite.Remove(name);
RemoveOpenFile(@out, name);
}
}
public BufferedIndexOutputWrapper(MockDirectoryWrapper outerInstance, int bufferSize, IndexOutput io)
: base(bufferSize)
{
this.OuterInstance = outerInstance;
this.Io = io;
}
/// <summary>Merge files with the extensions added up to now.
/// All files with these extensions are combined sequentially into the
/// compound stream. After successful merge, the source files
/// are deleted.
/// </summary>
/// <throws> IllegalStateException if close() had been called before or </throws>
/// <summary> if no file has been added to this object
/// </summary>
public void Dispose()
{
// Extract into protected method if class ever becomes unsealed
// TODO: Dispose shouldn't throw exceptions!
if (merged)
{
throw new SystemException("Merge already performed");
}
if ((entries.Count == 0))
{
throw new SystemException("No entries to merge have been defined");
}
merged = true;
// open the compound stream
IndexOutput os = null;
try
{
var state = StateHolder.Current.Value;
os = directory.CreateOutput(fileName, state);
// Write the number of entries
os.WriteVInt(entries.Count);
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
long totalSize = 0;
foreach (FileEntry fe in entries)
{
fe.directoryOffset = os.FilePointer;
os.WriteLong(0); // for now
os.WriteString(fe.file);
totalSize += directory.FileLength(fe.file, state);
}
// Pre-allocate size of file as optimization --
// this can potentially help IO performance as
// we write the file and also later during
// searching. It also uncovers a disk-full
// situation earlier and hopefully without
// actually filling disk to 100%:
long finalLength = totalSize + os.FilePointer;
os.SetLength(finalLength);
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
var buffer = new byte[16384];
foreach (FileEntry fe in entries)
{
fe.dataOffset = os.FilePointer;
CopyFile(fe, os, buffer, state);
}
// Write the data offsets into the directory of the compound stream
foreach (FileEntry fe in entries)
{
os.Seek(fe.directoryOffset);
os.WriteLong(fe.dataOffset);
}
System.Diagnostics.Debug.Assert(finalLength == os.Length);
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
IndexOutput tmp = os;
os = null;
tmp.Close();
}
finally
{
if (os != null)
{
try
{
os.Close();
}
catch (System.IO.IOException)
{
}
}
}
}
internal DirectCFSIndexOutput(CompoundFileWriter outerInstance, IndexOutput @delegate, FileEntry entry, bool isSeparate)
: base()
{
this.OuterInstance = outerInstance;
this.@delegate = @delegate;
this.Entry = entry;
entry.Offset = Offset = @delegate.FilePointer;
this.IsSeparate = isSeparate;
}
private void WriteEntryTable(ICollection<FileEntry> entries, IndexOutput entryOut)
{
CodecUtil.WriteHeader(entryOut, ENTRY_CODEC, VERSION_CURRENT);
entryOut.WriteVInt(entries.Count);
foreach (FileEntry fe in entries)
{
entryOut.WriteString(IndexFileNames.StripSegmentName(fe.File));
entryOut.WriteLong(fe.Offset);
entryOut.WriteLong(fe.Length);
}
CodecUtil.WriteFooter(entryOut);
}
private void Write(Directory directory)
{
string segmentsFileName = NextSegmentFileName;
// Always advance the generation on write:
if (_generation == -1)
{
_generation = 1;
}
else
{
_generation++;
}
IndexOutput segnOutput = null;
bool success = false;
var upgradedSIFiles = new HashSet<string>();
try
{
segnOutput = directory.CreateOutput(segmentsFileName, IOContext.DEFAULT);
CodecUtil.WriteHeader(segnOutput, "segments", VERSION_48);
segnOutput.WriteLong(Version);
segnOutput.WriteInt(Counter); // write counter
segnOutput.WriteInt(Size()); // write infos
foreach (SegmentCommitInfo siPerCommit in segments)
{
SegmentInfo si = siPerCommit.Info;
segnOutput.WriteString(si.Name);
segnOutput.WriteString(si.Codec.Name);
segnOutput.WriteLong(siPerCommit.DelGen);
int delCount = siPerCommit.DelCount;
if (delCount < 0 || delCount > si.DocCount)
{
throw new InvalidOperationException("cannot write segment: invalid docCount segment=" + si.Name + " docCount=" + si.DocCount + " delCount=" + delCount);
}
segnOutput.WriteInt(delCount);
segnOutput.WriteLong(siPerCommit.FieldInfosGen);
IDictionary<long, ISet<string>> genUpdatesFiles = siPerCommit.UpdatesFiles;
segnOutput.WriteInt(genUpdatesFiles.Count);
foreach (KeyValuePair<long, ISet<string>> e in genUpdatesFiles)
{
segnOutput.WriteLong(e.Key);
segnOutput.WriteStringSet(e.Value);
}
Debug.Assert(si.Dir == directory);
// If this segment is pre-4.x, perform a one-time
// "ugprade" to write the .si file for it:
string version = si.Version;
if (version == null || StringHelper.VersionComparator.Compare(version, "4.0") < 0)
{
if (!SegmentWasUpgraded(directory, si))
{
string markerFileName = IndexFileNames.SegmentFileName(si.Name, "upgraded", Lucene3xSegmentInfoFormat.UPGRADED_SI_EXTENSION);
si.AddFile(markerFileName);
string segmentFileName = Write3xInfo(directory, si, IOContext.DEFAULT);
upgradedSIFiles.Add(segmentFileName);
directory.Sync(/*Collections.singletonList(*/new[] { segmentFileName }/*)*/);
// Write separate marker file indicating upgrade
// is completed. this way, if there is a JVM
// kill/crash, OS crash, power loss, etc. while
// writing the upgraded file, the marker file
// will be missing:
IndexOutput @out = directory.CreateOutput(markerFileName, IOContext.DEFAULT);
try
{
CodecUtil.WriteHeader(@out, SEGMENT_INFO_UPGRADE_CODEC, SEGMENT_INFO_UPGRADE_VERSION);
}
finally
{
@out.Dispose();
}
upgradedSIFiles.Add(markerFileName);
directory.Sync(/*Collections.SingletonList(*/new[] { markerFileName }/*)*/);
}
}
}
segnOutput.WriteStringStringMap(_userData);
PendingSegnOutput = segnOutput;
success = true;
}
finally
{
if (!success)
{
// We hit an exception above; try to close the file
// but suppress any exception:
IOUtils.CloseWhileHandlingException(segnOutput);
foreach (string fileName in upgradedSIFiles)
{
try
{
directory.DeleteFile(fileName);
}
catch (Exception)
{
// Suppress so we keep throwing the original exception
}
}
try
{
// Try not to leave a truncated segments_N file in
// the index:
directory.DeleteFile(segmentsFileName);
}
catch (Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
}
protected override void Dispose(bool disposing)
{
_fileMutex.WaitOne();
try
{
// make sure it's all written out
_indexOutput.Flush();
long originalLength = _indexOutput.Length;
_indexOutput.Dispose();
var blobStream = new StreamInput(_cache.OpenInput(_name));
try
{
var elapsed = _cache.FileModified(_name);
// normalize RAMDirectory and FSDirectory times
if (elapsed > ticks1970)
{
elapsed -= ticks1970;
}
var cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime();
var wrapper = new ReadStreamWrapper(blobStream);
var data = new DataWithMetadata(wrapper, new Metadata
{
ContentLength = originalLength,
LastModified = cachedLastModifiedUTC
});
_saveTask(data).WaitAndWrap();
LeoTrace.WriteLine(string.Format("PUT {1} bytes to {0} in cloud", _name, blobStream.Length));
}
finally
{
blobStream.Dispose();
}
// clean up
_indexOutput = null;
_cache = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
internal CopyThread(IndexInput src, IndexOutput dst)
{
this.Src = src;
this.Dst = dst;
}
/// <summary>
/// Construct an empty output buffer. </summary>
public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput @delegate, string name)
{
this.Dir = dir;
this.Name = name;
this.@delegate = @delegate;
}
public VariableIntBlockIndexOutputAnonymousHelper(IndexOutput output, int baseBlockSize)
: base(output, baseBlockSize)
{
this.baseBlockSize = baseBlockSize;
this.buffer = new int[2 + 2 * baseBlockSize];
}
/// <summary>
/// Writes a codec footer, which records both a checksum
/// algorithm ID and a checksum. this footer can
/// be parsed and validated with
/// <seealso cref="#checkFooter(ChecksumIndexInput) checkFooter()"/>.
/// <p>
/// CodecFooter --> Magic,AlgorithmID,Checksum
/// <ul>
/// <li>Magic --> <seealso cref="DataOutput#writeInt Uint32"/>. this
/// identifies the start of the footer. It is always {@value #FOOTER_MAGIC}.
/// <li>AlgorithmID --> <seealso cref="DataOutput#writeInt Uint32"/>. this
/// indicates the checksum algorithm used. Currently this is always 0,
/// for zlib-crc32.
/// <li>Checksum --> <seealso cref="DataOutput#writeLong Uint32"/>. The
/// actual checksum value for all previous bytes in the stream, including
/// the bytes from Magic and AlgorithmID.
/// </ul>
/// </summary>
/// <param name="out"> Output stream </param>
/// <exception cref="IOException"> If there is an I/O error writing to the underlying medium. </exception>
public static void WriteFooter(IndexOutput @out)
{
@out.WriteInt(FOOTER_MAGIC);
@out.WriteInt(0);
@out.WriteLong(@out.Checksum);
}
public ChecksumIndexOutput(IndexOutput main)
{
this.main = main;
digest = new CRC32();
}
public virtual void TestDirectInstantiation()
{
System.IO.FileInfo path = new System.IO.FileInfo(SupportClass.AppSettings.Get("tempDir", System.IO.Path.GetTempPath()));
int sz = 2;
Directory[] dirs = new Directory[sz];
dirs[0] = new SimpleFSDirectory(path, null);
// dirs[1] = new NIOFSDirectory(path, null);
System.Console.WriteLine("Skipping NIOFSDirectory() test under Lucene.Net");
dirs[1] = new MMapDirectory(path, null);
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
System.String fname = "foo." + i;
System.String lockname = "foo" + i + ".lck";
IndexOutput out_Renamed = dir.CreateOutput(fname);
out_Renamed.WriteByte((byte)i);
out_Renamed.Close();
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
d2.EnsureOpen();
Assert.IsTrue(d2.FileExists(fname));
Assert.AreEqual(1, d2.FileLength(fname));
// don't test read on MMapDirectory, since it can't really be
// closed and will cause a failure to delete the file.
if (d2 is MMapDirectory)
{
continue;
}
IndexInput input = d2.OpenInput(fname);
Assert.AreEqual((byte)i, input.ReadByte());
input.Close();
}
// delete with a different dir
dirs[(i + 1) % sz].DeleteFile(fname);
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Assert.IsFalse(d2.FileExists(fname));
}
Lock lock_Renamed = dir.MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Lock lock2 = d2.MakeLock(lockname);
try
{
Assert.IsFalse(lock2.Obtain(1));
}
catch (LockObtainFailedException e)
{
// OK
}
}
lock_Renamed.Release();
// now lock with different dir
lock_Renamed = dirs[(i + 1) % sz].MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
lock_Renamed.Release();
}
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
dir.Close();
Assert.IsFalse(dir.isOpen_ForNUnit);
}
}
public virtual void TestCopyBytesMem()
{
int num = AtLeast(10);
for (int iter = 0; iter < num; iter++)
{
Directory dir = NewDirectory();
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + " dir=" + dir);
}
// make random file
IndexOutput @out = dir.CreateOutput("test", NewIOContext(Random));
var bytes = new byte[TestUtil.NextInt32(Random, 1, 77777)];
int size = TestUtil.NextInt32(Random, 1, 1777777);
int upto = 0;
int byteUpto = 0;
while (upto < size)
{
bytes[byteUpto++] = Value(upto);
upto++;
if (byteUpto == bytes.Length)
{
@out.WriteBytes(bytes, 0, bytes.Length);
byteUpto = 0;
}
}
@out.WriteBytes(bytes, 0, byteUpto);
Assert.AreEqual(size, @out.GetFilePointer());
@out.Dispose();
Assert.AreEqual(size, dir.FileLength("test"));
// copy from test -> test2
IndexInput @in = dir.OpenInput("test", NewIOContext(Random));
@out = dir.CreateOutput("test2", NewIOContext(Random));
upto = 0;
while (upto < size)
{
if (Random.NextBoolean())
{
@out.WriteByte(@in.ReadByte());
upto++;
}
else
{
int chunk = Math.Min(TestUtil.NextInt32(Random, 1, bytes.Length), size - upto);
@out.CopyBytes(@in, chunk);
upto += chunk;
}
}
Assert.AreEqual(size, upto);
@out.Dispose();
@in.Dispose();
// verify
IndexInput in2 = dir.OpenInput("test2", NewIOContext(Random));
upto = 0;
while (upto < size)
{
if (Random.NextBoolean())
{
var v = in2.ReadByte();
Assert.AreEqual(Value(upto), v);
upto++;
}
else
{
int limit = Math.Min(TestUtil.NextInt32(Random, 1, bytes.Length), size - upto);
in2.ReadBytes(bytes, 0, limit);
for (int byteIdx = 0; byteIdx < limit; byteIdx++)
{
Assert.AreEqual(Value(upto), bytes[byteIdx]);
upto++;
}
}
}
in2.Dispose();
dir.DeleteFile("test");
dir.DeleteFile("test2");
dir.Dispose();
}
}
public override void Close()
{
_fileMutex.WaitOne();
try
{
string fileName = _name;
// make sure it's all written out
_indexOutput.Flush();
long originalLength = _indexOutput.Length();
_indexOutput.Close();
Stream blobStream;
// optionally put a compressor around the blob stream
if (_azureDirectory.ShouldCompressFile(_name))
{
blobStream = CompressStream(fileName, originalLength);
}
else
{
blobStream = new StreamInput(CacheDirectory.OpenInput(fileName));
}
try
{
// push the blobStream up to the cloud
_blob.UploadFromStream(blobStream);
// set the metadata with the original index file properties
_blob.Metadata["CachedLength"] = originalLength.ToString();
_blob.Metadata["CachedLastModified"] = CacheDirectory.FileModified(fileName).ToString();
_blob.SetMetadata();
Debug.WriteLine(string.Format("PUT {1} bytes to {0} in cloud", _name, blobStream.Length));
}
finally
{
blobStream.Dispose();
}
#if FULLDEBUG
Debug.WriteLine(string.Format("CLOSED WRITESTREAM {0}", _name));
#endif
// clean up
_indexOutput = null;
_blobContainer = null;
_blob = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
private int NumBufferedDocs; // docBase + numBufferedDocs == current doc ID
/// <summary>
/// Sole constructor. </summary>
public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, string segmentSuffix, IOContext context, string formatName, CompressionMode compressionMode, int chunkSize)
{
Debug.Assert(directory != null);
this.Directory = directory;
this.Segment = si.Name;
this.SegmentSuffix = segmentSuffix;
this.CompressionMode = compressionMode;
this.Compressor = compressionMode.NewCompressor();
this.ChunkSize = chunkSize;
this.DocBase = 0;
this.BufferedDocs = new GrowableByteArrayDataOutput(chunkSize);
this.NumStoredFields = new int[16];
this.EndOffsets = new int[16];
this.NumBufferedDocs = 0;
bool success = false;
IndexOutput indexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(Segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION), context);
try
{
FieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(Segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context);
string codecNameIdx = formatName + CODEC_SFX_IDX;
string codecNameDat = formatName + CODEC_SFX_DAT;
CodecUtil.WriteHeader(indexStream, codecNameIdx, VERSION_CURRENT);
CodecUtil.WriteHeader(FieldsStream, codecNameDat, VERSION_CURRENT);
Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == FieldsStream.FilePointer);
Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.FilePointer);
IndexWriter = new CompressingStoredFieldsIndexWriter(indexStream);
indexStream = null;
FieldsStream.WriteVInt(chunkSize);
FieldsStream.WriteVInt(PackedInts.VERSION_CURRENT);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(indexStream);
Abort();
}
}
}
internal void RollbackCommit(Directory dir)
{
if (PendingSegnOutput != null)
{
// Suppress so we keep throwing the original exception
// in our caller
IOUtils.CloseWhileHandlingException(PendingSegnOutput);
PendingSegnOutput = null;
// Must carefully compute fileName from "generation"
// since lastGeneration isn't incremented:
string segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", _generation);
// Suppress so we keep throwing the original exception
// in our caller
IOUtils.DeleteFilesIgnoringExceptions(dir, segmentFileName);
}
}
public StreamOutput(IndexOutput output)
{
Output = output;
}
internal void FinishCommit(Directory dir)
{
if (PendingSegnOutput == null)
{
throw new InvalidOperationException("prepareCommit was not called");
}
bool success = false;
try
{
CodecUtil.WriteFooter(PendingSegnOutput);
success = true;
}
finally
{
if (!success)
{
// Closes pendingSegnOutput & deletes partial segments_N:
RollbackCommit(dir);
}
else
{
success = false;
try
{
PendingSegnOutput.Dispose();
success = true;
}
finally
{
if (!success)
{
// Closes pendingSegnOutput & deletes partial segments_N:
RollbackCommit(dir);
}
else
{
PendingSegnOutput = null;
}
}
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
var fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", _generation);
success = false;
try
{
dir.Sync(Collections.Singleton(fileName));
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName);
}
catch (Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
_lastGeneration = _generation;
WriteSegmentsGen(dir, _generation);
}
public virtual void TestDirectInstantiation()
{
DirectoryInfo path = CreateTempDir("testDirectInstantiation");
byte[] largeBuffer = new byte[Random.Next(256 * 1024)], largeReadBuffer = new byte[largeBuffer.Length];
for (int i = 0; i < largeBuffer.Length; i++)
{
largeBuffer[i] = (byte)i; // automatically loops with modulo
}
var dirs = new FSDirectory[] { new SimpleFSDirectory(path, null), new NIOFSDirectory(path, null), new MMapDirectory(path, null) };
for (int i = 0; i < dirs.Length; i++)
{
FSDirectory dir = dirs[i];
dir.EnsureOpen();
string fname = "foo." + i;
string lockname = "foo" + i + ".lck";
IndexOutput @out = dir.CreateOutput(fname, NewIOContext(Random));
@out.WriteByte((byte)i);
@out.WriteBytes(largeBuffer, largeBuffer.Length);
@out.Dispose();
for (int j = 0; j < dirs.Length; j++)
{
FSDirectory d2 = dirs[j];
d2.EnsureOpen();
Assert.IsTrue(SlowFileExists(d2, fname));
Assert.AreEqual(1 + largeBuffer.Length, d2.FileLength(fname));
// LUCENENET specific - unmap hack not needed
//// don't do read tests if unmapping is not supported!
//if (d2 is MMapDirectory && !((MMapDirectory)d2).UseUnmap)
//{
// continue;
//}
IndexInput input = d2.OpenInput(fname, NewIOContext(Random));
Assert.AreEqual((byte)i, input.ReadByte());
// read array with buffering enabled
Arrays.Fill(largeReadBuffer, (byte)0);
input.ReadBytes(largeReadBuffer, 0, largeReadBuffer.Length, true);
Assert.AreEqual(largeBuffer, largeReadBuffer);
// read again without using buffer
input.Seek(1L);
Arrays.Fill(largeReadBuffer, (byte)0);
input.ReadBytes(largeReadBuffer, 0, largeReadBuffer.Length, false);
Assert.AreEqual(largeBuffer, largeReadBuffer);
input.Dispose();
}
// delete with a different dir
dirs[(i + 1) % dirs.Length].DeleteFile(fname);
for (int j = 0; j < dirs.Length; j++)
{
FSDirectory d2 = dirs[j];
Assert.IsFalse(SlowFileExists(d2, fname));
}
Lock @lock = dir.MakeLock(lockname);
Assert.IsTrue(@lock.Obtain());
for (int j = 0; j < dirs.Length; j++)
{
FSDirectory d2 = dirs[j];
Lock lock2 = d2.MakeLock(lockname);
try
{
Assert.IsFalse(lock2.Obtain(1));
}
#pragma warning disable 168
catch (LockObtainFailedException e)
#pragma warning restore 168
{
// OK
}
}
@lock.Dispose();
// now lock with different dir
@lock = dirs[(i + 1) % dirs.Length].MakeLock(lockname);
Assert.IsTrue(@lock.Obtain());
@lock.Dispose();
}
for (int i = 0; i < dirs.Length; i++)
{
FSDirectory dir = dirs[i];
dir.EnsureOpen();
dir.Dispose();
Assert.IsFalse(dir.IsOpen);
}
}
/// <summary>
/// Copy the contents of the file with specified extension into the provided
/// output stream.
/// </summary>
private long CopyFileEntry(IndexOutput dataOut, FileEntry fileEntry)
{
IndexInput @is = fileEntry.Dir.OpenInput(fileEntry.File, IOContext.READONCE);
bool success = false;
try
{
long startPtr = dataOut.FilePointer;
long length = fileEntry.Length;
dataOut.CopyBytes(@is, length);
// Verify that the output length diff is equal to original file
long endPtr = dataOut.FilePointer;
long diff = endPtr - startPtr;
if (diff != length)
{
throw new System.IO.IOException("Difference in the output file offsets " + diff + " does not match the original file length " + length);
}
fileEntry.Offset = startPtr;
success = true;
return length;
}
finally
{
if (success)
{
IOUtils.Close(@is);
// copy successful - delete file
fileEntry.Dir.DeleteFile(fileEntry.File);
}
else
{
IOUtils.CloseWhileHandlingException(@is);
}
}
}
