internal FieldsWriter(Directory d, System.String segment, FieldInfos fn)
{
fieldInfos = fn;
fieldsStream = d.CreateOutput(segment + ".fdt");
indexStream = d.CreateOutput(segment + ".fdx");
doClose = true;
}
public PreFlexRWStoredFieldsWriter(Directory directory, string segment, IOContext context)
{
Debug.Assert(directory != null);
this.Directory = directory;
this.Segment = segment;
bool success = false;
try
{
FieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION), context);
IndexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION), context);
FieldsStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT);
IndexStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
Abort();
}
}
}
public TermVectorsWriter(Directory directory, System.String segment, FieldInfos fieldInfos)
{
// Open files for TermVector storage
tvx = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION);
tvx.WriteInt(TermVectorsReader.FORMAT_CURRENT);
tvd = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION);
tvd.WriteInt(TermVectorsReader.FORMAT_CURRENT);
tvf = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION);
tvf.WriteInt(TermVectorsReader.FORMAT_CURRENT);
this.fieldInfos = fieldInfos;
}
public TermVectorsWriter(Directory directory, System.String segment, FieldInfos fieldInfos)
{
// Open files for TermVector storage
tvx = directory.CreateOutput(segment + TVX_EXTENSION);
tvx.WriteInt(FORMAT_VERSION);
tvd = directory.CreateOutput(segment + TVD_EXTENSION);
tvd.WriteInt(FORMAT_VERSION);
tvf = directory.CreateOutput(segment + TVF_EXTENSION);
tvf.WriteInt(FORMAT_VERSION);
this.fieldInfos = fieldInfos;
fields = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(fieldInfos.Size()));
terms = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10));
}
/// <summary>
/// Save a single segment's info. </summary>
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
{
string fileName = IndexFileNames.SegmentFileName(si.Name, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
si.AddFile(fileName);
IndexOutput output = dir.CreateOutput(fileName, ioContext);
bool success = false;
try
{
CodecUtil.WriteHeader(output, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_CURRENT);
// Write the Lucene version that created this segment, since 3.1
output.WriteString(si.Version);
output.WriteInt(si.DocCount);
output.WriteByte((byte)(sbyte)(si.UseCompoundFile ? SegmentInfo.YES : SegmentInfo.NO));
output.WriteStringStringMap(si.Diagnostics);
output.WriteStringSet(si.Files);
CodecUtil.WriteFooter(output);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(output);
si.Dir.DeleteFile(fileName);
}
else
{
output.Dispose();
}
}
}
/// <summary>Creates a file of the specified size with random data. </summary>
private void CreateRandomFile(Directory dir, System.String name, int size)
{
IndexOutput os = dir.CreateOutput(name);
for (int i = 0; i < size; i++)
{
byte b = (byte) ((new System.Random().NextDouble()) * 256);
os.WriteByte(b);
}
os.Close();
}
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi)
{
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
output = directory.CreateOutput(segment + (isIndex ? ".tii" : ".tis"));
output.WriteInt(FORMAT); // write format
output.WriteLong(0); // leave space for size
output.WriteInt(indexInterval); // write indexInterval
output.WriteInt(skipInterval); // write skipInterval
}
public PreFlexRWTermVectorsWriter(Directory directory, string segment, IOContext context)
{
this.Directory = directory;
this.Segment = segment;
bool success = false;
try
{
// Open files for TermVector storage
Tvx = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_INDEX_EXTENSION), context);
Tvx.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT);
Tvd = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_DOCUMENTS_EXTENSION), context);
Tvd.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT);
Tvf = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION), context);
Tvf.WriteInt(Lucene3xTermVectorsReader.FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
Abort();
}
}
}
public virtual void CopyFile(Directory dir, string src, string dest)
{
IndexInput @in = dir.OpenInput(src, NewIOContext(Random()));
IndexOutput @out = dir.CreateOutput(dest, NewIOContext(Random()));
var b = new byte[1024];
long remainder = @in.Length();
while (remainder > 0)
{
int len = (int)Math.Min(b.Length, remainder);
@in.ReadBytes(b, 0, len);
@out.WriteBytes(b, len);
remainder -= len;
}
@in.Dispose();
@out.Dispose();
}
public virtual FieldInfos CreateAndWriteFieldInfos(Directory dir, string filename)
{
//Positive test of FieldInfos
Assert.IsTrue(TestDoc != null);
FieldInfos.Builder builder = new FieldInfos.Builder();
foreach (IndexableField field in TestDoc)
{
builder.AddOrUpdate(field.Name(), field.FieldType());
}
FieldInfos fieldInfos = builder.Finish();
//Since the complement is stored as well in the fields map
Assert.IsTrue(fieldInfos.Size() == DocHelper.All.Count); //this is all b/c we are using the no-arg constructor
IndexOutput output = dir.CreateOutput(filename, NewIOContext(Random()));
Assert.IsTrue(output != null);
//Use a RAMOutputStream
FieldInfosWriter writer = Codec.Default.FieldInfosFormat().FieldInfosWriter;
writer.Write(dir, filename, "", fieldInfos, IOContext.DEFAULT);
output.Dispose();
return fieldInfos;
}
private int LastFieldNumber = -1; // only for assert
#endregion Fields
#region Constructors
public PreFlexRWNormsConsumer(Directory directory, string segment, IOContext context)
{
string normsFileName = IndexFileNames.SegmentFileName(segment, "", NORMS_EXTENSION);
bool success = false;
IndexOutput output = null;
try
{
output = directory.CreateOutput(normsFileName, context);
// output.WriteBytes(NORMS_HEADER, 0, NORMS_HEADER.Length);
foreach (var @sbyte in NORMS_HEADER)
{
output.WriteByte((byte)@sbyte);
}
@out = output;
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
internal void FinishCommit(Directory dir)
{
if (pendingSegnOutput == null)
{
throw new System.SystemException("prepareCommit was not called");
}
bool success = false;
try
{
pendingSegnOutput.FinishCommit();
pendingSegnOutput.Close();
pendingSegnOutput = null;
success = true;
}
finally
{
if (!success)
{
RollbackCommit(dir);
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception t)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
public void FinishCommit(Directory dir)
{
if (pendingOutput == null)
throw new System.Exception("prepareCommit was not called");
bool success = false;
try
{
pendingOutput.FinishCommit();
pendingOutput.Close();
pendingOutput = null;
success = true;
}
finally
{
if (!success)
RollbackCommit(dir);
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
public void Write(Directory d, System.String name)
{
IndexOutput output = d.CreateOutput(name);
try
{
Write(output);
}
finally
{
output.Close();
}
}
public CachedIndexInput(ICloudProvider CloudProvider, Directory CacheDirectory, string Name)
{
this.name = Name;
#if FULLDEBUG
Debug.WriteLine("Opening " + this.name);
#endif
this.fileMutex = BlobMutexManager.GrabMutex(this.name);
this.fileMutex.WaitOne();
try {
bool fFileNeeded = false;
FileMetadata cloudMetadata = CloudProvider.FileMetadata(this.name);
if (!cloudMetadata.Exists)
{
fFileNeeded = false;
// TODO: Delete local if it doesn't exist on cloud?
/*
* if (CacheDirectory.FileExists(this.name)) {
* CacheDirectory.DeleteFile(this.name);
* }
*/
}
else if (!CacheDirectory.FileExists(this.name))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(this.name);
long blobLength = cloudMetadata.Length;
DateTime blobLastModifiedUTC = cloudMetadata.LastModified.ToUniversalTime();
if (!cloudMetadata.Exists || cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
DateTime cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(this.name), DateTimeKind.Local).ToUniversalTime();
if (cachedLastModifiedUTC < blobLastModifiedUTC)
{
TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine("Using cache for " + this.name + ": " + timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
using (StreamOutput fileStream = new StreamOutput(CacheDirectory.CreateOutput(this.name))) {
Stream blobStream = CloudProvider.Download(this.name);
blobStream.CopyTo(fileStream);
fileStream.Flush();
Debug.WriteLine("GET {0} RETREIVED {1} bytes", this.name, fileStream.Length);
}
}
else
{
#if FULLDEBUG
if (!cloudMetadata.Exists)
{
Debug.WriteLine("Cloud doesn't have " + this.name);
}
else
{
Debug.WriteLine("Using cached file for " + this.name);
}
#endif
}
// open the file in read only mode
this.indexInput = CacheDirectory.OpenInput(this.name);
} finally {
this.fileMutex.ReleaseMutex();
}
}
/// <summary>
/// Writes this vector to the file <code>name</code> in Directory
/// <code>d</code>, in a format that can be read by the constructor {@link
/// #BitVector(Directory, String, IOContext)}.
/// </summary>
public void Write(Directory d, string name, IOContext context)
{
Debug.Assert(!(d is CompoundFileDirectory));
IndexOutput output = d.CreateOutput(name, context);
try
{
output.WriteInt(-2);
CodecUtil.WriteHeader(output, CODEC, VERSION_CURRENT);
if (Sparse)
{
// sparse bit-set more efficiently saved as d-gaps.
WriteClearedDgaps(output);
}
else
{
WriteBits(output);
}
CodecUtil.WriteFooter(output);
Debug.Assert(VerifyCount());
}
finally
{
IOUtils.Close(output);
}
}
/// <summary>Writes this vector to the file <code>name</code> in Directory
/// <code>d</code>, in a format that can be read by the constructor {@link
/// #BitVector(Directory, String)}.
/// </summary>
public void Write(Directory d, System.String name)
{
IndexOutput output = d.CreateOutput(name);
try
{
if (IsSparse())
{
WriteDgaps(output); // sparse bit-set more efficiently saved as d-gaps.
}
else
{
WriteBits(output);
}
}
finally
{
output.Close();
}
}
public void Write(Directory directory)
{
System.String segmentFileName = GetNextSegmentFileName();
// Always advance the generation on write:
if (generation == - 1)
{
generation = 1;
}
else
{
generation++;
}
IndexOutput output = directory.CreateOutput(segmentFileName);
bool success = false;
try
{
output.WriteInt(CURRENT_FORMAT); // write FORMAT
output.WriteLong(++version); // every write changes
// the index
output.WriteInt(counter); // write counter
output.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
Info(i).Write(output);
}
}
finally
{
try
{
output.Close();
success = true;
}
finally
{
if (!success)
{
// Try not to leave a truncated segments_N file in
// the index:
directory.DeleteFile(segmentFileName);
}
}
}
try
{
output = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
output.WriteInt(FORMAT_LOCKLESS);
output.WriteLong(generation);
output.WriteLong(generation);
}
finally
{
output.Close();
}
}
catch (System.IO.IOException e)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
lastGeneration = generation;
}
private void Initialize(Directory directory, string segment, FieldInfos fis, int interval, bool isi)
{
IndexInterval = interval;
FieldInfos = fis;
IsIndex = isi;
Output = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", (IsIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION : Lucene3xPostingsFormat.TERMS_EXTENSION)), IOContext.DEFAULT);
bool success = false;
try
{
Output.WriteInt(FORMAT_CURRENT); // write format
Output.WriteLong(0); // leave space for size
Output.WriteInt(IndexInterval); // write indexInterval
Output.WriteInt(SkipInterval); // write skipInterval
Output.WriteInt(MaxSkipLevels); // write maxSkipLevels
Debug.Assert(InitUTF16Results());
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(Output);
try
{
directory.DeleteFile(IndexFileNames.SegmentFileName(segment, "", (IsIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION : Lucene3xPostingsFormat.TERMS_EXTENSION)));
}
catch (IOException ignored)
{
}
}
}
}
/// <summary>
/// Sole constructor. </summary>
public Lucene40TermVectorsWriter(Directory directory, string segment, IOContext context)
{
this.Directory = directory;
this.Segment = segment;
bool success = false;
try
{
// Open files for TermVector storage
Tvx = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene40TermVectorsReader.VECTORS_INDEX_EXTENSION), context);
CodecUtil.WriteHeader(Tvx, Lucene40TermVectorsReader.CODEC_NAME_INDEX, Lucene40TermVectorsReader.VERSION_CURRENT);
Tvd = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene40TermVectorsReader.VECTORS_DOCUMENTS_EXTENSION), context);
CodecUtil.WriteHeader(Tvd, Lucene40TermVectorsReader.CODEC_NAME_DOCS, Lucene40TermVectorsReader.VERSION_CURRENT);
Tvf = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene40TermVectorsReader.VECTORS_FIELDS_EXTENSION), context);
CodecUtil.WriteHeader(Tvf, Lucene40TermVectorsReader.CODEC_NAME_FIELDS, Lucene40TermVectorsReader.VERSION_CURRENT);
Debug.Assert(Lucene40TermVectorsReader.HEADER_LENGTH_INDEX == Tvx.FilePointer);
Debug.Assert(Lucene40TermVectorsReader.HEADER_LENGTH_DOCS == Tvd.FilePointer);
Debug.Assert(Lucene40TermVectorsReader.HEADER_LENGTH_FIELDS == Tvf.FilePointer);
success = true;
}
finally
{
if (!success)
{
Abort();
}
}
}
/// <summary>
/// Sole constructor. </summary>
public Lucene40StoredFieldsWriter(Directory directory, string segment, IOContext context)
{
Debug.Assert(directory != null);
this.Directory = directory;
this.Segment = segment;
bool success = false;
try
{
FieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", FIELDS_EXTENSION), context);
IndexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context);
CodecUtil.WriteHeader(FieldsStream, CODEC_NAME_DAT, VERSION_CURRENT);
CodecUtil.WriteHeader(IndexStream, CODEC_NAME_IDX, VERSION_CURRENT);
Debug.Assert(HEADER_LENGTH_DAT == FieldsStream.FilePointer);
Debug.Assert(HEADER_LENGTH_IDX == IndexStream.FilePointer);
success = true;
}
finally
{
if (!success)
{
Abort();
}
}
}
/// <summary>Merge files with the extensions added up to now.
/// All files with these extensions are combined sequentially into the
/// compound stream. After successful merge, the source files
/// are deleted.
/// </summary>
/// <throws> IllegalStateException if close() had been called before or </throws>
/// <summary> if no file has been added to this object
/// </summary>
public void Dispose()
{
// Extract into protected method if class ever becomes unsealed
// TODO: Dispose shouldn't throw exceptions!
if (merged)
{
throw new SystemException("Merge already performed");
}
if ((entries.Count == 0))
{
throw new SystemException("No entries to merge have been defined");
}
merged = true;
// open the compound stream
IndexOutput os = null;
try
{
os = directory.CreateOutput(fileName);
// Write the number of entries
os.WriteVInt(entries.Count);
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
long totalSize = 0;
foreach (FileEntry fe in entries)
{
fe.directoryOffset = os.FilePointer;
os.WriteLong(0); // for now
os.WriteString(fe.file);
totalSize += directory.FileLength(fe.file);
}
// Pre-allocate size of file as optimization --
// this can potentially help IO performance as
// we write the file and also later during
// searching. It also uncovers a disk-full
// situation earlier and hopefully without
// actually filling disk to 100%:
long finalLength = totalSize + os.FilePointer;
os.SetLength(finalLength);
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
var buffer = new byte[16384];
foreach (FileEntry fe in entries)
{
fe.dataOffset = os.FilePointer;
CopyFile(fe, os, buffer);
}
// Write the data offsets into the directory of the compound stream
foreach (FileEntry fe in entries)
{
os.Seek(fe.directoryOffset);
os.WriteLong(fe.dataOffset);
}
System.Diagnostics.Debug.Assert(finalLength == os.Length);
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
IndexOutput tmp = os;
os = null;
tmp.Dispose(); // AA:
}
finally
{
if (os != null)
{
try
{
os.Dispose(); // AA:
}
catch (System.IO.IOException)
{
}
}
}
}
/// <summary>Merge files with the extensions added up to now.
/// All files with these extensions are combined sequentially into the
/// compound stream. After successful merge, the source files
/// are deleted.
/// </summary>
/// <throws> IllegalStateException if close() had been called before or </throws>
/// <summary> if no file has been added to this object
/// </summary>
public void Close()
{
if (merged)
{
throw new System.SystemException("Merge already performed");
}
if ((entries.Count == 0))
{
throw new System.SystemException("No entries to merge have been defined");
}
merged = true;
// open the compound stream
IndexOutput os = null;
try
{
os = directory.CreateOutput(fileName);
// Write the number of entries
os.WriteVInt(entries.Count);
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
System.Collections.IEnumerator it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
fe.directoryOffset = os.GetFilePointer();
os.WriteLong(0); // for now
os.WriteString(fe.file);
}
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
byte[] buffer = new byte[16384];
it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
fe.dataOffset = os.GetFilePointer();
CopyFile(fe, os, buffer);
}
// Write the data offsets into the directory of the compound stream
it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
os.Seek(fe.directoryOffset);
os.WriteLong(fe.dataOffset);
}
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
IndexOutput tmp = os;
os = null;
tmp.Close();
}
finally
{
if (os != null)
{
try
{
os.Close();
}
catch (System.IO.IOException)
{
}
}
}
}
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi)
{
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
output = directory.CreateOutput(segment + (isIndex ? ".tii" : ".tis"));
output.WriteInt(FORMAT_CURRENT); // write format
output.WriteLong(0); // leave space for size
output.WriteInt(indexInterval); // write indexInterval
output.WriteInt(skipInterval); // write skipInterval
output.WriteInt(maxSkipLevels); // write maxSkipLevels
System.Diagnostics.Debug.Assert(InitUTF16Results());
}
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
IndexOutput output = directory.CreateOutput(fileName, context);
bool success = false;
try
{
CodecUtil.WriteHeader(output, Lucene42FieldInfosFormat.CODEC_NAME, Lucene42FieldInfosFormat.FORMAT_CURRENT);
output.WriteVInt(infos.Size());
foreach (FieldInfo fi in infos)
{
FieldInfo.IndexOptions? indexOptions = fi.FieldIndexOptions;
sbyte bits = 0x0;
if (fi.HasVectors())
{
bits |= Lucene42FieldInfosFormat.STORE_TERMVECTOR;
}
if (fi.OmitsNorms())
{
bits |= Lucene42FieldInfosFormat.OMIT_NORMS;
}
if (fi.HasPayloads())
{
bits |= Lucene42FieldInfosFormat.STORE_PAYLOADS;
}
if (fi.Indexed)
{
bits |= Lucene42FieldInfosFormat.IS_INDEXED;
Debug.Assert(indexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads());
if (indexOptions == FieldInfo.IndexOptions.DOCS_ONLY)
{
bits |= Lucene42FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS;
}
else if (indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
{
bits |= Lucene42FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS;
}
else if (indexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS)
{
bits |= Lucene42FieldInfosFormat.OMIT_POSITIONS;
}
}
output.WriteString(fi.Name);
output.WriteVInt(fi.Number);
output.WriteByte((byte)bits);
// pack the DV types in one byte
var dv = DocValuesByte(fi.DocValuesType);
var nrm = DocValuesByte(fi.NormType);
Debug.Assert((dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0);
var val = unchecked((sbyte)(0xff & ((nrm << 4) | dv)));
output.WriteByte((byte)val);
output.WriteStringStringMap(fi.Attributes());
}
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
internal FieldsWriter(Directory d, System.String segment, FieldInfos fn)
{
fieldInfos = fn;
bool success = false;
System.String fieldsName = segment + "." + IndexFileNames.FIELDS_EXTENSION;
try
{
fieldsStream = d.CreateOutput(fieldsName);
fieldsStream.WriteInt(FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
try
{
Close();
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
try
{
d.DeleteFile(fieldsName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
success = false;
System.String indexName = segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION;
try
{
indexStream = d.CreateOutput(indexName);
indexStream.WriteInt(FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
try
{
Close();
}
catch (System.IO.IOException ioe)
{
}
try
{
d.DeleteFile(fieldsName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
try
{
d.DeleteFile(indexName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
doClose = true;
}
internal FieldsWriter(Directory d, System.String segment, FieldInfos fn)
{
fieldInfos = fn;
fieldsStream = d.CreateOutput(segment + ".fdt");
indexStream = d.CreateOutput(segment + ".fdx");
}
private void WritePostings(Posting[] postings, System.String segment)
{
IndexOutput freq = null, prox = null;
TermInfosWriter tis = null;
TermVectorsWriter termVectorWriter = null;
try
{
//open files for inverse index storage
freq = directory.CreateOutput(segment + ".frq");
prox = directory.CreateOutput(segment + ".prx");
tis = new TermInfosWriter(directory, segment, fieldInfos, termIndexInterval);
TermInfo ti = new TermInfo();
System.String currentField = null;
for (int i = 0; i < postings.Length; i++)
{
Posting posting = postings[i];
// add an entry to the dictionary with pointers to prox and freq files
ti.Set(1, freq.GetFilePointer(), prox.GetFilePointer(), -1);
tis.Add(posting.term, ti);
// add an entry to the freq file
int postingFreq = posting.freq;
if (postingFreq == 1)
{
// optimize freq=1
freq.WriteVInt(1);
}
// set low bit of doc num.
else
{
freq.WriteVInt(0); // the document number
freq.WriteVInt(postingFreq); // frequency in doc
}
int lastPosition = 0; // write positions
int[] positions = posting.positions;
for (int j = 0; j < postingFreq; j++)
{
// use delta-encoding
int position = positions[j];
prox.WriteVInt(position - lastPosition);
lastPosition = position;
}
// check to see if we switched to a new field
System.String termField = posting.term.Field();
if (currentField != termField)
{
// changing field - see if there is something to save
currentField = termField;
FieldInfo fi = fieldInfos.FieldInfo(currentField);
if (fi.storeTermVector)
{
if (termVectorWriter == null)
{
termVectorWriter = new TermVectorsWriter(directory, segment, fieldInfos);
termVectorWriter.OpenDocument();
}
termVectorWriter.OpenField(currentField);
}
else if (termVectorWriter != null)
{
termVectorWriter.CloseField();
}
}
if (termVectorWriter != null && termVectorWriter.IsFieldOpen())
{
termVectorWriter.AddTerm(posting.term.Text(), postingFreq, posting.positions, posting.offsets);
}
}
if (termVectorWriter != null)
{
termVectorWriter.CloseDocument();
}
}
finally
{
// make an effort to close all streams we can but remember and re-throw
// the first exception encountered in this process
System.IO.IOException keep = null;
if (freq != null)
{
try
{
freq.Close();
}
catch (System.IO.IOException e)
{
if (keep == null)
{
keep = e;
}
}
}
if (prox != null)
{
try
{
prox.Close();
}
catch (System.IO.IOException e)
{
if (keep == null)
{
keep = e;
}
}
}
if (tis != null)
{
try
{
tis.Close();
}
catch (System.IO.IOException e)
{
if (keep == null)
{
keep = e;
}
}
}
if (termVectorWriter != null)
{
try
{
termVectorWriter.Close();
}
catch (System.IO.IOException e)
{
if (keep == null)
{
keep = e;
}
}
}
if (keep != null)
{
throw new System.IO.IOException(keep.StackTrace);
}
}
}
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
IndexOutput output = directory.CreateOutput(fileName, context);
bool success = false;
try
{
output.WriteVInt(FORMAT_PREFLEX_RW);
output.WriteVInt(infos.Size());
foreach (FieldInfo fi in infos)
{
sbyte bits = 0x0;
if (fi.HasVectors())
{
bits |= STORE_TERMVECTOR;
}
if (fi.OmitsNorms())
{
bits |= OMIT_NORMS;
}
if (fi.HasPayloads())
{
bits |= STORE_PAYLOADS;
}
if (fi.Indexed)
{
bits |= IS_INDEXED;
Debug.Assert(fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads());
if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_ONLY)
{
bits |= OMIT_TERM_FREQ_AND_POSITIONS;
}
else if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS)
{
bits |= OMIT_POSITIONS;
}
}
output.WriteString(fi.Name);
/*
* we need to write the field number since IW tries
* to stabelize the field numbers across segments so the
* FI ordinal is not necessarily equivalent to the field number
*/
output.WriteInt(fi.Number);
output.WriteByte((byte)bits);
if (fi.Indexed && !fi.OmitsNorms())
{
// to allow null norm types we need to indicate if norms are written
// only in RW case
output.WriteByte((byte)(sbyte)(fi.NormType == null ? 0 : 1));
}
Debug.Assert(fi.Attributes() == null); // not used or supported
}
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
public CachedIndexInput( ICloudProvider CloudProvider, Directory CacheDirectory, string Name )
{
this.name = Name;
#if FULLDEBUG
Debug.WriteLine( "Opening " + this.name );
#endif
this.fileMutex = BlobMutexManager.GrabMutex( this.name );
this.fileMutex.WaitOne();
try {
bool fFileNeeded = false;
FileMetadata cloudMetadata = CloudProvider.FileMetadata( this.name );
if ( !cloudMetadata.Exists ) {
fFileNeeded = false;
// TODO: Delete local if it doesn't exist on cloud?
/*
if (CacheDirectory.FileExists(this.name)) {
CacheDirectory.DeleteFile(this.name);
}
*/
} else if ( !CacheDirectory.FileExists( this.name ) ) {
fFileNeeded = true;
} else {
long cachedLength = CacheDirectory.FileLength( this.name );
long blobLength = cloudMetadata.Length;
DateTime blobLastModifiedUTC = cloudMetadata.LastModified.ToUniversalTime();
if ( !cloudMetadata.Exists || cachedLength != blobLength ) {
fFileNeeded = true;
} else {
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
DateTime cachedLastModifiedUTC = new DateTime( CacheDirectory.FileModified( this.name ), DateTimeKind.Local ).ToUniversalTime();
if ( cachedLastModifiedUTC < blobLastModifiedUTC ) {
TimeSpan timeSpan = blobLastModifiedUTC.Subtract( cachedLastModifiedUTC );
if ( timeSpan.TotalSeconds > 1 ) {
fFileNeeded = true;
} else {
#if FULLDEBUG
Debug.WriteLine( "Using cache for " + this.name + ": " + timeSpan.TotalSeconds );
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if ( fFileNeeded ) {
using ( StreamOutput fileStream = new StreamOutput( CacheDirectory.CreateOutput( this.name ) ) ) {
Stream blobStream = CloudProvider.Download( this.name );
blobStream.CopyTo( fileStream );
fileStream.Flush();
Debug.WriteLine( "GET {0} RETREIVED {1} bytes", this.name, fileStream.Length );
}
} else {
#if FULLDEBUG
if ( !cloudMetadata.Exists ) {
Debug.WriteLine( "Cloud doesn't have " + this.name );
} else {
Debug.WriteLine( "Using cached file for " + this.name );
}
#endif
}
// open the file in read only mode
this.indexInput = CacheDirectory.OpenInput( this.name );
} finally {
this.fileMutex.ReleaseMutex();
}
}
private void Demo_FSIndexInputBug(Directory fsdir, System.String file)
{
// Setup the test file - we need more than 1024 bytes
IndexOutput os = fsdir.CreateOutput(file);
for (int i = 0; i < 2000; i++)
{
os.WriteByte((byte) i);
}
os.Close();
IndexInput in_Renamed = fsdir.OpenInput(file);
// This read primes the buffer in IndexInput
byte b = in_Renamed.ReadByte();
// Close the file
in_Renamed.Close();
// ERROR: this call should fail, but succeeds because the buffer
// is still filled
b = in_Renamed.ReadByte();
// ERROR: this call should fail, but succeeds for some reason as well
in_Renamed.Seek(1099);
// OK: this call correctly fails. We are now past the 1024 internal
// buffer, so an actual IO is attempted, which fails
Assert.Throws<NullReferenceException>(() => in_Renamed.ReadByte(), "expected readByte() to throw exception");
}
public void Write(Directory directory)
{
IndexOutput output = directory.CreateOutput("segments.new");
try
{
output.WriteInt(FORMAT); // write FORMAT
output.WriteLong(++version); // every write changes the index
output.WriteInt(counter); // write counter
output.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
SegmentInfo si = Info(i);
output.WriteString(si.name);
output.WriteInt(si.docCount);
}
}
finally
{
output.Close();
}
// install new segment info
directory.RenameFile("segments.new", IndexFileNames.SEGMENTS);
}
/// <summary>Creates a file of the specified size with sequential data. The first
/// byte is written as the start byte provided. All subsequent bytes are
/// computed as start + offset where offset is the number of the byte.
/// </summary>
private void CreateSequenceFile(Directory dir, System.String name, byte start, int size)
{
IndexOutput os = dir.CreateOutput(name);
for (int i = 0; i < size; i++)
{
os.WriteByte(start);
start++;
}
os.Close();
}
/// <summary>Writes this vector to the file <code>name</code> in Directory
/// <code>d</code>, in a format that can be read by the constructor {@link
/// #BitVector(Directory, String)}.
/// </summary>
public void Write(Directory d, System.String name)
{
IndexOutput output = d.CreateOutput(name);
try
{
output.WriteInt(Size()); // write size
output.WriteInt(Count()); // write count
output.WriteBytes(bits, bits.Length); // write bits
}
finally
{
output.Close();
}
}
internal FieldsWriter(Directory d, System.String segment, FieldInfos fn)
{
fieldInfos = fn;
bool success = false;
System.String fieldsName = segment + "." + IndexFileNames.FIELDS_EXTENSION;
try
{
fieldsStream = d.CreateOutput(fieldsName);
fieldsStream.WriteInt(FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
try
{
Close();
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
try
{
d.DeleteFile(fieldsName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
success = false;
System.String indexName = segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION;
try
{
indexStream = d.CreateOutput(indexName);
indexStream.WriteInt(FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
try
{
Close();
}
catch (System.IO.IOException ioe)
{
}
try
{
d.DeleteFile(fieldsName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
try
{
d.DeleteFile(indexName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
doClose = true;
}
public virtual void CopyFile(Directory dir, System.String src, System.String dest)
{
IndexInput in_Renamed = dir.OpenInput(src);
IndexOutput out_Renamed = dir.CreateOutput(dest);
byte[] b = new byte[1024];
long remainder = in_Renamed.Length();
while (remainder > 0)
{
int len = (int) System.Math.Min(b.Length, remainder);
in_Renamed.ReadBytes(b, 0, len);
out_Renamed.WriteBytes(b, len);
remainder -= len;
}
in_Renamed.Close();
out_Renamed.Close();
}
private void Write(Directory directory)
{
System.String segmentFileName = GetNextSegmentFileName();
// Always advance the generation on write:
if (generation == -1)
{
generation = 1;
}
else
{
generation++;
}
ChecksumIndexOutput output = new ChecksumIndexOutput(directory.CreateOutput(segmentFileName));
bool success = false;
try
{
output.WriteInt(CURRENT_FORMAT); // write FORMAT
output.WriteLong(++version); // every write changes
// the index
output.WriteInt(counter); // write counter
output.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
Info(i).Write(output);
}
output.PrepareCommit();
success = true;
pendingOutput = output;
}
finally
{
if (!success)
{
// we hit an exception above; try to close the file but suppress any exception:
try
{
output.Close();
}
catch (System.Exception)
{
// suppress so we keep throwing the original exception
}
try
{
// try not to leave a truncated segments_N file int the index
directory.DeleteFile(segmentFileName);
}
catch (System.Exception)
{
// suppress so we keep throwing the original exception
}
}
}
}
private void Write(Directory directory)
{
System.String segmentFileName = GetNextSegmentFileName();
// Always advance the generation on write:
if (generation == -1)
{
generation = 1;
}
else
{
generation++;
}
ChecksumIndexOutput segnOutput = new ChecksumIndexOutput(directory.CreateOutput(segmentFileName));
bool success = false;
try
{
segnOutput.WriteInt(CURRENT_FORMAT); // write FORMAT
segnOutput.WriteLong(++version); // every write changes
// the index
segnOutput.WriteInt(counter); // write counter
segnOutput.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
Info(i).Write(segnOutput);
}
segnOutput.WriteStringStringMap(userData);
segnOutput.PrepareCommit();
success = true;
pendingSegnOutput = segnOutput;
}
finally
{
if (!success)
{
// We hit an exception above; try to close the file
// but suppress any exception:
try
{
segnOutput.Close();
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
try
{
// Try not to leave a truncated segments_N file in
// the index:
directory.DeleteFile(segmentFileName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
}
