/// <summary> Save this segment's info.</summary>
internal void Write(IndexOutput output)
{
output.WriteString(name);
output.WriteInt(docCount);
output.WriteLong(delGen);
output.WriteInt(docStoreOffset);
if (docStoreOffset != -1)
{
output.WriteString(docStoreSegment);
output.WriteByte((byte)(docStoreIsCompoundFile?1:0));
}
output.WriteByte((byte)(hasSingleNormFile?1:0));
if (normGen == null)
{
output.WriteInt(NO);
}
else
{
output.WriteInt(normGen.Length);
for (int j = 0; j < normGen.Length; j++)
{
output.WriteLong(normGen[j]);
}
}
output.WriteByte((byte)isCompoundFile);
output.WriteInt(delCount);
output.WriteByte((byte)(hasProx?1:0));
output.WriteStringStringMap(diagnostics);
}
public void Write(Directory directory)
{
System.String segmentFileName = GetNextSegmentFileName();
// Always advance the generation on write:
if (generation == -1)
{
generation = 1;
}
else
{
generation++;
}
IndexOutput output = directory.CreateOutput(segmentFileName);
try
{
output.WriteInt(FORMAT_SINGLE_NORM_FILE); // write FORMAT
output.WriteLong(++version); // every write changes
// the index
output.WriteInt(counter); // write counter
output.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
Info(i).Write(output);
}
}
finally
{
output.Close();
}
try
{
output = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
output.WriteInt(FORMAT_LOCKLESS);
output.WriteLong(generation);
output.WriteLong(generation);
}
finally
{
output.Close();
}
}
catch (System.IO.IOException e)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
lastGeneration = generation;
}
public void Write(Directory directory)
{
IndexOutput output = directory.CreateOutput("segments.new");
try
{
output.WriteInt(FORMAT); // write FORMAT
output.WriteLong(++version); // every write changes the index
output.WriteInt(counter); // write counter
output.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
SegmentInfo si = Info(i);
output.WriteString(si.name);
output.WriteInt(si.docCount);
}
}
finally
{
output.Close();
}
// install new segment info
directory.RenameFile("segments.new", IndexFileNames.SEGMENTS);
}
/// <summary>Fills in no-term-vectors for all docs we haven't seen
/// since the last doc that had term vectors.
/// </summary>
internal void Fill(int docID)
{
int docStoreOffset = docWriter.DocStoreOffset;
int end = docID + docStoreOffset;
if (lastDocID < end)
{
long tvfPosition = tvf.FilePointer;
while (lastDocID < end)
{
tvx.WriteLong(tvd.FilePointer);
tvd.WriteVInt(0);
tvx.WriteLong(tvfPosition);
lastDocID++;
}
}
}
/// <summary> Save this segment's info.</summary>
internal void Write(IndexOutput output)
{
output.WriteString(name);
output.WriteInt(docCount);
output.WriteLong(delGen);
output.WriteByte((byte)(hasSingleNormFile ? 1 : 0));
if (normGen == null)
{
output.WriteInt(-1);
}
else
{
output.WriteInt(normGen.Length);
for (int j = 0; j < normGen.Length; j++)
{
output.WriteLong(normGen[j]);
}
}
output.WriteByte((byte)isCompoundFile);
}
/// <summary>Called to complete TermInfos creation. </summary>
public /*internal*/ void Close()
{
output.Seek(4); // write size after format
output.WriteLong(size);
output.Close();
if (!isIndex)
{
other.Close();
}
}
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi)
{
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
output = directory.CreateOutput(segment + (isIndex ? ".tii" : ".tis"));
output.WriteInt(FORMAT); // write format
output.WriteLong(0); // leave space for size
output.WriteInt(indexInterval); // write indexInterval
output.WriteInt(skipInterval); // write skipInterval
}
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi)
{
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
output = directory.CreateOutput(segment + (isIndex?".tii":".tis"));
output.WriteInt(FORMAT_CURRENT); // write format
output.WriteLong(0); // leave space for size
output.WriteInt(indexInterval); // write indexInterval
output.WriteInt(skipInterval); // write skipInterval
output.WriteInt(maxSkipLevels); // write maxSkipLevels
System.Diagnostics.Debug.Assert(InitUTF16Results());
}
/// <summary>Called to complete TermInfos creation. </summary>
public void Dispose()
{
// Move to protected method if class becomes unsealed
if (isDisposed)
{
return;
}
output.Seek(4); // write size after format
output.WriteLong(size);
output.Dispose();
if (!isIndex)
{
other.Dispose();
}
isDisposed = true;
}
/// <summary>Merge files with the extensions added up to now.
/// All files with these extensions are combined sequentially into the
/// compound stream. After successful merge, the source files
/// are deleted.
/// </summary>
/// <throws> IllegalStateException if close() had been called before or </throws>
/// <summary> if no file has been added to this object
/// </summary>
public void Close()
{
if (merged)
{
throw new System.SystemException("Merge already performed");
}
if ((entries.Count == 0))
{
throw new System.SystemException("No entries to merge have been defined");
}
merged = true;
// open the compound stream
IndexOutput os = null;
try
{
os = directory.CreateOutput(fileName);
// Write the number of entries
os.WriteVInt(entries.Count);
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
System.Collections.IEnumerator it = entries.GetEnumerator();
long totalSize = 0;
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
fe.directoryOffset = os.GetFilePointer();
os.WriteLong(0); // for now
os.WriteString(fe.file);
totalSize += directory.FileLength(fe.file);
}
// Pre-allocate size of file as optimization --
// this can potentially help IO performance as
// we write the file and also later during
// searching. It also uncovers a disk-full
// situation earlier and hopefully without
// actually filling disk to 100%:
long finalLength = totalSize + os.GetFilePointer();
os.SetLength(finalLength);
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
byte[] buffer = new byte[16384];
it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
fe.dataOffset = os.GetFilePointer();
CopyFile(fe, os, buffer);
}
// Write the data offsets into the directory of the compound stream
it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
os.Seek(fe.directoryOffset);
os.WriteLong(fe.dataOffset);
}
System.Diagnostics.Debug.Assert(finalLength == os.Length());
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
IndexOutput tmp = os;
os = null;
tmp.Close();
}
finally
{
if (os != null)
{
try
{
os.Close();
}
catch (System.IO.IOException e)
{
}
}
}
}
internal void AddDocument(Document doc)
{
indexStream.WriteLong(fieldsStream.GetFilePointer());
int storedCount = 0;
System.Collections.IEnumerator fieldIterator = doc.GetFields().GetEnumerator();
while (fieldIterator.MoveNext())
{
Fieldable field = (Fieldable)fieldIterator.Current;
if (field.IsStored())
{
storedCount++;
}
}
fieldsStream.WriteVInt(storedCount);
fieldIterator = doc.GetFields().GetEnumerator();
while (fieldIterator.MoveNext())
{
Fieldable field = (Fieldable)fieldIterator.Current;
// if the field as an instanceof FieldsReader.FieldForMerge, we're in merge mode
// and field.binaryValue() already returns the compressed value for a field
// with isCompressed()==true, so we disable compression in that case
bool disableCompression = (field is FieldsReader.FieldForMerge);
if (field.IsStored())
{
fieldsStream.WriteVInt(fieldInfos.FieldNumber(field.Name()));
byte bits = 0;
if (field.IsTokenized())
{
bits |= FieldsWriter.FIELD_IS_TOKENIZED;
}
if (field.IsBinary())
{
bits |= FieldsWriter.FIELD_IS_BINARY;
}
if (field.IsCompressed())
{
bits |= FieldsWriter.FIELD_IS_COMPRESSED;
}
fieldsStream.WriteByte(bits);
if (field.IsCompressed())
{
// compression is enabled for the current field
byte[] data = null;
if (disableCompression)
{
// optimized case for merging, the data
// is already compressed
data = field.BinaryValue();
}
else
{
// check if it is a binary field
if (field.IsBinary())
{
data = Compress(field.BinaryValue());
}
else
{
data = Compress(System.Text.Encoding.GetEncoding("UTF-8").GetBytes(field.StringValue()));
}
}
int len = data.Length;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, len);
}
else
{
// compression is disabled for the current field
if (field.IsBinary())
{
byte[] data = field.BinaryValue();
int len = data.Length;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, len);
}
else
{
fieldsStream.WriteString(field.StringValue());
}
}
}
}
}
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi)
{
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
output = directory.CreateOutput(segment + (isIndex ? ".tii" : ".tis"));
output.WriteInt(FORMAT); // write format
output.WriteLong(0); // leave space for size
output.WriteInt(indexInterval); // write indexInterval
output.WriteInt(skipInterval); // write skipInterval
}
private void AddVarIntsField(FieldInfo field, IndexOutput output, IEnumerable<long?> values, long minValue, long maxValue)
{
field.PutAttribute(LegacyKey, LegacyDocValuesType.VAR_INTS.Name);
CodecUtil.WriteHeader(output, Lucene40DocValuesFormat.VAR_INTS_CODEC_NAME, Lucene40DocValuesFormat.VAR_INTS_VERSION_CURRENT);
long delta = maxValue - minValue;
if (delta < 0)
{
// writes longs
output.WriteByte((byte)Lucene40DocValuesFormat.VAR_INTS_FIXED_64);
foreach (long? n in values)
{
output.WriteLong(n == null ? 0 : n.Value);
}
}
else
{
// writes packed ints
output.WriteByte((byte)Lucene40DocValuesFormat.VAR_INTS_PACKED);
output.WriteLong(minValue);
output.WriteLong(0 - minValue); // default value (representation of 0)
PackedInts.Writer writer = PackedInts.GetWriter(output, State.SegmentInfo.DocCount, PackedInts.BitsRequired(delta), PackedInts.DEFAULT);
foreach (long? n in values)
{
long v = n == null ? 0 : (long)n;
writer.Add(v - minValue);
}
writer.Finish();
}
}
private void Initialize(Directory directory, string segment, FieldInfos fis, int interval, bool isi)
{
IndexInterval = interval;
FieldInfos = fis;
IsIndex = isi;
Output = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", (IsIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION : Lucene3xPostingsFormat.TERMS_EXTENSION)), IOContext.DEFAULT);
bool success = false;
try
{
Output.WriteInt(FORMAT_CURRENT); // write format
Output.WriteLong(0); // leave space for size
Output.WriteInt(IndexInterval); // write indexInterval
Output.WriteInt(SkipInterval); // write skipInterval
Output.WriteInt(MaxSkipLevels); // write maxSkipLevels
Debug.Assert(InitUTF16Results());
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(Output);
try
{
directory.DeleteFile(IndexFileNames.SegmentFileName(segment, "", (IsIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION : Lucene3xPostingsFormat.TERMS_EXTENSION)));
}
catch (IOException ignored)
{
}
}
}
}
/// <summary> Add a complete document specified by all its term vectors. If document has no
/// term vectors, add value for tvx.
///
/// </summary>
/// <param name="vectors">
/// </param>
/// <throws> IOException </throws>
public void AddAllDocVectors(TermFreqVector[] vectors)
{
tvx.WriteLong(tvd.GetFilePointer());
tvx.WriteLong(tvf.GetFilePointer());
if (vectors != null)
{
int numFields = vectors.Length;
tvd.WriteVInt(numFields);
long[] fieldPointers = new long[numFields];
for (int i = 0; i < numFields; i++)
{
fieldPointers[i] = tvf.GetFilePointer();
int fieldNumber = fieldInfos.FieldNumber(vectors[i].GetField());
// 1st pass: write field numbers to tvd
tvd.WriteVInt(fieldNumber);
int numTerms = vectors[i].Size();
tvf.WriteVInt(numTerms);
TermPositionVector tpVector;
byte bits;
bool storePositions;
bool storeOffsets;
if (vectors[i] is TermPositionVector)
{
// May have positions & offsets
tpVector = (TermPositionVector)vectors[i];
storePositions = tpVector.Size() > 0 && tpVector.GetTermPositions(0) != null;
storeOffsets = tpVector.Size() > 0 && tpVector.GetOffsets(0) != null;
bits = (byte)((storePositions ? TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR : (byte)0) + (storeOffsets ? TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR : (byte)0));
}
else
{
tpVector = null;
bits = 0;
storePositions = false;
storeOffsets = false;
}
tvf.WriteVInt(bits);
string[] terms = vectors[i].GetTerms();
int[] freqs = vectors[i].GetTermFrequencies();
int utf8Upto = 0;
utf8Results[1].length = 0;
for (int j = 0; j < numTerms; j++)
{
UnicodeUtil.UTF16toUTF8(terms[j], 0, terms[j].Length, utf8Results[utf8Upto]);
int start = StringHelper.bytesDifference(
utf8Results[1 - utf8Upto].result, utf8Results[1 - utf8Upto].length, utf8Results[utf8Upto].result, utf8Results[utf8Upto].length);
int length = utf8Results[utf8Upto].length - start;
tvf.WriteVInt(start); // write shared prefix length
tvf.WriteVInt(length); // write delta length
tvf.WriteBytes(utf8Results[utf8Upto].result, start, length); // write delta bytes
utf8Upto = 1 - utf8Upto;
int termFreq = freqs[j];
tvf.WriteVInt(termFreq);
if (storePositions)
{
int[] positions = tpVector.GetTermPositions(j);
if (positions == null)
{
throw new System.SystemException("Trying to write positions that are null!");
}
System.Diagnostics.Debug.Assert(positions.Length == termFreq);
// use delta encoding for positions
int lastPosition = 0;
for (int k = 0; k < positions.Length; k++)
{
int position = positions[k];
tvf.WriteVInt(position - lastPosition);
lastPosition = position;
}
}
if (storeOffsets)
{
TermVectorOffsetInfo[] offsets = tpVector.GetOffsets(j);
if (offsets == null)
{
throw new System.SystemException("Trying to write offsets that are null!");
}
System.Diagnostics.Debug.Assert(offsets.Length == termFreq);
// use delta encoding for offsets
int lastEndOffset = 0;
for (int k = 0; k < offsets.Length; k++)
{
int startOffset = offsets[k].GetStartOffset();
int endOffset = offsets[k].GetEndOffset();
tvf.WriteVInt(startOffset - lastEndOffset);
tvf.WriteVInt(endOffset - startOffset);
lastEndOffset = endOffset;
}
}
}
}
// 2nd pass: write field pointers to tvd
long lastFieldPointer = fieldPointers[0];
for (int i = 1; i < numFields; i++)
{
long fieldPointer = fieldPointers[i];
tvd.WriteVLong(fieldPointer - lastFieldPointer);
lastFieldPointer = fieldPointer;
}
}
else
{
tvd.WriteVInt(0);
}
}
internal void AddDocument(Document doc)
{
indexStream.WriteLong(fieldsStream.GetFilePointer());
int storedCount = 0;
foreach (Field field in doc.Fields())
{
if (field.IsStored())
{
storedCount++;
}
}
fieldsStream.WriteVInt(storedCount);
foreach (Field field in doc.Fields())
{
if (field.IsStored())
{
fieldsStream.WriteVInt(fieldInfos.FieldNumber(field.Name()));
byte bits = 0;
if (field.IsTokenized())
{
bits |= FieldsWriter.FIELD_IS_TOKENIZED;
}
if (field.IsBinary())
{
bits |= FieldsWriter.FIELD_IS_BINARY;
}
if (field.IsCompressed())
{
bits |= FieldsWriter.FIELD_IS_COMPRESSED;
}
fieldsStream.WriteByte(bits);
if (field.IsCompressed())
{
// compression is enabled for the current field
byte[] data = null;
// check if it is a binary field
if (field.IsBinary())
{
data = Compress(field.BinaryValue());
}
else
{
data = Compress(System.Text.Encoding.GetEncoding("UTF-8").GetBytes(field.StringValue()));
}
int len = data.Length;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, len);
}
else
{
// compression is disabled for the current field
if (field.IsBinary())
{
byte[] data = field.BinaryValue();
int len = data.Length;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, len);
}
else
{
fieldsStream.WriteString(field.StringValue());
}
}
}
}
}
/// <summary>Merge files with the extensions added up to now.
/// All files with these extensions are combined sequentially into the
/// compound stream. After successful merge, the source files
/// are deleted.
/// </summary>
/// <throws> IllegalStateException if close() had been called before or </throws>
/// <summary> if no file has been added to this object
/// </summary>
public void Dispose()
{
// Extract into protected method if class ever becomes unsealed
// TODO: Dispose shouldn't throw exceptions!
if (merged)
throw new SystemException("Merge already performed");
if ((entries.Count == 0))
throw new SystemException("No entries to merge have been defined");
merged = true;
// open the compound stream
IndexOutput os = null;
try
{
os = directory.CreateOutput(fileName);
// Write the number of entries
os.WriteVInt(entries.Count);
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
long totalSize = 0;
foreach (FileEntry fe in entries)
{
fe.directoryOffset = os.FilePointer;
os.WriteLong(0); // for now
os.WriteString(fe.file);
totalSize += directory.FileLength(fe.file);
}
// Pre-allocate size of file as optimization --
// this can potentially help IO performance as
// we write the file and also later during
// searching. It also uncovers a disk-full
// situation earlier and hopefully without
// actually filling disk to 100%:
long finalLength = totalSize + os.FilePointer;
os.SetLength(finalLength);
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
var buffer = new byte[16384];
foreach (FileEntry fe in entries)
{
fe.dataOffset = os.FilePointer;
CopyFile(fe, os, buffer);
}
// Write the data offsets into the directory of the compound stream
foreach (FileEntry fe in entries)
{
os.Seek(fe.directoryOffset);
os.WriteLong(fe.dataOffset);
}
System.Diagnostics.Debug.Assert(finalLength == os.Length);
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
IndexOutput tmp = os;
os = null;
tmp.Close();
}
finally
{
if (os != null)
try
{
os.Close();
}
catch (System.IO.IOException)
{
}
}
}
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi)
{
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
output = directory.CreateOutput(segment + (isIndex ? ".tii" : ".tis"));
output.WriteInt(FORMAT_CURRENT); // write format
output.WriteLong(0); // leave space for size
output.WriteInt(indexInterval); // write indexInterval
output.WriteInt(skipInterval); // write skipInterval
output.WriteInt(maxSkipLevels); // write maxSkipLevels
System.Diagnostics.Debug.Assert(InitUTF16Results());
}
// Writes the contents of buffer into the fields stream
// and adds a new entry for this document into the index
// stream. This assumes the buffer was already written
// in the correct fields format.
internal void FlushDocument(int numStoredFields, RAMOutputStream buffer)
{
indexStream.WriteLong(fieldsStream.GetFilePointer());
fieldsStream.WriteVInt(numStoredFields);
buffer.WriteTo(fieldsStream);
}
/// <summary>Merge files with the extensions added up to now.
/// All files with these extensions are combined sequentially into the
/// compound stream. After successful merge, the source files
/// are deleted.
/// </summary>
/// <throws> IllegalStateException if close() had been called before or </throws>
/// <summary> if no file has been added to this object
/// </summary>
public void Close()
{
if (merged)
{
throw new System.SystemException("Merge already performed");
}
if ((entries.Count == 0))
{
throw new System.SystemException("No entries to merge have been defined");
}
merged = true;
// open the compound stream
IndexOutput os = null;
try
{
os = directory.CreateOutput(fileName);
// Write the number of entries
os.WriteVInt(entries.Count);
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
System.Collections.IEnumerator it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
fe.directoryOffset = os.GetFilePointer();
os.WriteLong(0); // for now
os.WriteString(fe.file);
}
// Open the files and copy their data into the stream.
// Remember the locations of each file's data section.
byte[] buffer = new byte[1024];
it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
fe.dataOffset = os.GetFilePointer();
CopyFile(fe, os, buffer);
}
// Write the data offsets into the directory of the compound stream
it = entries.GetEnumerator();
while (it.MoveNext())
{
FileEntry fe = (FileEntry)it.Current;
os.Seek(fe.directoryOffset);
os.WriteLong(fe.dataOffset);
}
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
IndexOutput tmp = os;
os = null;
tmp.Close();
}
finally
{
if (os != null)
{
try
{
os.Close();
}
catch (System.IO.IOException)
{
}
}
}
}
/// <summary> Save this segment's info.</summary>
internal void Write(IndexOutput output)
{
output.WriteString(name);
output.WriteInt(docCount);
output.WriteLong(delGen);
output.WriteByte((byte) (hasSingleNormFile ? 1 : 0));
if (normGen == null)
{
output.WriteInt(- 1);
}
else
{
output.WriteInt(normGen.Length);
for (int j = 0; j < normGen.Length; j++)
{
output.WriteLong(normGen[j]);
}
}
output.WriteByte((byte) isCompoundFile);
}
private void AddVarDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable<BytesRef> values)
{
field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_DEREF.Name);
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
// deduplicate
SortedSet<BytesRef> dictionary = new SortedSet<BytesRef>();
foreach (BytesRef v in values)
{
dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v));
}
/* values */
long startPosition = data.FilePointer;
long currentAddress = 0;
Dictionary<BytesRef, long> valueToAddress = new Dictionary<BytesRef, long>();
foreach (BytesRef v in dictionary)
{
currentAddress = data.FilePointer - startPosition;
valueToAddress[v] = currentAddress;
WriteVShort(data, v.Length);
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
/* ordinals */
long totalBytes = data.FilePointer - startPosition;
index.WriteLong(totalBytes);
int maxDoc = State.SegmentInfo.DocCount;
PackedInts.Writer w = PackedInts.GetWriter(index, maxDoc, PackedInts.BitsRequired(currentAddress), PackedInts.DEFAULT);
foreach (BytesRef v in values)
{
w.Add(valueToAddress[v == null ? new BytesRef() : v]);
}
w.Finish();
}
internal void FinishCommit(Directory dir)
{
if (pendingSegnOutput == null)
{
throw new System.SystemException("prepareCommit was not called");
}
bool success = false;
try
{
pendingSegnOutput.FinishCommit();
pendingSegnOutput.Close();
pendingSegnOutput = null;
success = true;
}
finally
{
if (!success)
{
RollbackCommit(dir);
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName);
}
catch (System.Exception t)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception t)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
/// <summary> Save this segment's info.</summary>
internal void Write(IndexOutput output)
{
output.WriteString(name);
output.WriteInt(docCount);
output.WriteLong(delGen);
output.WriteInt(docStoreOffset);
if (docStoreOffset != - 1)
{
output.WriteString(docStoreSegment);
output.WriteByte((byte) (docStoreIsCompoundFile?1:0));
}
output.WriteByte((byte) (hasSingleNormFile?1:0));
if (normGen == null)
{
output.WriteInt(NO);
}
else
{
output.WriteInt(normGen.Length);
for (int j = 0; j < normGen.Length; j++)
{
output.WriteLong(normGen[j]);
}
}
output.WriteByte((byte) isCompoundFile);
output.WriteInt(delCount);
output.WriteByte((byte) (hasProx?1:0));
output.WriteStringStringMap(diagnostics);
}
private void AddVarSortedBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd)
{
field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_SORTED.Name);
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
/* values */
long startPos = data.FilePointer;
int valueCount = 0;
foreach (BytesRef v in values)
{
data.WriteBytes(v.Bytes, v.Offset, v.Length);
valueCount++;
}
/* addresses */
long maxAddress = data.FilePointer - startPos;
index.WriteLong(maxAddress);
Debug.Assert(valueCount != int.MaxValue); // unsupported by the 4.0 impl
PackedInts.Writer w = PackedInts.GetWriter(index, valueCount + 1, PackedInts.BitsRequired(maxAddress), PackedInts.DEFAULT);
long currentPosition = 0;
foreach (BytesRef v in values)
{
w.Add(currentPosition);
currentPosition += v.Length;
}
// write sentinel
Debug.Assert(currentPosition == maxAddress);
w.Add(currentPosition);
w.Finish();
/* ordinals */
int maxDoc = State.SegmentInfo.DocCount;
Debug.Assert(valueCount > 0);
PackedInts.Writer ords = PackedInts.GetWriter(index, maxDoc, PackedInts.BitsRequired(valueCount - 1), PackedInts.DEFAULT);
foreach (long n in docToOrd)
{
ords.Add((long)n);
}
ords.Finish();
}