public PreFlexRWStoredFieldsWriter(Directory directory, string segment, IOContext context)
{
Debug.Assert(directory != null);
this.Directory = directory;
this.Segment = segment;
bool success = false;
try
{
FieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_EXTENSION), context);
IndexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(segment, "", Lucene3xStoredFieldsReader.FIELDS_INDEX_EXTENSION), context);
FieldsStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT);
IndexStream.WriteInt(Lucene3xStoredFieldsReader.FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
Abort();
}
}
}
public FixedGapTermsIndexWriter(SegmentWriteState state)
{
String indexFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
TERMS_INDEX_EXTENSION);
_termIndexInterval = state.TermIndexInterval;
Output = state.Directory.CreateOutput(indexFileName, state.Context);
bool success = false;
try
{
_fieldInfos = state.FieldInfos;
WriteHeader(Output);
Output.WriteInt(_termIndexInterval);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(Output);
}
}
}
public virtual void TestDoubleClose()
{
Directory newDir = NewDirectory();
CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", NewIOContext(Random()), true);
IndexOutput @out = csw.CreateOutput("d.xyz", NewIOContext(Random()));
@out.WriteInt(0);
@out.Dispose();
csw.Dispose();
// close a second time - must have no effect according to IDisposable
csw.Dispose();
csw = new CompoundFileDirectory(newDir, "d.cfs", NewIOContext(Random()), false);
IndexInput openInput = csw.OpenInput("d.xyz", NewIOContext(Random()));
Assert.AreEqual(0, openInput.ReadInt());
openInput.Dispose();
csw.Dispose();
// close a second time - must have no effect according to IDisposable
csw.Dispose();
newDir.Dispose();
}
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
IndexOutput output = directory.CreateOutput(fileName, context);
bool success = false;
try
{
output.WriteVInt(FORMAT_PREFLEX_RW);
output.WriteVInt(infos.Size());
foreach (FieldInfo fi in infos)
{
sbyte bits = 0x0;
if (fi.HasVectors())
{
bits |= STORE_TERMVECTOR;
}
if (fi.OmitsNorms())
{
bits |= OMIT_NORMS;
}
if (fi.HasPayloads())
{
bits |= STORE_PAYLOADS;
}
if (fi.Indexed)
{
bits |= IS_INDEXED;
Debug.Assert(fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads());
if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_ONLY)
{
bits |= OMIT_TERM_FREQ_AND_POSITIONS;
}
else if (fi.FieldIndexOptions == FieldInfo.IndexOptions.DOCS_AND_FREQS)
{
bits |= OMIT_POSITIONS;
}
}
output.WriteString(fi.Name);
/*
* we need to write the field number since IW tries
* to stabelize the field numbers across segments so the
* FI ordinal is not necessarily equivalent to the field number
*/
output.WriteInt(fi.Number);
output.WriteByte(bits);
if (fi.Indexed && !fi.OmitsNorms())
{
// to allow null norm types we need to indicate if norms are written
// only in RW case
output.WriteByte((sbyte)(fi.NormType == null ? 0 : 1));
}
Debug.Assert(fi.Attributes() == null); // not used or supported
}
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
private void FlushOffsets(int[] fieldNums)
{
bool hasOffsets = false;
long[] sumPos = new long[fieldNums.Length];
long[] sumOffsets = new long[fieldNums.Length];
foreach (DocData dd in PendingDocs)
{
foreach (FieldData fd in dd.Fields)
{
hasOffsets |= fd.HasOffsets;
if (fd.HasOffsets && fd.HasPositions)
{
int fieldNumOff = Array.BinarySearch(fieldNums, fd.FieldNum);
int pos = 0;
for (int i = 0; i < fd.NumTerms; ++i)
{
int previousPos = 0;
int previousOff = 0;
for (int j = 0; j < fd.Freqs[i]; ++j)
{
int position = PositionsBuf[fd.PosStart + pos];
int startOffset = StartOffsetsBuf[fd.OffStart + pos];
sumPos[fieldNumOff] += position - previousPos;
sumOffsets[fieldNumOff] += startOffset - previousOff;
previousPos = position;
previousOff = startOffset;
++pos;
}
}
Debug.Assert(pos == fd.TotalPositions);
}
}
}
if (!hasOffsets)
{
// nothing to do
return;
}
float[] charsPerTerm = new float[fieldNums.Length];
for (int i = 0; i < fieldNums.Length; ++i)
{
charsPerTerm[i] = (sumPos[i] <= 0 || sumOffsets[i] <= 0) ? 0 : (float)((double)sumOffsets[i] / sumPos[i]);
}
// start offsets
for (int i = 0; i < fieldNums.Length; ++i)
{
VectorsStream.WriteInt(Number.FloatToIntBits(charsPerTerm[i]));
}
Writer.Reset(VectorsStream);
foreach (DocData dd in PendingDocs)
{
foreach (FieldData fd in dd.Fields)
{
if ((fd.Flags & OFFSETS) != 0)
{
int fieldNumOff = Array.BinarySearch(fieldNums, fd.FieldNum);
float cpt = charsPerTerm[fieldNumOff];
int pos = 0;
for (int i = 0; i < fd.NumTerms; ++i)
{
int previousPos = 0;
int previousOff = 0;
for (int j = 0; j < fd.Freqs[i]; ++j)
{
int position = fd.HasPositions ? PositionsBuf[fd.PosStart + pos] : 0;
int startOffset = StartOffsetsBuf[fd.OffStart + pos];
Writer.Add(startOffset - previousOff - (int)(cpt * (position - previousPos)));
previousPos = position;
previousOff = startOffset;
++pos;
}
}
}
}
}
Writer.Finish();
// lengths
Writer.Reset(VectorsStream);
foreach (DocData dd in PendingDocs)
{
foreach (FieldData fd in dd.Fields)
{
if ((fd.Flags & OFFSETS) != 0)
{
int pos = 0;
for (int i = 0; i < fd.NumTerms; ++i)
{
for (int j = 0; j < fd.Freqs[i]; ++j)
{
Writer.Add(LengthsBuf[fd.OffStart + pos++] - fd.PrefixLengths[i] - fd.SuffixLengths[i]);
}
}
Debug.Assert(pos == fd.TotalPositions);
}
}
}
Writer.Finish();
}
public override void WriteField(FieldInfo info, IndexableField field)
{
FieldsStream.WriteVInt(info.Number);
int bits = 0;
BytesRef bytes;
string @string;
// TODO: maybe a field should serialize itself?
// this way we don't bake into indexer all these
// specific encodings for different fields? and apps
// can customize...
object number = (object)field.NumericValue;
if (number != null)
{
if (number is sbyte || number is short || number is int)
{
bits |= FIELD_IS_NUMERIC_INT;
}
else if (number is long)
{
bits |= FIELD_IS_NUMERIC_LONG;
}
else if (number is float)
{
bits |= FIELD_IS_NUMERIC_FLOAT;
}
else if (number is double)
{
bits |= FIELD_IS_NUMERIC_DOUBLE;
}
else
{
throw new System.ArgumentException("cannot store numeric type " + number.GetType());
}
@string = null;
bytes = null;
}
else
{
bytes = field.BinaryValue();
if (bytes != null)
{
bits |= FIELD_IS_BINARY;
@string = null;
}
else
{
@string = field.StringValue;
if (@string == null)
{
throw new System.ArgumentException("field " + field.Name() + " is stored but does not have binaryValue, stringValue nor numericValue");
}
}
}
FieldsStream.WriteByte((byte)(sbyte)bits);
if (bytes != null)
{
FieldsStream.WriteVInt(bytes.Length);
FieldsStream.WriteBytes(bytes.Bytes, bytes.Offset, bytes.Length);
}
else if (@string != null)
{
FieldsStream.WriteString(field.StringValue);
}
else
{
if (number is sbyte || number is short || number is int)
{
FieldsStream.WriteInt((int)number);
}
else if (number is long)
{
FieldsStream.WriteLong((long)number);
}
else if (number is float)
{
FieldsStream.WriteInt(Number.FloatToIntBits((float)number));
}
else if (number is double)
{
FieldsStream.WriteLong(BitConverter.DoubleToInt64Bits((double)number));
}
else
{
throw new InvalidOperationException("Cannot get here");
}
}
}
internal void FinishCommit(Directory dir)
{
if (pendingSegnOutput == null)
{
throw new System.SystemException("prepareCommit was not called");
}
bool success = false;
try
{
pendingSegnOutput.FinishCommit();
pendingSegnOutput.Close();
pendingSegnOutput = null;
success = true;
}
finally
{
if (!success)
{
RollbackCommit(dir);
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
internal FieldsWriter(Directory d, System.String segment, FieldInfos fn)
{
fieldInfos = fn;
bool success = false;
String fieldsName = segment + "." + IndexFileNames.FIELDS_EXTENSION;
try
{
fieldsStream = d.CreateOutput(fieldsName);
fieldsStream.WriteInt(FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
try
{
Dispose();
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
try
{
d.DeleteFile(fieldsName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
success = false;
String indexName = segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION;
try
{
indexStream = d.CreateOutput(indexName);
indexStream.WriteInt(FORMAT_CURRENT);
success = true;
}
finally
{
if (!success)
{
try
{
Dispose();
}
catch (System.IO.IOException)
{
}
try
{
d.DeleteFile(fieldsName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
try
{
d.DeleteFile(indexName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
doClose = true;
}
/// <summary>Write as a bit set </summary>
private void WriteBits(IndexOutput output)
{
output.WriteInt(Size()); // write size
output.WriteInt(Count()); // write count
output.WriteBytes(bits, bits.Length);
}
/// <summary>
/// Writes a codec footer, which records both a checksum
/// algorithm ID and a checksum. this footer can
/// be parsed and validated with
/// <seealso cref="#checkFooter(ChecksumIndexInput) checkFooter()"/>.
/// <p>
/// CodecFooter --> Magic,AlgorithmID,Checksum
/// <ul>
/// <li>Magic --> <seealso cref="DataOutput#writeInt Uint32"/>. this
/// identifies the start of the footer. It is always {@value #FOOTER_MAGIC}.
/// <li>AlgorithmID --> <seealso cref="DataOutput#writeInt Uint32"/>. this
/// indicates the checksum algorithm used. Currently this is always 0,
/// for zlib-crc32.
/// <li>Checksum --> <seealso cref="DataOutput#writeLong Uint32"/>. The
/// actual checksum value for all previous bytes in the stream, including
/// the bytes from Magic and AlgorithmID.
/// </ul>
/// </summary>
/// <param name="out"> Output stream </param>
/// <exception cref="IOException"> If there is an I/O error writing to the underlying medium. </exception>
public static void WriteFooter(IndexOutput @out)
{
@out.WriteInt(FOOTER_MAGIC);
@out.WriteInt(0);
@out.WriteLong(@out.Checksum);
}
public override void Init(IndexOutput termsOut)
{
CodecUtil.WriteHeader(termsOut, CODEC, VERSION_CURRENT);
// TODO: -- just ask skipper to "start" here
termsOut.WriteInt(SKIP_INTERVAL); // write skipInterval
termsOut.WriteInt(MAX_SKIP_LEVELS); // write maxSkipLevels
termsOut.WriteInt(SKIP_MINIMUM); // write skipMinimum
}
