internal DirectDocValuesConsumer(SegmentWriteState state, string dataCodec, string dataExtension,
string metaCodec, string metaExtension)
{
maxDoc = state.SegmentInfo.DocCount;
bool success = false;
try
{
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
dataExtension);
data = state.Directory.CreateOutput(dataName, state.Context);
CodecUtil.WriteHeader(data, dataCodec, DirectDocValuesProducer.VERSION_CURRENT);
string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
metaExtension);
meta = state.Directory.CreateOutput(metaName, state.Context);
CodecUtil.WriteHeader(meta, metaCodec, DirectDocValuesProducer.VERSION_CURRENT);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(this);
}
}
}
public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter)
{
var termsFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
TERMS_EXTENSION);
_postingsWriter = postingsWriter;
_fieldInfos = state.FieldInfos;
_output = state.Directory.CreateOutput(termsFileName, state.Context);
var success = false;
try
{
WriteHeader(_output);
_postingsWriter.Init(_output);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(_output);
}
}
}
protected override void Dispose(bool disposing)
{
if (!disposing)
{
return;
}
var success = false;
try
{
if (meta != null)
{
meta.WriteVInt(-1); // write EOF marker
CodecUtil.WriteFooter(meta); // write checksum
}
if (data != null)
{
CodecUtil.WriteFooter(data);
}
success = true;
}
finally
{
if (success)
{
IOUtils.Close(data, meta);
}
else
{
IOUtils.CloseWhileHandlingException(data, meta);
}
data = meta = null;
}
}
protected override void Dispose(bool disposing)
{
if (data == null || disposing)
{
return;
}
var success = false;
try
{
Debug.Assert(_fieldsSeen.Count > 0);
// java : sheisty to do this here?
SimpleTextUtil.Write(data, END);
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.WriteChecksum(data, scratch);
success = true;
}
finally
{
if (success)
{
IOUtils.Close(data);
}
else
{
IOUtils.CloseWhileHandlingException(data);
}
data = null;
}
}
public override void Dispose()
{
try
{
IOUtils.Close(postingsReader);
}
finally
{
fields.Clear();
}
}
public override sealed void Abort()
{
try
{
Dispose();
}
finally
{
IOUtils.DeleteFilesIgnoringExceptions(_directory,
IndexFileNames.SegmentFileName(_segment, "", VECTORS_EXTENSION));
}
}
protected override void Dispose(bool disposing)
{
if (!disposing)
{
return;
}
try
{
IOUtils.Close(_output);
}
finally
{
_output = null;
}
}
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
{
string termsFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
this.postingsReader = postingsReader;
IndexInput @in = state.Directory.OpenInput(termsFileName, state.Context);
bool success = false;
try
{
version = ReadHeader(@in);
if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM)
{
CodecUtil.ChecksumEntireFile(@in);
}
this.postingsReader.Init(@in);
SeekDir(@in);
FieldInfos fieldInfos = state.FieldInfos;
int numFields = @in.ReadVInt();
for (int i = 0; i < numFields; i++)
{
int fieldNumber = @in.ReadVInt();
FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber);
long numTerms = @in.ReadVLong();
long sumTotalTermFreq = fieldInfo.FieldIndexOptions == IndexOptions.DOCS_ONLY ? -1 : @in.ReadVLong();
long sumDocFreq = @in.ReadVLong();
int docCount = @in.ReadVInt();
int longsSize = @in.ReadVInt();
TermsReader current = new TermsReader(this, fieldInfo, @in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize);
TermsReader previous = fields[fieldInfo.Name] = current;
CheckFieldSummary(state.SegmentInfo, @in, current, previous);
}
success = true;
}
finally
{
if (success)
{
IOUtils.Close(@in);
}
else
{
IOUtils.CloseWhileHandlingException(@in);
}
}
}
public override Bits ReadLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context)
{
Debug.Assert(info.HasDeletions());
var scratch = new BytesRef();
var scratchUtf16 = new CharsRef();
var fileName = IndexFileNames.FileNameFromGeneration(info.Info.Name, LIVEDOCS_EXTENSION, info.DelGen);
ChecksumIndexInput input = null;
var success = false;
try
{
input = dir.OpenChecksumInput(fileName, context);
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SIZE));
var size = ParseIntAt(scratch, SIZE.Length, scratchUtf16);
var bits = new BitArray(size);
SimpleTextUtil.ReadLine(input, scratch);
while (!scratch.Equals(END))
{
Debug.Assert(StringHelper.StartsWith(scratch, DOC));
var docid = ParseIntAt(scratch, DOC.Length, scratchUtf16);
bits.Set(docid, true);
SimpleTextUtil.ReadLine(input, scratch);
}
SimpleTextUtil.CheckFooter(input);
success = true;
return(new SimpleTextBits(bits, size));
}
finally
{
if (success)
{
IOUtils.Close(input);
}
else
{
IOUtils.CloseWhileHandlingException(input);
}
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
return;
}
try
{
IOUtils.Close(_input);
}
finally
{
_input = null;
_offsets = null;
}
}
public override void WriteLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount,
IOContext context)
{
var set = ((SimpleTextBits)bits).BITS;
var size = bits.Length();
var scratch = new BytesRef();
var fileName = IndexFileNames.FileNameFromGeneration(info.Info.Name, LIVEDOCS_EXTENSION, info.NextDelGen);
IndexOutput output = null;
var success = false;
try
{
output = dir.CreateOutput(fileName, context);
SimpleTextUtil.Write(output, SIZE);
SimpleTextUtil.Write(output, Convert.ToString(size), scratch);
SimpleTextUtil.WriteNewline(output);
for (int i = set.NextSetBit(0); i >= 0; i = set.NextSetBit(i + 1))
{
SimpleTextUtil.Write(output, DOC);
SimpleTextUtil.Write(output, Convert.ToString(i), scratch);
SimpleTextUtil.WriteNewline(output);
}
SimpleTextUtil.Write(output, END);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.WriteChecksum(output, scratch);
success = true;
}
finally
{
if (success)
{
IOUtils.Close(output);
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
public override FieldsProducer FieldsProducer(SegmentReadState state)
{
PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.Directory, state.FieldInfos,
state.SegmentInfo, state.Context, state.SegmentSuffix);
bool success = false;
try
{
FieldsProducer ret = new FSTTermsReader(state, postingsReader);
success = true;
return(ret);
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(postingsReader);
}
}
}
public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
{
PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state);
bool success = false;
try
{
FieldsConsumer ret = new FSTTermsWriter(state, postingsWriter);
success = true;
return(ret);
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(postingsWriter);
}
}
}
public override void Dispose()
{
if (_output == null)
{
return;
}
IOException ioe = null;
try
{
// write field summary
var dirStart = _output.FilePointer;
_output.WriteVInt(_fields.Count);
foreach (var field in _fields)
{
_output.WriteVInt(field.FieldInfo.Number);
_output.WriteVLong(field.NumTerms);
if (field.FieldInfo.FieldIndexOptions != IndexOptions.DOCS_ONLY)
{
_output.WriteVLong(field.SumTotalTermFreq);
}
_output.WriteVLong(field.SumDocFreq);
_output.WriteVInt(field.DocCount);
_output.WriteVInt(field.LongsSize);
field.Dict.Save(_output);
}
WriteTrailer(_output, dirStart);
CodecUtil.WriteFooter(_output);
}
catch (IOException ioe2)
{
ioe = ioe2;
}
finally
{
IOUtils.CloseWhileHandlingException(ioe, _output, _postingsWriter);
_output = null;
}
}
public override FieldsProducer FieldsProducer(SegmentReadState state)
{
PostingsReaderBase docsReader = null;
PostingsReaderBase pulsingReader = null;
bool success = false;
try
{
docsReader = _wrappedPostingsBaseFormat.PostingsReaderBase(state);
pulsingReader = new PulsingPostingsReader(state, docsReader);
FieldsProducer ret = new FSTOrdTermsReader(state, pulsingReader);
success = true;
return(ret);
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(docsReader, pulsingReader);
}
}
}
public SimpleTextFieldsReader(SegmentReadState state)
{
_maxDoc = state.SegmentInfo.DocCount;
_fieldInfos = state.FieldInfos;
_input =
state.Directory.OpenInput(
SimpleTextPostingsFormat.GetPostingsFileName(state.SegmentInfo.Name, state.SegmentSuffix),
state.Context);
bool success = false;
try
{
_fields = ReadFields((IndexInput)_input.Clone());
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException();
}
}
}
public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
{
PostingsWriterBase docsWriter = null;
PostingsWriterBase pulsingWriter = null;
bool success = false;
try
{
docsWriter = _wrappedPostingsBaseFormat.PostingsWriterBase(state);
pulsingWriter = new PulsingPostingsWriter(state, _freqCutoff, docsWriter);
FieldsConsumer ret = new FSTOrdTermsWriter(state, pulsingWriter);
success = true;
return(ret);
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(docsWriter, pulsingWriter);
}
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
// TODO: add a finish() at least to PushBase? DV too...?
bool success = false;
try
{
if (DocOut != null)
{
CodecUtil.WriteFooter(DocOut);
}
if (PosOut != null)
{
CodecUtil.WriteFooter(PosOut);
}
if (PayOut != null)
{
CodecUtil.WriteFooter(PayOut);
}
success = true;
}
finally
{
if (success)
{
IOUtils.Close(DocOut, PosOut, PayOut);
}
else
{
IOUtils.CloseWhileHandlingException(DocOut, PosOut, PayOut);
}
DocOut = PosOut = PayOut = null;
}
}
}
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
{
var segFileName = IndexFileNames.SegmentFileName(si.Name, "", SimpleTextSegmentInfoFormat.SI_EXTENSION);
si.AddFile(segFileName);
var success = false;
var output = dir.CreateOutput(segFileName, ioContext);
try
{
var scratch = new BytesRef();
SimpleTextUtil.Write(output, SI_VERSION);
SimpleTextUtil.Write(output, si.Version, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, SI_DOCCOUNT);
SimpleTextUtil.Write(output, Convert.ToString(si.DocCount, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, SI_USECOMPOUND);
SimpleTextUtil.Write(output, Convert.ToString(si.UseCompoundFile, CultureInfo.InvariantCulture).ToLowerInvariant(), scratch);
SimpleTextUtil.WriteNewline(output);
IDictionary <string, string> diagnostics = si.Diagnostics;
int numDiagnostics = diagnostics == null ? 0 : diagnostics.Count;
SimpleTextUtil.Write(output, SI_NUM_DIAG);
SimpleTextUtil.Write(output, Convert.ToString(numDiagnostics, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
if (numDiagnostics > 0)
{
foreach (var diagEntry in diagnostics)
{
SimpleTextUtil.Write(output, SI_DIAG_KEY);
SimpleTextUtil.Write(output, diagEntry.Key, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, SI_DIAG_VALUE);
SimpleTextUtil.Write(output, diagEntry.Value, scratch);
SimpleTextUtil.WriteNewline(output);
}
}
var files = si.Files;
var numFiles = files == null ? 0 : files.Count;
SimpleTextUtil.Write(output, SI_NUM_FILES);
SimpleTextUtil.Write(output, Convert.ToString(numFiles, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
if (numFiles > 0)
{
foreach (var fileName in files)
{
SimpleTextUtil.Write(output, SI_FILE);
SimpleTextUtil.Write(output, fileName, scratch);
SimpleTextUtil.WriteNewline(output);
}
}
SimpleTextUtil.WriteChecksum(output, scratch);
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(output);
try
{
dir.DeleteFile(segFileName);
}
catch (Exception)
{
//Esnure we throw original exeception
}
}
else
{
output.Dispose();
}
}
}
/// <summary>
/// Creates a postings writer with the specified PackedInts overhead ratio </summary>
// TODO: does this ctor even make sense?
public Lucene41PostingsWriter(SegmentWriteState state, float acceptableOverheadRatio)
: base()
{
DocOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), state.Context);
IndexOutput posOut = null;
IndexOutput payOut = null;
bool success = false;
try
{
CodecUtil.WriteHeader(DocOut, DOC_CODEC, VERSION_CURRENT);
ForUtil = new ForUtil(acceptableOverheadRatio, DocOut);
if (state.FieldInfos.HasProx())
{
PosDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
posOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), state.Context);
CodecUtil.WriteHeader(posOut, POS_CODEC, VERSION_CURRENT);
if (state.FieldInfos.HasPayloads())
{
PayloadBytes = new sbyte[128];
PayloadLengthBuffer = new int[ForUtil.MAX_DATA_SIZE];
}
else
{
PayloadBytes = null;
PayloadLengthBuffer = null;
}
if (state.FieldInfos.HasOffsets())
{
OffsetStartDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
OffsetLengthBuffer = new int[ForUtil.MAX_DATA_SIZE];
}
else
{
OffsetStartDeltaBuffer = null;
OffsetLengthBuffer = null;
}
if (state.FieldInfos.HasPayloads() || state.FieldInfos.HasOffsets())
{
payOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), state.Context);
CodecUtil.WriteHeader(payOut, PAY_CODEC, VERSION_CURRENT);
}
}
else
{
PosDeltaBuffer = null;
PayloadLengthBuffer = null;
OffsetStartDeltaBuffer = null;
OffsetLengthBuffer = null;
PayloadBytes = null;
}
this.PayOut = payOut;
this.PosOut = posOut;
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(DocOut, posOut, payOut);
}
}
DocDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
FreqBuffer = new int[ForUtil.MAX_DATA_SIZE];
// TODO: should we try skipping every 2/4 blocks...?
SkipWriter = new Lucene41SkipWriter(MaxSkipLevels, Lucene41PostingsFormat.BLOCK_SIZE, state.SegmentInfo.DocCount, DocOut, posOut, payOut);
Encoded = new byte[ForUtil.MAX_ENCODED_SIZE];
}
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos,
IOContext context)
{
var fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
var output = directory.CreateOutput(fileName, context);
var scratch = new BytesRef();
var success = false;
try
{
SimpleTextUtil.Write(output, NUMFIELDS);
SimpleTextUtil.Write(output, Convert.ToString(infos.Size()), scratch);
SimpleTextUtil.WriteNewline(output);
foreach (FieldInfo fi in infos)
{
SimpleTextUtil.Write(output, NAME);
SimpleTextUtil.Write(output, fi.Name, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NUMBER);
SimpleTextUtil.Write(output, Convert.ToString(fi.Number), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, ISINDEXED);
SimpleTextUtil.Write(output, Convert.ToString(fi.Indexed), scratch);
SimpleTextUtil.WriteNewline(output);
if (fi.Indexed)
{
Debug.Assert(fi.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads());
SimpleTextUtil.Write(output, INDEXOPTIONS);
SimpleTextUtil.Write(output, fi.FieldIndexOptions.ToString(), scratch);
SimpleTextUtil.WriteNewline(output);
}
SimpleTextUtil.Write(output, STORETV);
SimpleTextUtil.Write(output, Convert.ToString(fi.HasVectors()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, PAYLOADS);
SimpleTextUtil.Write(output, Convert.ToString(fi.HasPayloads()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NORMS);
SimpleTextUtil.Write(output, Convert.ToString(!fi.OmitsNorms()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NORMS_TYPE);
SimpleTextUtil.Write(output, GetDocValuesType(fi.NormType), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, DOCVALUES);
SimpleTextUtil.Write(output, GetDocValuesType(fi.DocValuesType), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, DOCVALUES_GEN);
SimpleTextUtil.Write(output, Convert.ToString(fi.DocValuesGen), scratch);
SimpleTextUtil.WriteNewline(output);
IDictionary <string, string> atts = fi.Attributes();
int numAtts = atts == null ? 0 : atts.Count;
SimpleTextUtil.Write(output, NUM_ATTS);
SimpleTextUtil.Write(output, Convert.ToString(numAtts), scratch);
SimpleTextUtil.WriteNewline(output);
if (numAtts <= 0 || atts == null)
{
continue;
}
foreach (var entry in atts)
{
SimpleTextUtil.Write(output, ATT_KEY);
SimpleTextUtil.Write(output, entry.Key, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, ATT_VALUE);
SimpleTextUtil.Write(output, entry.Value, scratch);
SimpleTextUtil.WriteNewline(output);
}
}
SimpleTextUtil.WriteChecksum(output, scratch);
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
public override SegmentInfo Read(Directory directory, string segmentName, IOContext context)
{
BytesRef scratch = new BytesRef();
string segFileName = IndexFileNames.SegmentFileName(segmentName, "",
SimpleTextSegmentInfoFormat.SI_EXTENSION);
ChecksumIndexInput input = directory.OpenChecksumInput(segFileName, context);
bool success = false;
try
{
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_VERSION));
string version = ReadString(SI_VERSION.length, scratch);
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_DOCCOUNT));
int docCount = Convert.ToInt32(ReadString(SI_DOCCOUNT.length, scratch));
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_USECOMPOUND));
bool isCompoundFile = Convert.ToBoolean(ReadString(SI_USECOMPOUND.length, scratch));
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_NUM_DIAG));
int numDiag = Convert.ToInt32(ReadString(SI_NUM_DIAG.length, scratch));
IDictionary <string, string> diagnostics = new Dictionary <string, string>();
for (int i = 0; i < numDiag; i++)
{
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_DIAG_KEY));
string key = ReadString(SI_DIAG_KEY.length, scratch);
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_DIAG_VALUE));
string value = ReadString(SI_DIAG_VALUE.length, scratch);
diagnostics[key] = value;
}
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_NUM_FILES));
int numFiles = Convert.ToInt32(ReadString(SI_NUM_FILES.length, scratch));
HashSet <string> files = new HashSet <string>();
for (int i = 0; i < numFiles; i++)
{
SimpleTextUtil.ReadLine(input, scratch);
Debug.Assert(StringHelper.StartsWith(scratch, SI_FILE));
string fileName = ReadString(SI_FILE.length, scratch);
files.Add(fileName);
}
SimpleTextUtil.CheckFooter(input);
SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount, isCompoundFile, null,
diagnostics);
info.Files = files;
success = true;
return(info);
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(input);
}
else
{
input.Close();
}
}
}
