| public static Write ( Store.DataOutput output, Util.BytesRef b ) : void | ||
| output | Store.DataOutput | |
| b | Util.BytesRef | |
| return | void | |
protected override void Dispose(bool disposing)
{
if (data == null || !disposing)
{
return;
}
var success = false;
try
{
Debug.Assert(_fieldsSeen.Count > 0);
// java : sheisty to do this here?
SimpleTextUtil.Write(data, END);
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.WriteChecksum(data, scratch);
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(data);
}
else
{
IOUtils.DisposeWhileHandlingException(data);
}
data = null;
}
}
/// <summary>Write the header for this field. </summary>
private void WriteFieldEntry(FieldInfo field, DocValuesType type)
{
SimpleTextUtil.Write(data, FIELD);
SimpleTextUtil.Write(data, field.Name, scratch);
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.Write(data, TYPE);
SimpleTextUtil.Write(data, type.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
}
public override void WriteLiveDocs(IMutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount,
IOContext context)
{
var set = ((SimpleTextBits)bits).bits;
var size = bits.Length;
var scratch = new BytesRef();
var fileName = IndexFileNames.FileNameFromGeneration(info.Info.Name, LIVEDOCS_EXTENSION, info.NextDelGen);
IndexOutput output = null;
var success = false;
try
{
output = dir.CreateOutput(fileName, context);
SimpleTextUtil.Write(output, SIZE);
SimpleTextUtil.Write(output, Convert.ToString(size, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
for (int i = set.NextSetBit(0); i >= 0; i = set.NextSetBit(i + 1))
{
SimpleTextUtil.Write(output, DOC);
SimpleTextUtil.Write(output, Convert.ToString(i, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
}
SimpleTextUtil.Write(output, END);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.WriteChecksum(output, scratch);
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(output);
}
else
{
IOUtils.DisposeWhileHandlingException(output);
}
}
}
public override void AddSortedSetField(FieldInfo field, IEnumerable <BytesRef> values,
IEnumerable <long?> docToOrdCount, IEnumerable <long?> ords)
{
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == DocValuesType.SORTED_SET);
WriteFieldEntry(field, DocValuesType.SORTED_SET);
long valueCount = 0;
int maxLength = 0;
foreach (var value in values)
{
maxLength = Math.Max(maxLength, value.Length);
valueCount++;
}
// write numValues
SimpleTextUtil.Write(data, NUMVALUES);
SimpleTextUtil.Write(data, valueCount.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
// write maxLength
SimpleTextUtil.Write(data, MAXLENGTH);
SimpleTextUtil.Write(data, maxLength.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
int maxBytesLength = maxLength.ToString(CultureInfo.InvariantCulture).Length;
var sb = new StringBuilder();
for (int i = 0; i < maxBytesLength; i++)
{
sb.Append('0');
}
// write our pattern for encoding lengths
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
string encoderFormat = sb.ToString();
// compute ord pattern: this is funny, we encode all values for all docs to find the maximum length
var maxOrdListLength = 0;
var sb2 = new StringBuilder();
var ordStream = ords.GetEnumerator();
foreach (var n in docToOrdCount)
{
sb2.Length = 0;
var count = (int)n;
for (int i = 0; i < count; i++)
{
ordStream.MoveNext();
var ord = ordStream.Current;
if (sb2.Length > 0)
{
sb2.Append(",");
}
sb2.Append(ord.GetValueOrDefault().ToString(CultureInfo.InvariantCulture));
}
maxOrdListLength = Math.Max(maxOrdListLength, sb2.Length);
}
sb2.Length = 0;
for (int i = 0; i < maxOrdListLength; i++)
{
sb2.Append('X');
}
// write our pattern for ord lists
SimpleTextUtil.Write(data, ORDPATTERN);
SimpleTextUtil.Write(data, sb2.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
// for asserts:
long valuesSeen = 0;
foreach (var value in values)
{
// write length
SimpleTextUtil.Write(data, LENGTH);
SimpleTextUtil.Write(data, value.Length.ToString(encoderFormat, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
// write bytes -- don't use SimpleText.Write
// because it escapes:
data.WriteBytes(value.Bytes, value.Offset, value.Length);
// pad to fit
for (var i = value.Length; i < maxLength; i++)
{
data.WriteByte((byte)' ');
}
SimpleTextUtil.WriteNewline(data);
valuesSeen++;
Debug.Assert(valuesSeen <= valueCount);
}
Debug.Assert(valuesSeen == valueCount);
ordStream = ords.GetEnumerator();
// write the ords for each doc comma-separated
foreach (var n in docToOrdCount)
{
sb2.Length = 0;
var count = (int)n;
for (var i = 0; i < count; i++)
{
ordStream.MoveNext();
var ord = ordStream.Current;
if (sb2.Length > 0)
{
sb2.Append(",");
}
sb2.Append(ord);
}
// now pad to fit: these are numbers so spaces work well. reader calls trim()
var numPadding = maxOrdListLength - sb2.Length;
for (var i = 0; i < numPadding; i++)
{
sb2.Append(' ');
}
SimpleTextUtil.Write(data, sb2.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
}
}
public override void AddSortedField(FieldInfo field, IEnumerable <BytesRef> values, IEnumerable <long?> docToOrd)
{
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == DocValuesType.SORTED);
WriteFieldEntry(field, DocValuesType.SORTED);
int valueCount = 0;
int maxLength = -1;
foreach (BytesRef value in values)
{
maxLength = Math.Max(maxLength, value.Length);
valueCount++;
}
// write numValues
SimpleTextUtil.Write(data, NUMVALUES);
SimpleTextUtil.Write(data, valueCount.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
// write maxLength
SimpleTextUtil.Write(data, MAXLENGTH);
SimpleTextUtil.Write(data, maxLength.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
int maxBytesLength = maxLength.ToString(CultureInfo.InvariantCulture).Length;
var sb = new StringBuilder();
for (int i = 0; i < maxBytesLength; i++)
{
sb.Append('0');
}
// write our pattern for encoding lengths
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
var encoderFormat = sb.ToString();
int maxOrdBytes = (valueCount + 1L).ToString(CultureInfo.InvariantCulture).Length;
sb.Length = 0;
for (int i = 0; i < maxOrdBytes; i++)
{
sb.Append('0');
}
// write our pattern for ords
SimpleTextUtil.Write(data, ORDPATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
var ordEncoderFormat = sb.ToString();
// for asserts:
int valuesSeen = 0;
foreach (BytesRef value in values)
{
// write length
SimpleTextUtil.Write(data, LENGTH);
SimpleTextUtil.Write(data, value.Length.ToString(encoderFormat, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
// write bytes -- don't use SimpleText.Write
// because it escapes:
data.WriteBytes(value.Bytes, value.Offset, value.Length);
// pad to fit
for (int i = value.Length; i < maxLength; i++)
{
data.WriteByte((byte)' ');
}
SimpleTextUtil.WriteNewline(data);
valuesSeen++;
Debug.Assert(valuesSeen <= valueCount);
}
Debug.Assert(valuesSeen == valueCount);
foreach (var ord in docToOrd)
{
SimpleTextUtil.Write(data, (ord + 1).GetValueOrDefault().ToString(ordEncoderFormat, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
}
}
public override void AddBinaryField(FieldInfo field, IEnumerable <BytesRef> values)
{
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == DocValuesType.BINARY);
var maxLength = 0;
foreach (var value in values)
{
var length = value == null ? 0 : value.Length;
maxLength = Math.Max(maxLength, length);
}
WriteFieldEntry(field, DocValuesType.BINARY);
// write maxLength
SimpleTextUtil.Write(data, MAXLENGTH);
SimpleTextUtil.Write(data, maxLength.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
var maxBytesLength = maxLength.ToString(CultureInfo.InvariantCulture).Length;
var sb = new StringBuilder();
for (var i = 0; i < maxBytesLength; i++)
{
sb.Append('0');
}
// write our pattern for encoding lengths
var patternString = sb.ToString();
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, patternString, scratch);
SimpleTextUtil.WriteNewline(data);
int numDocsWritten = 0;
foreach (BytesRef value in values)
{
int length = value == null ? 0 : value.Length;
SimpleTextUtil.Write(data, LENGTH);
SimpleTextUtil.Write(data, length.ToString(patternString, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
// write bytes -- don't use SimpleText.Write
// because it escapes:
if (value != null)
{
data.WriteBytes(value.Bytes, value.Offset, value.Length);
}
// pad to fit
for (int i = length; i < maxLength; i++)
{
data.WriteByte((byte)(sbyte)' ');
}
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.Write(data, value == null ? "F" : "T", scratch);
SimpleTextUtil.WriteNewline(data);
numDocsWritten++;
}
Debug.Assert(numDocs == numDocsWritten);
}
private void Write(BytesRef bytes)
{
SimpleTextUtil.Write(_output, bytes);
}
public override void AddSortedField(FieldInfo field, IEnumerable <BytesRef> values, IEnumerable <long> docToOrd)
{
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == FieldInfo.DocValuesType_e.SORTED);
WriteFieldEntry(field, FieldInfo.DocValuesType_e.SORTED);
int valueCount = 0;
int maxLength = -1;
foreach (BytesRef value in values)
{
maxLength = Math.Max(maxLength, value.Length);
valueCount++;
}
// write numValues
SimpleTextUtil.Write(data, NUMVALUES);
SimpleTextUtil.Write(data, Convert.ToString(valueCount), scratch);
SimpleTextUtil.WriteNewline(data);
// write maxLength
SimpleTextUtil.Write(data, MAXLENGTH);
SimpleTextUtil.Write(data, Convert.ToString(maxLength), scratch);
SimpleTextUtil.WriteNewline(data);
int maxBytesLength = Convert.ToString(maxLength).Length;
StringBuilder sb = new StringBuilder();
for (int i = 0; i < maxBytesLength; i++)
{
sb.Append('0');
}
// write our pattern for encoding lengths
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
DecimalFormat encoder = new DecimalFormat(sb.ToString(), new DecimalFormatSymbols(Locale.ROOT));
int maxOrdBytes = Convert.ToString(valueCount + 1L).Length;
sb.Length = 0;
for (int i = 0; i < maxOrdBytes; i++)
{
sb.Append('0');
}
// write our pattern for ords
SimpleTextUtil.Write(data, ORDPATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
DecimalFormat ordEncoder = new DecimalFormat(sb.ToString(), new DecimalFormatSymbols(Locale.ROOT));
// for asserts:
int valuesSeen = 0;
foreach (BytesRef value in values)
{
// write length
SimpleTextUtil.Write(data, LENGTH);
SimpleTextUtil.Write(data, encoder.format(value.Length), scratch);
SimpleTextUtil.WriteNewline(data);
// write bytes -- don't use SimpleText.Write
// because it escapes:
data.WriteBytes(value.Bytes, value.Offset, value.Length);
// pad to fit
for (int i = value.Length; i < maxLength; i++)
{
data.WriteByte((sbyte)' ');
}
SimpleTextUtil.WriteNewline(data);
valuesSeen++;
Debug.Assert(valuesSeen <= valueCount);
}
Debug.Assert(valuesSeen == valueCount);
foreach (var ord in docToOrd)
{
SimpleTextUtil.Write(data, ordEncoder.format(ord + 1), scratch);
SimpleTextUtil.WriteNewline(data);
}
}
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos,
IOContext context)
{
var fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
var output = directory.CreateOutput(fileName, context);
var scratch = new BytesRef();
var success = false;
try
{
SimpleTextUtil.Write(output, NUMFIELDS);
SimpleTextUtil.Write(output, Convert.ToString(infos.Size()), scratch);
SimpleTextUtil.WriteNewline(output);
foreach (FieldInfo fi in infos)
{
SimpleTextUtil.Write(output, NAME);
SimpleTextUtil.Write(output, fi.Name, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NUMBER);
SimpleTextUtil.Write(output, Convert.ToString(fi.Number), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, ISINDEXED);
SimpleTextUtil.Write(output, Convert.ToString(fi.Indexed), scratch);
SimpleTextUtil.WriteNewline(output);
if (fi.Indexed)
{
Debug.Assert(fi.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads());
SimpleTextUtil.Write(output, INDEXOPTIONS);
SimpleTextUtil.Write(output, fi.FieldIndexOptions.ToString(), scratch);
SimpleTextUtil.WriteNewline(output);
}
SimpleTextUtil.Write(output, STORETV);
SimpleTextUtil.Write(output, Convert.ToString(fi.HasVectors()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, PAYLOADS);
SimpleTextUtil.Write(output, Convert.ToString(fi.HasPayloads()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NORMS);
SimpleTextUtil.Write(output, Convert.ToString(!fi.OmitsNorms()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NORMS_TYPE);
SimpleTextUtil.Write(output, GetDocValuesType(fi.NormType), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, DOCVALUES);
SimpleTextUtil.Write(output, GetDocValuesType(fi.DocValuesType), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, DOCVALUES_GEN);
SimpleTextUtil.Write(output, Convert.ToString(fi.DocValuesGen), scratch);
SimpleTextUtil.WriteNewline(output);
IDictionary <string, string> atts = fi.Attributes();
int numAtts = atts == null ? 0 : atts.Count;
SimpleTextUtil.Write(output, NUM_ATTS);
SimpleTextUtil.Write(output, Convert.ToString(numAtts), scratch);
SimpleTextUtil.WriteNewline(output);
if (numAtts <= 0 || atts == null)
{
continue;
}
foreach (var entry in atts)
{
SimpleTextUtil.Write(output, ATT_KEY);
SimpleTextUtil.Write(output, entry.Key, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, ATT_VALUE);
SimpleTextUtil.Write(output, entry.Value, scratch);
SimpleTextUtil.WriteNewline(output);
}
}
SimpleTextUtil.WriteChecksum(output, scratch);
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.CloseWhileHandlingException(output);
}
}
}
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext)
{
var segFileName = IndexFileNames.SegmentFileName(si.Name, "", SimpleTextSegmentInfoFormat.SI_EXTENSION);
si.AddFile(segFileName);
var success = false;
var output = dir.CreateOutput(segFileName, ioContext);
try
{
var scratch = new BytesRef();
SimpleTextUtil.Write(output, SI_VERSION);
SimpleTextUtil.Write(output, si.Version, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, SI_DOCCOUNT);
SimpleTextUtil.Write(output, Convert.ToString(si.DocCount, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, SI_USECOMPOUND);
SimpleTextUtil.Write(output, Convert.ToString(si.UseCompoundFile, CultureInfo.InvariantCulture).ToLowerInvariant(), scratch);
SimpleTextUtil.WriteNewline(output);
IDictionary <string, string> diagnostics = si.Diagnostics;
int numDiagnostics = diagnostics is null ? 0 : diagnostics.Count;
SimpleTextUtil.Write(output, SI_NUM_DIAG);
SimpleTextUtil.Write(output, Convert.ToString(numDiagnostics, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
if (numDiagnostics > 0)
{
foreach (var diagEntry in diagnostics)
{
SimpleTextUtil.Write(output, SI_DIAG_KEY);
SimpleTextUtil.Write(output, diagEntry.Key, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, SI_DIAG_VALUE);
SimpleTextUtil.Write(output, diagEntry.Value, scratch);
SimpleTextUtil.WriteNewline(output);
}
}
var files = si.GetFiles();
var numFiles = files is null ? 0 : files.Count;
SimpleTextUtil.Write(output, SI_NUM_FILES);
SimpleTextUtil.Write(output, Convert.ToString(numFiles, CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
if (numFiles > 0)
{
foreach (var fileName in files)
{
SimpleTextUtil.Write(output, SI_FILE);
SimpleTextUtil.Write(output, fileName, scratch);
SimpleTextUtil.WriteNewline(output);
}
}
SimpleTextUtil.WriteChecksum(output, scratch);
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(output);
try
{
dir.DeleteFile(segFileName);
}
catch (Exception t) when(t.IsThrowable())
{
//Esnure we throw original exeception
}
}
else
{
output.Dispose();
}
}
}
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos,
IOContext context)
{
var fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
var output = directory.CreateOutput(fileName, context);
var scratch = new BytesRef();
var success = false;
try
{
SimpleTextUtil.Write(output, NUMFIELDS);
SimpleTextUtil.Write(output, infos.Count.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
foreach (FieldInfo fi in infos)
{
SimpleTextUtil.Write(output, NAME);
SimpleTextUtil.Write(output, fi.Name, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NUMBER);
SimpleTextUtil.Write(output, fi.Number.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, ISINDEXED);
SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower(fi.IsIndexed.ToString()), scratch);
SimpleTextUtil.WriteNewline(output);
if (fi.IsIndexed)
{
// LUCENENET specific - to avoid boxing, changed from CompareTo() to IndexOptionsComparer.Compare()
if (Debugging.AssertsEnabled)
{
Debugging.Assert(IndexOptionsComparer.Default.Compare(fi.IndexOptions, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.HasPayloads);
}
SimpleTextUtil.Write(output, INDEXOPTIONS);
SimpleTextUtil.Write(output,
fi.IndexOptions != IndexOptions.NONE ? fi.IndexOptions.ToString() : string.Empty,
scratch);
SimpleTextUtil.WriteNewline(output);
}
SimpleTextUtil.Write(output, STORETV);
SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower(fi.HasVectors.ToString()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, PAYLOADS);
SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower(fi.HasPayloads.ToString()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NORMS);
SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower((!fi.OmitsNorms).ToString()), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, NORMS_TYPE);
SimpleTextUtil.Write(output, GetDocValuesType(fi.NormType), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, DOCVALUES);
SimpleTextUtil.Write(output, GetDocValuesType(fi.DocValuesType), scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, DOCVALUES_GEN);
SimpleTextUtil.Write(output, fi.DocValuesGen.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
IDictionary <string, string> atts = fi.Attributes;
int numAtts = atts == null ? 0 : atts.Count;
SimpleTextUtil.Write(output, NUM_ATTS);
SimpleTextUtil.Write(output, numAtts.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(output);
if (numAtts <= 0 || atts == null)
{
continue;
}
foreach (var entry in atts)
{
SimpleTextUtil.Write(output, ATT_KEY);
SimpleTextUtil.Write(output, entry.Key, scratch);
SimpleTextUtil.WriteNewline(output);
SimpleTextUtil.Write(output, ATT_VALUE);
SimpleTextUtil.Write(output, entry.Value, scratch);
SimpleTextUtil.WriteNewline(output);
}
}
SimpleTextUtil.WriteChecksum(output, scratch);
success = true;
}
finally
{
if (success)
{
output.Dispose();
}
else
{
IOUtils.DisposeWhileHandlingException(output);
}
}
}
public override void AddNumericField(FieldInfo field, IEnumerable <long?> values)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(FieldSeen(field.Name));
Debugging.Assert(field.DocValuesType == DocValuesType.NUMERIC ||
field.NormType == DocValuesType.NUMERIC);
}
WriteFieldEntry(field, DocValuesType.NUMERIC);
// first pass to find min/max
var minValue = long.MaxValue;
var maxValue = long.MinValue;
foreach (var n in values)
{
var v = n.GetValueOrDefault();
minValue = Math.Min(minValue, v);
maxValue = Math.Max(maxValue, v);
}
// write our minimum value to the .dat, all entries are deltas from that
SimpleTextUtil.Write(data, MINVALUE);
SimpleTextUtil.Write(data, minValue.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
// build up our fixed-width "simple text packed ints" format
var diffBig = (decimal)maxValue - (decimal)minValue; // LUCENENET specific - use decimal rather than BigInteger
var maxBytesPerValue = diffBig.ToString(CultureInfo.InvariantCulture).Length;
var sb = new StringBuilder();
for (var i = 0; i < maxBytesPerValue; i++)
{
sb.Append('0');
}
var patternString = sb.ToString(); // LUCENENET specific - only get the string once
// write our pattern to the .dat
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, patternString, scratch);
SimpleTextUtil.WriteNewline(data);
int numDocsWritten = 0;
// second pass to write the values
foreach (var n in values)
{
long value = n.GetValueOrDefault();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(value >= minValue);
}
var delta = (decimal)value - (decimal)minValue; // LUCENENET specific - use decimal rather than BigInteger
string s = delta.ToString(patternString, CultureInfo.InvariantCulture);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(s.Length == patternString.Length);
}
SimpleTextUtil.Write(data, s, scratch);
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.Write(data, n is null ? "F" : "T", scratch);
SimpleTextUtil.WriteNewline(data);
numDocsWritten++;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numDocsWritten <= numDocs);
}
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(numDocs == numDocsWritten, "numDocs={0} numDocsWritten={1}", numDocs, numDocsWritten);
}
}
private void Write(BytesRef b)
{
SimpleTextUtil.Write(output, b);
}
public override void AddNumericField(FieldInfo field, IEnumerable <long> values)
{
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == FieldInfo.DocValuesType_e.NUMERIC ||
field.NormType == FieldInfo.DocValuesType_e.NUMERIC);
WriteFieldEntry(field, FieldInfo.DocValuesType_e.NUMERIC);
// first pass to find min/max
var minValue = long.MaxValue;
var maxValue = long.MinValue;
foreach (var n in values)
{
var v = n;
minValue = Math.Min(minValue, v);
maxValue = Math.Max(maxValue, v);
}
// write our minimum value to the .dat, all entries are deltas from that
SimpleTextUtil.Write(data, MINVALUE);
SimpleTextUtil.Write(data, Convert.ToString(minValue), scratch);
SimpleTextUtil.WriteNewline(data);
// build up our fixed-width "simple text packed ints" format
System.Numerics.BigInteger maxBig = maxValue;
System.Numerics.BigInteger minBig = minValue;
var diffBig = maxBig - minBig;
var maxBytesPerValue = diffBig.ToString().Length;
var sb = new StringBuilder();
for (var i = 0; i < maxBytesPerValue; i++)
{
sb.Append('0');
}
// write our pattern to the .dat
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
var patternString = sb.ToString();
DecimalFormat encoder = new DecimalFormat(patternString, new DecimalFormatSymbols(Locale.ROOT));
int numDocsWritten = 0;
// second pass to write the values
foreach (var value in values)
{
Debug.Assert(value >= minValue);
var delta = value - minValue;
string s = encoder.format(delta);
Debug.Assert(s.Length == patternString.Length);
SimpleTextUtil.Write(data, s, scratch);
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.Write(data, n == null ? "F" : "T", scratch);
SimpleTextUtil.WriteNewline(data);
numDocsWritten++;
Debug.Assert(numDocsWritten <= numDocs);
}
Debug.Assert(numDocs == numDocsWritten, "numDocs=" + numDocs + " numDocsWritten=" + numDocsWritten);
}
public override void AddNumericField(FieldInfo field, IEnumerable <long?> values)
{
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == DocValuesType.NUMERIC ||
field.NormType == DocValuesType.NUMERIC);
WriteFieldEntry(field, DocValuesType.NUMERIC);
// first pass to find min/max
var minValue = long.MaxValue;
var maxValue = long.MinValue;
foreach (var n in values)
{
var v = n.GetValueOrDefault();
minValue = Math.Min(minValue, v);
maxValue = Math.Max(maxValue, v);
}
// write our minimum value to the .dat, all entries are deltas from that
SimpleTextUtil.Write(data, MINVALUE);
SimpleTextUtil.Write(data, minValue.ToString(CultureInfo.InvariantCulture), scratch);
SimpleTextUtil.WriteNewline(data);
// build up our fixed-width "simple text packed ints" format
BigInteger maxBig = maxValue;
BigInteger minBig = minValue;
var diffBig = BigInteger.Subtract(maxBig, minBig);
var maxBytesPerValue = diffBig.ToString(CultureInfo.InvariantCulture).Length;
var sb = new StringBuilder();
for (var i = 0; i < maxBytesPerValue; i++)
{
sb.Append('0');
}
// write our pattern to the .dat
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
var patternString = sb.ToString();
int numDocsWritten = 0;
// second pass to write the values
foreach (var n in values)
{
long value = n.GetValueOrDefault();
Debug.Assert(value >= minValue);
var delta = BigInteger.Subtract(value, minValue);
string s = delta.ToString(patternString, CultureInfo.InvariantCulture);
Debug.Assert(s.Length == patternString.Length);
SimpleTextUtil.Write(data, s, scratch);
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.Write(data, n == null ? "F" : "T", scratch);
SimpleTextUtil.WriteNewline(data);
numDocsWritten++;
Debug.Assert(numDocsWritten <= numDocs);
}
Debug.Assert(numDocs == numDocsWritten, "numDocs=" + numDocs + " numDocsWritten=" + numDocsWritten);
}
private void Write(string s)
{
SimpleTextUtil.Write(_output, s, _scratch);
}
public override void AddBinaryField(FieldInfo field, IEnumerable <BytesRef> values)
{
Debug.Assert(FieldSeen(field.Name));
Debug.Assert(field.DocValuesType == FieldInfo.DocValuesType_e.BINARY);
var maxLength = 0;
foreach (var value in values)
{
var length = value == null ? 0 : value.Length;
maxLength = Math.Max(maxLength, length);
}
WriteFieldEntry(field, FieldInfo.DocValuesType_e.BINARY);
// write maxLength
SimpleTextUtil.Write(data, MAXLENGTH);
SimpleTextUtil.Write(data, Convert.ToString(maxLength), scratch);
SimpleTextUtil.WriteNewline(data);
var maxBytesLength = Convert.ToString(maxLength).Length;
var sb = new StringBuilder();
for (var i = 0; i < maxBytesLength; i++)
{
sb.Append('0');
}
// write our pattern for encoding lengths
SimpleTextUtil.Write(data, PATTERN);
SimpleTextUtil.Write(data, sb.ToString(), scratch);
SimpleTextUtil.WriteNewline(data);
DecimalFormat encoder = new DecimalFormat(sb.ToString(), new DecimalFormatSymbols(Locale.ROOT));
int numDocsWritten = 0;
foreach (BytesRef value in values)
{
int length = value == null ? 0 : value.Length;
SimpleTextUtil.Write(data, LENGTH);
SimpleTextUtil.Write(data, encoder.format(length), scratch);
SimpleTextUtil.WriteNewline(data);
// write bytes -- don't use SimpleText.Write
// because it escapes:
if (value != null)
{
data.WriteBytes(value.Bytes, value.Offset, value.Length);
}
// pad to fit
for (int i = length; i < maxLength; i++)
{
data.WriteByte((sbyte)' ');
}
SimpleTextUtil.WriteNewline(data);
SimpleTextUtil.Write(data, value == null ? "F" : "T", scratch);
SimpleTextUtil.WriteNewline(data);
numDocsWritten++;
}
Debug.Assert(numDocs == numDocsWritten);
}