public static Write ( Store.DataOutput output, Util.BytesRef b ) : void | ||
output | Store.DataOutput | |
b | Util.BytesRef | |
return | void |
protected override void Dispose(bool disposing) { if (data == null || !disposing) { return; } var success = false; try { Debug.Assert(_fieldsSeen.Count > 0); // java : sheisty to do this here? SimpleTextUtil.Write(data, END); SimpleTextUtil.WriteNewline(data); SimpleTextUtil.WriteChecksum(data, scratch); success = true; } finally { if (success) { IOUtils.Dispose(data); } else { IOUtils.DisposeWhileHandlingException(data); } data = null; } }
/// <summary>Write the header for this field. </summary> private void WriteFieldEntry(FieldInfo field, DocValuesType type) { SimpleTextUtil.Write(data, FIELD); SimpleTextUtil.Write(data, field.Name, scratch); SimpleTextUtil.WriteNewline(data); SimpleTextUtil.Write(data, TYPE); SimpleTextUtil.Write(data, type.ToString(), scratch); SimpleTextUtil.WriteNewline(data); }
public override void WriteLiveDocs(IMutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) { var set = ((SimpleTextBits)bits).bits; var size = bits.Length; var scratch = new BytesRef(); var fileName = IndexFileNames.FileNameFromGeneration(info.Info.Name, LIVEDOCS_EXTENSION, info.NextDelGen); IndexOutput output = null; var success = false; try { output = dir.CreateOutput(fileName, context); SimpleTextUtil.Write(output, SIZE); SimpleTextUtil.Write(output, Convert.ToString(size, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); for (int i = set.NextSetBit(0); i >= 0; i = set.NextSetBit(i + 1)) { SimpleTextUtil.Write(output, DOC); SimpleTextUtil.Write(output, Convert.ToString(i, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); } SimpleTextUtil.Write(output, END); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.WriteChecksum(output, scratch); success = true; } finally { if (success) { IOUtils.Dispose(output); } else { IOUtils.DisposeWhileHandlingException(output); } } }
public override void AddSortedSetField(FieldInfo field, IEnumerable <BytesRef> values, IEnumerable <long?> docToOrdCount, IEnumerable <long?> ords) { Debug.Assert(FieldSeen(field.Name)); Debug.Assert(field.DocValuesType == DocValuesType.SORTED_SET); WriteFieldEntry(field, DocValuesType.SORTED_SET); long valueCount = 0; int maxLength = 0; foreach (var value in values) { maxLength = Math.Max(maxLength, value.Length); valueCount++; } // write numValues SimpleTextUtil.Write(data, NUMVALUES); SimpleTextUtil.Write(data, valueCount.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); // write maxLength SimpleTextUtil.Write(data, MAXLENGTH); SimpleTextUtil.Write(data, maxLength.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); int maxBytesLength = maxLength.ToString(CultureInfo.InvariantCulture).Length; var sb = new StringBuilder(); for (int i = 0; i < maxBytesLength; i++) { sb.Append('0'); } // write our pattern for encoding lengths SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); string encoderFormat = sb.ToString(); // compute ord pattern: this is funny, we encode all values for all docs to find the maximum length var maxOrdListLength = 0; var sb2 = new StringBuilder(); var ordStream = ords.GetEnumerator(); foreach (var n in docToOrdCount) { sb2.Length = 0; var count = (int)n; for (int i = 0; i < count; i++) { ordStream.MoveNext(); var ord = ordStream.Current; if (sb2.Length > 0) { sb2.Append(","); } sb2.Append(ord.GetValueOrDefault().ToString(CultureInfo.InvariantCulture)); } maxOrdListLength = Math.Max(maxOrdListLength, sb2.Length); } sb2.Length = 0; for (int i = 0; i < maxOrdListLength; i++) { sb2.Append('X'); } // write our pattern for ord lists SimpleTextUtil.Write(data, ORDPATTERN); SimpleTextUtil.Write(data, sb2.ToString(), scratch); SimpleTextUtil.WriteNewline(data); // for asserts: long valuesSeen = 0; foreach (var value in values) { // write length SimpleTextUtil.Write(data, LENGTH); SimpleTextUtil.Write(data, value.Length.ToString(encoderFormat, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); // write bytes -- don't use SimpleText.Write // because it escapes: data.WriteBytes(value.Bytes, value.Offset, value.Length); // pad to fit for (var i = value.Length; i < maxLength; i++) { data.WriteByte((byte)' '); } SimpleTextUtil.WriteNewline(data); valuesSeen++; Debug.Assert(valuesSeen <= valueCount); } Debug.Assert(valuesSeen == valueCount); ordStream = ords.GetEnumerator(); // write the ords for each doc comma-separated foreach (var n in docToOrdCount) { sb2.Length = 0; var count = (int)n; for (var i = 0; i < count; i++) { ordStream.MoveNext(); var ord = ordStream.Current; if (sb2.Length > 0) { sb2.Append(","); } sb2.Append(ord); } // now pad to fit: these are numbers so spaces work well. reader calls trim() var numPadding = maxOrdListLength - sb2.Length; for (var i = 0; i < numPadding; i++) { sb2.Append(' '); } SimpleTextUtil.Write(data, sb2.ToString(), scratch); SimpleTextUtil.WriteNewline(data); } }
public override void AddSortedField(FieldInfo field, IEnumerable <BytesRef> values, IEnumerable <long?> docToOrd) { Debug.Assert(FieldSeen(field.Name)); Debug.Assert(field.DocValuesType == DocValuesType.SORTED); WriteFieldEntry(field, DocValuesType.SORTED); int valueCount = 0; int maxLength = -1; foreach (BytesRef value in values) { maxLength = Math.Max(maxLength, value.Length); valueCount++; } // write numValues SimpleTextUtil.Write(data, NUMVALUES); SimpleTextUtil.Write(data, valueCount.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); // write maxLength SimpleTextUtil.Write(data, MAXLENGTH); SimpleTextUtil.Write(data, maxLength.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); int maxBytesLength = maxLength.ToString(CultureInfo.InvariantCulture).Length; var sb = new StringBuilder(); for (int i = 0; i < maxBytesLength; i++) { sb.Append('0'); } // write our pattern for encoding lengths SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); var encoderFormat = sb.ToString(); int maxOrdBytes = (valueCount + 1L).ToString(CultureInfo.InvariantCulture).Length; sb.Length = 0; for (int i = 0; i < maxOrdBytes; i++) { sb.Append('0'); } // write our pattern for ords SimpleTextUtil.Write(data, ORDPATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); var ordEncoderFormat = sb.ToString(); // for asserts: int valuesSeen = 0; foreach (BytesRef value in values) { // write length SimpleTextUtil.Write(data, LENGTH); SimpleTextUtil.Write(data, value.Length.ToString(encoderFormat, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); // write bytes -- don't use SimpleText.Write // because it escapes: data.WriteBytes(value.Bytes, value.Offset, value.Length); // pad to fit for (int i = value.Length; i < maxLength; i++) { data.WriteByte((byte)' '); } SimpleTextUtil.WriteNewline(data); valuesSeen++; Debug.Assert(valuesSeen <= valueCount); } Debug.Assert(valuesSeen == valueCount); foreach (var ord in docToOrd) { SimpleTextUtil.Write(data, (ord + 1).GetValueOrDefault().ToString(ordEncoderFormat, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); } }
public override void AddBinaryField(FieldInfo field, IEnumerable <BytesRef> values) { Debug.Assert(FieldSeen(field.Name)); Debug.Assert(field.DocValuesType == DocValuesType.BINARY); var maxLength = 0; foreach (var value in values) { var length = value == null ? 0 : value.Length; maxLength = Math.Max(maxLength, length); } WriteFieldEntry(field, DocValuesType.BINARY); // write maxLength SimpleTextUtil.Write(data, MAXLENGTH); SimpleTextUtil.Write(data, maxLength.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); var maxBytesLength = maxLength.ToString(CultureInfo.InvariantCulture).Length; var sb = new StringBuilder(); for (var i = 0; i < maxBytesLength; i++) { sb.Append('0'); } // write our pattern for encoding lengths var patternString = sb.ToString(); SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, patternString, scratch); SimpleTextUtil.WriteNewline(data); int numDocsWritten = 0; foreach (BytesRef value in values) { int length = value == null ? 0 : value.Length; SimpleTextUtil.Write(data, LENGTH); SimpleTextUtil.Write(data, length.ToString(patternString, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); // write bytes -- don't use SimpleText.Write // because it escapes: if (value != null) { data.WriteBytes(value.Bytes, value.Offset, value.Length); } // pad to fit for (int i = length; i < maxLength; i++) { data.WriteByte((byte)(sbyte)' '); } SimpleTextUtil.WriteNewline(data); SimpleTextUtil.Write(data, value == null ? "F" : "T", scratch); SimpleTextUtil.WriteNewline(data); numDocsWritten++; } Debug.Assert(numDocs == numDocsWritten); }
private void Write(BytesRef bytes) { SimpleTextUtil.Write(_output, bytes); }
public override void AddSortedField(FieldInfo field, IEnumerable <BytesRef> values, IEnumerable <long> docToOrd) { Debug.Assert(FieldSeen(field.Name)); Debug.Assert(field.DocValuesType == FieldInfo.DocValuesType_e.SORTED); WriteFieldEntry(field, FieldInfo.DocValuesType_e.SORTED); int valueCount = 0; int maxLength = -1; foreach (BytesRef value in values) { maxLength = Math.Max(maxLength, value.Length); valueCount++; } // write numValues SimpleTextUtil.Write(data, NUMVALUES); SimpleTextUtil.Write(data, Convert.ToString(valueCount), scratch); SimpleTextUtil.WriteNewline(data); // write maxLength SimpleTextUtil.Write(data, MAXLENGTH); SimpleTextUtil.Write(data, Convert.ToString(maxLength), scratch); SimpleTextUtil.WriteNewline(data); int maxBytesLength = Convert.ToString(maxLength).Length; StringBuilder sb = new StringBuilder(); for (int i = 0; i < maxBytesLength; i++) { sb.Append('0'); } // write our pattern for encoding lengths SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); DecimalFormat encoder = new DecimalFormat(sb.ToString(), new DecimalFormatSymbols(Locale.ROOT)); int maxOrdBytes = Convert.ToString(valueCount + 1L).Length; sb.Length = 0; for (int i = 0; i < maxOrdBytes; i++) { sb.Append('0'); } // write our pattern for ords SimpleTextUtil.Write(data, ORDPATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); DecimalFormat ordEncoder = new DecimalFormat(sb.ToString(), new DecimalFormatSymbols(Locale.ROOT)); // for asserts: int valuesSeen = 0; foreach (BytesRef value in values) { // write length SimpleTextUtil.Write(data, LENGTH); SimpleTextUtil.Write(data, encoder.format(value.Length), scratch); SimpleTextUtil.WriteNewline(data); // write bytes -- don't use SimpleText.Write // because it escapes: data.WriteBytes(value.Bytes, value.Offset, value.Length); // pad to fit for (int i = value.Length; i < maxLength; i++) { data.WriteByte((sbyte)' '); } SimpleTextUtil.WriteNewline(data); valuesSeen++; Debug.Assert(valuesSeen <= valueCount); } Debug.Assert(valuesSeen == valueCount); foreach (var ord in docToOrd) { SimpleTextUtil.Write(data, ordEncoder.format(ord + 1), scratch); SimpleTextUtil.WriteNewline(data); } }
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context) { var fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION); var output = directory.CreateOutput(fileName, context); var scratch = new BytesRef(); var success = false; try { SimpleTextUtil.Write(output, NUMFIELDS); SimpleTextUtil.Write(output, Convert.ToString(infos.Size()), scratch); SimpleTextUtil.WriteNewline(output); foreach (FieldInfo fi in infos) { SimpleTextUtil.Write(output, NAME); SimpleTextUtil.Write(output, fi.Name, scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, NUMBER); SimpleTextUtil.Write(output, Convert.ToString(fi.Number), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, ISINDEXED); SimpleTextUtil.Write(output, Convert.ToString(fi.Indexed), scratch); SimpleTextUtil.WriteNewline(output); if (fi.Indexed) { Debug.Assert(fi.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads()); SimpleTextUtil.Write(output, INDEXOPTIONS); SimpleTextUtil.Write(output, fi.FieldIndexOptions.ToString(), scratch); SimpleTextUtil.WriteNewline(output); } SimpleTextUtil.Write(output, STORETV); SimpleTextUtil.Write(output, Convert.ToString(fi.HasVectors()), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, PAYLOADS); SimpleTextUtil.Write(output, Convert.ToString(fi.HasPayloads()), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, NORMS); SimpleTextUtil.Write(output, Convert.ToString(!fi.OmitsNorms()), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, NORMS_TYPE); SimpleTextUtil.Write(output, GetDocValuesType(fi.NormType), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, DOCVALUES); SimpleTextUtil.Write(output, GetDocValuesType(fi.DocValuesType), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, DOCVALUES_GEN); SimpleTextUtil.Write(output, Convert.ToString(fi.DocValuesGen), scratch); SimpleTextUtil.WriteNewline(output); IDictionary <string, string> atts = fi.Attributes(); int numAtts = atts == null ? 0 : atts.Count; SimpleTextUtil.Write(output, NUM_ATTS); SimpleTextUtil.Write(output, Convert.ToString(numAtts), scratch); SimpleTextUtil.WriteNewline(output); if (numAtts <= 0 || atts == null) { continue; } foreach (var entry in atts) { SimpleTextUtil.Write(output, ATT_KEY); SimpleTextUtil.Write(output, entry.Key, scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, ATT_VALUE); SimpleTextUtil.Write(output, entry.Value, scratch); SimpleTextUtil.WriteNewline(output); } } SimpleTextUtil.WriteChecksum(output, scratch); success = true; } finally { if (success) { output.Dispose(); } else { IOUtils.CloseWhileHandlingException(output); } } }
public override void Write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) { var segFileName = IndexFileNames.SegmentFileName(si.Name, "", SimpleTextSegmentInfoFormat.SI_EXTENSION); si.AddFile(segFileName); var success = false; var output = dir.CreateOutput(segFileName, ioContext); try { var scratch = new BytesRef(); SimpleTextUtil.Write(output, SI_VERSION); SimpleTextUtil.Write(output, si.Version, scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, SI_DOCCOUNT); SimpleTextUtil.Write(output, Convert.ToString(si.DocCount, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, SI_USECOMPOUND); SimpleTextUtil.Write(output, Convert.ToString(si.UseCompoundFile, CultureInfo.InvariantCulture).ToLowerInvariant(), scratch); SimpleTextUtil.WriteNewline(output); IDictionary <string, string> diagnostics = si.Diagnostics; int numDiagnostics = diagnostics is null ? 0 : diagnostics.Count; SimpleTextUtil.Write(output, SI_NUM_DIAG); SimpleTextUtil.Write(output, Convert.ToString(numDiagnostics, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); if (numDiagnostics > 0) { foreach (var diagEntry in diagnostics) { SimpleTextUtil.Write(output, SI_DIAG_KEY); SimpleTextUtil.Write(output, diagEntry.Key, scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, SI_DIAG_VALUE); SimpleTextUtil.Write(output, diagEntry.Value, scratch); SimpleTextUtil.WriteNewline(output); } } var files = si.GetFiles(); var numFiles = files is null ? 0 : files.Count; SimpleTextUtil.Write(output, SI_NUM_FILES); SimpleTextUtil.Write(output, Convert.ToString(numFiles, CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); if (numFiles > 0) { foreach (var fileName in files) { SimpleTextUtil.Write(output, SI_FILE); SimpleTextUtil.Write(output, fileName, scratch); SimpleTextUtil.WriteNewline(output); } } SimpleTextUtil.WriteChecksum(output, scratch); success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(output); try { dir.DeleteFile(segFileName); } catch (Exception t) when(t.IsThrowable()) { //Esnure we throw original exeception } } else { output.Dispose(); } } }
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context) { var fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION); var output = directory.CreateOutput(fileName, context); var scratch = new BytesRef(); var success = false; try { SimpleTextUtil.Write(output, NUMFIELDS); SimpleTextUtil.Write(output, infos.Count.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); foreach (FieldInfo fi in infos) { SimpleTextUtil.Write(output, NAME); SimpleTextUtil.Write(output, fi.Name, scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, NUMBER); SimpleTextUtil.Write(output, fi.Number.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, ISINDEXED); SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower(fi.IsIndexed.ToString()), scratch); SimpleTextUtil.WriteNewline(output); if (fi.IsIndexed) { // LUCENENET specific - to avoid boxing, changed from CompareTo() to IndexOptionsComparer.Compare() if (Debugging.AssertsEnabled) { Debugging.Assert(IndexOptionsComparer.Default.Compare(fi.IndexOptions, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.HasPayloads); } SimpleTextUtil.Write(output, INDEXOPTIONS); SimpleTextUtil.Write(output, fi.IndexOptions != IndexOptions.NONE ? fi.IndexOptions.ToString() : string.Empty, scratch); SimpleTextUtil.WriteNewline(output); } SimpleTextUtil.Write(output, STORETV); SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower(fi.HasVectors.ToString()), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, PAYLOADS); SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower(fi.HasPayloads.ToString()), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, NORMS); SimpleTextUtil.Write(output, CultureInfo.InvariantCulture.TextInfo.ToLower((!fi.OmitsNorms).ToString()), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, NORMS_TYPE); SimpleTextUtil.Write(output, GetDocValuesType(fi.NormType), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, DOCVALUES); SimpleTextUtil.Write(output, GetDocValuesType(fi.DocValuesType), scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, DOCVALUES_GEN); SimpleTextUtil.Write(output, fi.DocValuesGen.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); IDictionary <string, string> atts = fi.Attributes; int numAtts = atts == null ? 0 : atts.Count; SimpleTextUtil.Write(output, NUM_ATTS); SimpleTextUtil.Write(output, numAtts.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(output); if (numAtts <= 0 || atts == null) { continue; } foreach (var entry in atts) { SimpleTextUtil.Write(output, ATT_KEY); SimpleTextUtil.Write(output, entry.Key, scratch); SimpleTextUtil.WriteNewline(output); SimpleTextUtil.Write(output, ATT_VALUE); SimpleTextUtil.Write(output, entry.Value, scratch); SimpleTextUtil.WriteNewline(output); } } SimpleTextUtil.WriteChecksum(output, scratch); success = true; } finally { if (success) { output.Dispose(); } else { IOUtils.DisposeWhileHandlingException(output); } } }
public override void AddNumericField(FieldInfo field, IEnumerable <long?> values) { if (Debugging.AssertsEnabled) { Debugging.Assert(FieldSeen(field.Name)); Debugging.Assert(field.DocValuesType == DocValuesType.NUMERIC || field.NormType == DocValuesType.NUMERIC); } WriteFieldEntry(field, DocValuesType.NUMERIC); // first pass to find min/max var minValue = long.MaxValue; var maxValue = long.MinValue; foreach (var n in values) { var v = n.GetValueOrDefault(); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } // write our minimum value to the .dat, all entries are deltas from that SimpleTextUtil.Write(data, MINVALUE); SimpleTextUtil.Write(data, minValue.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); // build up our fixed-width "simple text packed ints" format var diffBig = (decimal)maxValue - (decimal)minValue; // LUCENENET specific - use decimal rather than BigInteger var maxBytesPerValue = diffBig.ToString(CultureInfo.InvariantCulture).Length; var sb = new StringBuilder(); for (var i = 0; i < maxBytesPerValue; i++) { sb.Append('0'); } var patternString = sb.ToString(); // LUCENENET specific - only get the string once // write our pattern to the .dat SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, patternString, scratch); SimpleTextUtil.WriteNewline(data); int numDocsWritten = 0; // second pass to write the values foreach (var n in values) { long value = n.GetValueOrDefault(); if (Debugging.AssertsEnabled) { Debugging.Assert(value >= minValue); } var delta = (decimal)value - (decimal)minValue; // LUCENENET specific - use decimal rather than BigInteger string s = delta.ToString(patternString, CultureInfo.InvariantCulture); if (Debugging.AssertsEnabled) { Debugging.Assert(s.Length == patternString.Length); } SimpleTextUtil.Write(data, s, scratch); SimpleTextUtil.WriteNewline(data); SimpleTextUtil.Write(data, n is null ? "F" : "T", scratch); SimpleTextUtil.WriteNewline(data); numDocsWritten++; if (Debugging.AssertsEnabled) { Debugging.Assert(numDocsWritten <= numDocs); } } if (Debugging.AssertsEnabled) { Debugging.Assert(numDocs == numDocsWritten, "numDocs={0} numDocsWritten={1}", numDocs, numDocsWritten); } }
private void Write(BytesRef b) { SimpleTextUtil.Write(output, b); }
public override void AddNumericField(FieldInfo field, IEnumerable <long> values) { Debug.Assert(FieldSeen(field.Name)); Debug.Assert(field.DocValuesType == FieldInfo.DocValuesType_e.NUMERIC || field.NormType == FieldInfo.DocValuesType_e.NUMERIC); WriteFieldEntry(field, FieldInfo.DocValuesType_e.NUMERIC); // first pass to find min/max var minValue = long.MaxValue; var maxValue = long.MinValue; foreach (var n in values) { var v = n; minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } // write our minimum value to the .dat, all entries are deltas from that SimpleTextUtil.Write(data, MINVALUE); SimpleTextUtil.Write(data, Convert.ToString(minValue), scratch); SimpleTextUtil.WriteNewline(data); // build up our fixed-width "simple text packed ints" format System.Numerics.BigInteger maxBig = maxValue; System.Numerics.BigInteger minBig = minValue; var diffBig = maxBig - minBig; var maxBytesPerValue = diffBig.ToString().Length; var sb = new StringBuilder(); for (var i = 0; i < maxBytesPerValue; i++) { sb.Append('0'); } // write our pattern to the .dat SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); var patternString = sb.ToString(); DecimalFormat encoder = new DecimalFormat(patternString, new DecimalFormatSymbols(Locale.ROOT)); int numDocsWritten = 0; // second pass to write the values foreach (var value in values) { Debug.Assert(value >= minValue); var delta = value - minValue; string s = encoder.format(delta); Debug.Assert(s.Length == patternString.Length); SimpleTextUtil.Write(data, s, scratch); SimpleTextUtil.WriteNewline(data); SimpleTextUtil.Write(data, n == null ? "F" : "T", scratch); SimpleTextUtil.WriteNewline(data); numDocsWritten++; Debug.Assert(numDocsWritten <= numDocs); } Debug.Assert(numDocs == numDocsWritten, "numDocs=" + numDocs + " numDocsWritten=" + numDocsWritten); }
public override void AddNumericField(FieldInfo field, IEnumerable <long?> values) { Debug.Assert(FieldSeen(field.Name)); Debug.Assert(field.DocValuesType == DocValuesType.NUMERIC || field.NormType == DocValuesType.NUMERIC); WriteFieldEntry(field, DocValuesType.NUMERIC); // first pass to find min/max var minValue = long.MaxValue; var maxValue = long.MinValue; foreach (var n in values) { var v = n.GetValueOrDefault(); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } // write our minimum value to the .dat, all entries are deltas from that SimpleTextUtil.Write(data, MINVALUE); SimpleTextUtil.Write(data, minValue.ToString(CultureInfo.InvariantCulture), scratch); SimpleTextUtil.WriteNewline(data); // build up our fixed-width "simple text packed ints" format BigInteger maxBig = maxValue; BigInteger minBig = minValue; var diffBig = BigInteger.Subtract(maxBig, minBig); var maxBytesPerValue = diffBig.ToString(CultureInfo.InvariantCulture).Length; var sb = new StringBuilder(); for (var i = 0; i < maxBytesPerValue; i++) { sb.Append('0'); } // write our pattern to the .dat SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); var patternString = sb.ToString(); int numDocsWritten = 0; // second pass to write the values foreach (var n in values) { long value = n.GetValueOrDefault(); Debug.Assert(value >= minValue); var delta = BigInteger.Subtract(value, minValue); string s = delta.ToString(patternString, CultureInfo.InvariantCulture); Debug.Assert(s.Length == patternString.Length); SimpleTextUtil.Write(data, s, scratch); SimpleTextUtil.WriteNewline(data); SimpleTextUtil.Write(data, n == null ? "F" : "T", scratch); SimpleTextUtil.WriteNewline(data); numDocsWritten++; Debug.Assert(numDocsWritten <= numDocs); } Debug.Assert(numDocs == numDocsWritten, "numDocs=" + numDocs + " numDocsWritten=" + numDocsWritten); }
private void Write(string s) { SimpleTextUtil.Write(_output, s, _scratch); }
public override void AddBinaryField(FieldInfo field, IEnumerable <BytesRef> values) { Debug.Assert(FieldSeen(field.Name)); Debug.Assert(field.DocValuesType == FieldInfo.DocValuesType_e.BINARY); var maxLength = 0; foreach (var value in values) { var length = value == null ? 0 : value.Length; maxLength = Math.Max(maxLength, length); } WriteFieldEntry(field, FieldInfo.DocValuesType_e.BINARY); // write maxLength SimpleTextUtil.Write(data, MAXLENGTH); SimpleTextUtil.Write(data, Convert.ToString(maxLength), scratch); SimpleTextUtil.WriteNewline(data); var maxBytesLength = Convert.ToString(maxLength).Length; var sb = new StringBuilder(); for (var i = 0; i < maxBytesLength; i++) { sb.Append('0'); } // write our pattern for encoding lengths SimpleTextUtil.Write(data, PATTERN); SimpleTextUtil.Write(data, sb.ToString(), scratch); SimpleTextUtil.WriteNewline(data); DecimalFormat encoder = new DecimalFormat(sb.ToString(), new DecimalFormatSymbols(Locale.ROOT)); int numDocsWritten = 0; foreach (BytesRef value in values) { int length = value == null ? 0 : value.Length; SimpleTextUtil.Write(data, LENGTH); SimpleTextUtil.Write(data, encoder.format(length), scratch); SimpleTextUtil.WriteNewline(data); // write bytes -- don't use SimpleText.Write // because it escapes: if (value != null) { data.WriteBytes(value.Bytes, value.Offset, value.Length); } // pad to fit for (int i = length; i < maxLength; i++) { data.WriteByte((sbyte)' '); } SimpleTextUtil.WriteNewline(data); SimpleTextUtil.Write(data, value == null ? "F" : "T", scratch); SimpleTextUtil.WriteNewline(data); numDocsWritten++; } Debug.Assert(numDocs == numDocsWritten); }