public SegmentStats Segment(string label, IEnumerable <T> stream, int maxWorkingMemoryBytes = 0,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var histogram = metrics?.Histogram(typeof(FileMemoryProvider <T>), "line_length", SampleType.Uniform);
var count = 0L;
var segments = 0;
var sw = new StreamWriter(CreateSegment(label, segments));
try
{
foreach (var item in stream)
{
var line = _serialize(item);
histogram?.Update(line.Length);
sw.WriteLine(line);
count++;
if (maxWorkingMemoryBytes == 0 || sw.BaseStream.Length < maxWorkingMemoryBytes)
{
continue;
}
sw.Flush();
sw.Close();
segments++;
sw = new StreamWriter(CreateSegment(label, segments));
}
}
finally
{
sw.Flush();
sw.Close();
}
return(new SegmentStats
{
RecordCount = count,
RecordLength = (int)(histogram?.Mean ?? 0) + 1,
SegmentCount = segments
});
}
/// <summary>
/// Creates a new histogram metric and registers it under the given type and name
/// </summary>
/// <param name="name">The metric name</param>
/// <param name="sampleType">The sample type</param>
/// <returns></returns>
public HistogramMetric Histogram(string name, SampleType sampleType)
{
return(_host.Histogram(typeof(TOwner), name, sampleType));
}
public static double GetMeanLineLength <T>(IMetricsHost metrics)
{
return(metrics?.Histogram(typeof(FileMemoryProvider <T>), "line_length", SampleType.Uniform).Mean ?? 0);
}
public static void LineLength <T>(IMetricsHost metrics, int length)
{
metrics?.Histogram(typeof(FileMemoryProvider <T>), "line_length", SampleType.Uniform).Update(length);
}