public static long ReadLines(Stream stream, Encoding encoding, NewLineAsString onNewLine,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
unsafe
{
var pendingLength = 0;
var buffer = Constants.Buffer;
return(ReadOrCountLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken));
void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding x)
{
var target = new Span <byte>(buffer, pendingLength, length);
var segment = new ReadOnlySpan <byte>(start, length);
segment.CopyTo(target);
if (partial)
{
pendingLength = length;
}
else
{
var line = new ReadOnlySpan <byte>(buffer, 0, length + pendingLength);
onNewLine?.Invoke(lineNumber, encoding.GetString(line));
pendingLength = 0;
}
}
}
}
public DocumentDbQueryableProvider(DocumentDbConnectionFactory factory, IMetricsHost <DocumentClient> metrics,
IOptions <DocumentDbOptions> options)
{
_options = options;
_factory = factory;
_metrics = metrics;
}
public void Add(IMetricsHost host)
{
var key = Environment.MachineName + "." + Environment.CurrentManagedThreadId;
_registry.AddOrUpdate(key,
host, (n, r) => r);
}
public static long ReadLines(Stream stream, Encoding encoding, NewLineAsString onNewLine,
IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
unsafe
{
var pendingLength = 0;
byte[] buffer = null; // TODO convert to allocator
NewLine newLine = (lineNumber, partial, start, length, x, m) =>
{
if (buffer == null)
{
buffer = new byte[Constants.ReadAheadSize * 2];
}
var target = new Span <byte>(buffer, pendingLength, length);
var segment = new ReadOnlySpan <byte>(start, length);
segment.CopyTo(target);
if (partial)
{
pendingLength = length;
}
else
{
var line = new ReadOnlySpan <byte>(buffer, 0, length + pendingLength);
onNewLine?.Invoke(lineNumber, encoding.GetString(line), m);
pendingLength = 0;
buffer = null;
}
};
return(ReadOrCountLines(stream, encoding, Constants.Buffer, newLine, cancellationToken, metrics));
}
}
public void Sort(string fromLabel, string toLabel, IComparer <T> sort, ISafeLogger logger = null,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var streams = GetAllSegments(fromLabel);
var data = new Queue <Stream>(streams);
for (var i = 0; i < data.Count; i++)
{
using (var stream = data.Dequeue())
{
using var sw = new StreamWriter(CreateSegment(toLabel, i));
var items = Read(stream, sort, logger, metrics, cancellationToken);
foreach (var item in items)
{
var line = _serialize(item);
sw.WriteLine(line);
}
sw.Flush();
}
DeleteSegment(fromLabel, i);
}
}
public IEnumerable <T> Read(string label, int index, IComparer <T> sort, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
var stream = File.OpenRead(GetFilePathForSegment(label, index));
return(Read(stream, sort, metrics, cancellationToken));
}
public EventSourceHostedService(IMetricsHost host, IMetricsRegistry registry,
IEnumerable <IMetricsBuilder> builders, IOptionsMonitor <MetricsOptions> options)
{
_sources = new HashSet <EventSource>();
_sourceNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (var builder in builders)
{
foreach (var source in builder.Build(host))
{
_sourceNames.Add(source);
}
}
_registry = registry;
_options = options;
_changes = _options.OnChange(o =>
{
foreach (var source in _sources)
{
DisableEvents(source);
SubscribeToEventSource(source);
}
});
}
public ISet <string> Build(IMetricsHost host)
{
foreach (var registration in _eventSourceCounters)
{
registration(host);
}
return(_eventSourceNames);
}
public DocumentDbQueryableProvider(string slot, DocumentDbConnectionFactory factory,
IMetricsHost <DocumentClient> metrics,
IOptionsMonitor <DocumentDbOptions> options)
{
_slot = slot;
_options = options;
_factory = factory;
_metrics = metrics;
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLineAsString onNewLine,
IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
unsafe
{
NewLine newLine = (n, p, s, l, e, m) => { onNewLine?.Invoke(n, e.GetString(s, l), m); };
return(ReadOrCountLines(stream, encoding, workingBuffer, newLine, cancellationToken, metrics));
}
}
public MetricHealthCheck(string name, IMetricsHost host, Func <IMetricsHost, TMetric> builderFunc,
Func <TMetric, TValue> valueFunc, Func <TValue, bool> checkFunc,
HealthStatus onCheckFailure = HealthStatus.Unhealthy)
{
_name = name;
_host = host;
_builderFunc = builderFunc;
_valueFunc = valueFunc;
_checkFunc = checkFunc;
_onCheckFailure = onCheckFailure;
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, string separator,
NewValueAsSpan onNewValue, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
unsafe
{
NewLine onNewLine = (lineNumber, partial, start, length, e, m) =>
{
LineValuesReader.ReadValues(lineNumber, start, length, e, separator, onNewValue, m);
};
return(ReadLines(stream, encoding, workingBuffer, onNewLine, metrics, cancellationToken));
}
}
public IEnumerable <T> Read(Stream stream, IComparer <T> sort, ISafeLogger logger = null,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var list = new List <T>();
LineReader.ReadLines(stream, Encoding.UTF8, (lineNumber, line) =>
{
var t = _deserialize(line);
list.Add(t);
}, logger, metrics, cancellationToken);
list.Sort(sort);
return(list);
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, NewLineAsString onNewLine,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
unsafe
{
void NewLine(long n, bool p, byte *s, int l, Encoding e)
{
onNewLine?.Invoke(n, e.GetString(s, l));
}
return(ReadOrCountLines(stream, encoding, buffer, NewLine, logger, metrics, cancellationToken));
}
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, string separator,
NewValue onNewValue, ISafeLogger logger = null, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
unsafe
{
void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding e)
{
LineValuesReader.ReadValues(lineNumber, start, length, e, separator, onNewValue);
}
return(ReadLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken));
}
}
public static void ReadValues(long lineNumber, ReadOnlySpan <byte> line, Encoding encoding, byte[] separator,
NewValueAsSpan newValue, IMetricsHost metrics = null)
{
var position = 0;
while (true)
{
var next = line.IndexOf(separator);
if (next == -1)
{
newValue?.Invoke(lineNumber, position, line, encoding, metrics);
break;
}
newValue?.Invoke(lineNumber, position, line.Slice(0, next), encoding, metrics);
line = line.Slice(next + separator.Length);
position += next + separator.Length;
}
}
public SegmentStats Segment(string label, IEnumerable <T> stream, int maxWorkingMemoryBytes = 0,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var histogram = metrics?.Histogram(typeof(FileMemoryProvider <T>), "line_length", SampleType.Uniform);
var count = 0L;
var segments = 0;
var sw = new StreamWriter(CreateSegment(label, segments));
try
{
foreach (var item in stream)
{
var line = _serialize(item);
histogram?.Update(line.Length);
sw.WriteLine(line);
count++;
if (maxWorkingMemoryBytes == 0 || sw.BaseStream.Length < maxWorkingMemoryBytes)
{
continue;
}
sw.Flush();
sw.Close();
segments++;
sw = new StreamWriter(CreateSegment(label, segments));
}
}
finally
{
sw.Flush();
sw.Close();
}
return(new SegmentStats
{
RecordCount = count,
RecordLength = (int)(histogram?.Mean ?? 0) + 1,
SegmentCount = segments
});
}
public SegmentStats Segment(string label, IEnumerable <T> stream, int maxWorkingMemoryBytes = 0,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var count = 0L;
var segments = 0;
var sw = new StreamWriter(CreateSegment(label, segments));
try
{
foreach (var item in stream)
{
var line = _serialize(item);
BuiltInMetrics.LineLength <T>(metrics, line.Length);
sw.WriteLine(line);
count++;
if (maxWorkingMemoryBytes == 0 || sw.BaseStream.Length < maxWorkingMemoryBytes)
{
continue;
}
sw.Flush();
sw.Close();
segments++;
sw = new StreamWriter(CreateSegment(label, segments));
}
}
finally
{
sw.Flush();
sw.Close();
}
return(new SegmentStats
{
RecordCount = count,
RecordLength = (int)BuiltInMetrics.GetMeanLineLength <T>(metrics) + 1,
SegmentCount = segments
});
}
public static unsafe void ReadValues(long lineNumber, byte *start, int length, Encoding encoding,
byte[] separator, NewValue newValue, IMetricsHost metrics = null)
{
var position = 0;
while (true)
{
var line = new ReadOnlySpan <byte>(start, length);
var next = line.IndexOf(separator);
if (next == -1)
{
newValue?.Invoke(lineNumber, position, start, length, encoding, metrics);
break;
}
newValue?.Invoke(lineNumber, position, start, next, encoding, metrics);
var consumed = next + separator.Length;
start += consumed;
position += consumed;
}
}
public static IEnumerable <LineConstructor> ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer,
byte[] separator, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>());
void ReadLines(Encoding e)
{
try
{
unsafe
{
LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, start, length, x, m) =>
{
// TODO convert to row buffer/allocator
var buffer = new byte[length];
new ReadOnlySpan <byte>(start, length).CopyTo(buffer);
var ctor = new LineConstructor
{
lineNumber = lineNumber,
length = length,
start = buffer
};
queue.Add(ctor, cancellationToken);
}, metrics, cancellationToken);
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
queue.CompleteAdding();
}
}
Task.Run(() => ReadLines(encoding), cancellationToken);
return(queue.GetConsumingEnumerable());
}
public MetricsHost(IMetricsHost host)
{
_host = host;
}
// Derived from MimeKit's MimeParser
private static long ReadOrCountLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLine onNewLine,
CancellationToken cancellationToken, IMetricsHost metrics)
{
var count = 0L;
var offset = stream.CanSeek ? stream.Position : 0L;
var from = Constants.ReadAheadSize;
var to = Constants.ReadAheadSize;
var endOfStream = false;
var preamble = encoding.GetPreambleBuffer();
unsafe
{
fixed(byte *buffer = workingBuffer)
{
if (stream.CanSeek && stream.Position != offset)
{
stream.Seek(offset, SeekOrigin.Begin);
}
if (!ReadPreamble(stream, preamble, buffer, workingBuffer, ref from, ref to, ref endOfStream,
cancellationToken))
{
throw new FormatException(ErrorStrings.UnexpectedEndOfStream);
}
do
{
if (ReadAhead(stream, workingBuffer, Constants.ReadAheadSize, 2, ref from, ref to,
ref endOfStream,
cancellationToken) <= 0)
{
break;
}
var position = buffer + from;
var end = buffer + to;
var startIndex = from;
*end = (byte)'\n';
while (position < end)
{
var alignment = (startIndex + 3) & ~3;
var aligned = buffer + alignment;
var start = position;
var c = *aligned;
*aligned = Constants.LineFeed;
while (*position != Constants.LineFeed)
{
position++;
}
*aligned = c;
if (position == aligned && c != Constants.LineFeed)
{
var dword = (uint *)position;
uint mask;
do
{
mask = *dword++ ^ 0x0A0A0A0A;
mask = (mask - 0x01010101) & ~mask & 0x80808080;
} while (mask == 0);
position = (byte *)(dword - 1);
while (*position != Constants.LineFeed)
{
position++;
}
}
var length = (int)(position - start);
if (position < end)
{
length++;
position++;
count++;
BytesPerSecond(metrics, length);
onNewLine?.Invoke(count, false, start, length, encoding, metrics);
}
else if (count == 0 && position == end)
{
BytesPerSecond(metrics, length);
onNewLine?.Invoke(count, false, start, length, encoding, metrics);
return(1);
}
else
{
// line spans across the read-ahead buffer
BytesPerSecond(metrics, length);
onNewLine?.Invoke(count, true, start, length, encoding, metrics);
}
startIndex += length;
}
from = startIndex;
} while (true);
}
}
return(count);
}
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] workingBuffer,
byte[] separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>());
void ReadLines(Encoding e)
{
var pendingLength = 0;
byte[] buffer = null; // TODO convert to allocator
try
{
unsafe
{
LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, partial, start, length, x, m) =>
{
if (buffer == null)
{
buffer = new byte[Math.Max(length, Constants.ReadAheadSize * 2)];
}
var target = new Span <byte>(buffer, pendingLength, length);
var segment = new ReadOnlySpan <byte>(start, length);
segment.CopyTo(target);
if (partial)
{
pendingLength = length;
}
else
{
var ctor = new LineConstructor
{
lineNumber = lineNumber, length = length + pendingLength, buffer = buffer
};
if (maxWorkingMemoryBytes > 0)
{
var usedBytes = queue.Count * (buffer.Length + sizeof(long) + sizeof(int));
while (usedBytes > maxWorkingMemoryBytes)
{
Task.Delay(10, cancellationToken).Wait(cancellationToken);
}
}
queue.Add(ctor, cancellationToken);
pendingLength = 0;
buffer = null;
}
}, metrics, cancellationToken);
}
}
finally
{
queue.CompleteAdding();
}
}
Task.Run(() => ReadLines(encoding), cancellationToken);
return(queue.GetConsumingEnumerable());
}
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding,
byte[] separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
return(StreamLines(stream, encoding, Constants.Buffer, separator, maxWorkingMemoryBytes, metrics,
cancellationToken));
}
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] workingBuffer,
string separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
return(StreamLines(stream, encoding, workingBuffer,
encoding.GetSeparatorBuffer(separator ?? Environment.NewLine), maxWorkingMemoryBytes, metrics,
cancellationToken));
}
public static long ReadLines(Stream stream, Encoding encoding, string separator, NewValueAsSpan onNewValue,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
return(ReadLines(stream, encoding, Constants.Buffer, separator, onNewValue, metrics, cancellationToken));
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLine onNewLine,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
return(ReadOrCountLines(stream, encoding, workingBuffer, onNewLine, cancellationToken, metrics));
}
public static long CountLines(Stream stream, Encoding encoding, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
return(CountLines(stream, encoding, Constants.Buffer, metrics, cancellationToken));
}
private static void BytesPerSecond(IMetricsHost metrics, int length)
{
metrics?.Meter(typeof(LineReader), "bytes_read_per_second", "bytes", TimeUnit.Seconds).Mark(length);
}
public static long CountLines(Stream stream, Encoding encoding, byte[] workingBuffer, ISafeLogger logger = null,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
return(ReadOrCountLines(stream, encoding, workingBuffer, null, logger, metrics, cancellationToken));
}
