public static long ReadLines(Stream stream, Encoding encoding, NewLineAsString onNewLine, ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { unsafe { var pendingLength = 0; var buffer = Constants.Buffer; return(ReadOrCountLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken)); void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding x) { var target = new Span <byte>(buffer, pendingLength, length); var segment = new ReadOnlySpan <byte>(start, length); segment.CopyTo(target); if (partial) { pendingLength = length; } else { var line = new ReadOnlySpan <byte>(buffer, 0, length + pendingLength); onNewLine?.Invoke(lineNumber, encoding.GetString(line)); pendingLength = 0; } } } }
public DocumentDbQueryableProvider(DocumentDbConnectionFactory factory, IMetricsHost <DocumentClient> metrics, IOptions <DocumentDbOptions> options) { _options = options; _factory = factory; _metrics = metrics; }
public void Add(IMetricsHost host) { var key = Environment.MachineName + "." + Environment.CurrentManagedThreadId; _registry.AddOrUpdate(key, host, (n, r) => r); }
public static long ReadLines(Stream stream, Encoding encoding, NewLineAsString onNewLine, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { unsafe { var pendingLength = 0; byte[] buffer = null; // TODO convert to allocator NewLine newLine = (lineNumber, partial, start, length, x, m) => { if (buffer == null) { buffer = new byte[Constants.ReadAheadSize * 2]; } var target = new Span <byte>(buffer, pendingLength, length); var segment = new ReadOnlySpan <byte>(start, length); segment.CopyTo(target); if (partial) { pendingLength = length; } else { var line = new ReadOnlySpan <byte>(buffer, 0, length + pendingLength); onNewLine?.Invoke(lineNumber, encoding.GetString(line), m); pendingLength = 0; buffer = null; } }; return(ReadOrCountLines(stream, encoding, Constants.Buffer, newLine, cancellationToken, metrics)); } }
public void Sort(string fromLabel, string toLabel, IComparer <T> sort, ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var streams = GetAllSegments(fromLabel); var data = new Queue <Stream>(streams); for (var i = 0; i < data.Count; i++) { using (var stream = data.Dequeue()) { using var sw = new StreamWriter(CreateSegment(toLabel, i)); var items = Read(stream, sort, logger, metrics, cancellationToken); foreach (var item in items) { var line = _serialize(item); sw.WriteLine(line); } sw.Flush(); } DeleteSegment(fromLabel, i); } }
public IEnumerable <T> Read(string label, int index, IComparer <T> sort, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var stream = File.OpenRead(GetFilePathForSegment(label, index)); return(Read(stream, sort, metrics, cancellationToken)); }
public EventSourceHostedService(IMetricsHost host, IMetricsRegistry registry, IEnumerable <IMetricsBuilder> builders, IOptionsMonitor <MetricsOptions> options) { _sources = new HashSet <EventSource>(); _sourceNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase); foreach (var builder in builders) { foreach (var source in builder.Build(host)) { _sourceNames.Add(source); } } _registry = registry; _options = options; _changes = _options.OnChange(o => { foreach (var source in _sources) { DisableEvents(source); SubscribeToEventSource(source); } }); }
public ISet <string> Build(IMetricsHost host) { foreach (var registration in _eventSourceCounters) { registration(host); } return(_eventSourceNames); }
public DocumentDbQueryableProvider(string slot, DocumentDbConnectionFactory factory, IMetricsHost <DocumentClient> metrics, IOptionsMonitor <DocumentDbOptions> options) { _slot = slot; _options = options; _factory = factory; _metrics = metrics; }
public static long ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLineAsString onNewLine, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { unsafe { NewLine newLine = (n, p, s, l, e, m) => { onNewLine?.Invoke(n, e.GetString(s, l), m); }; return(ReadOrCountLines(stream, encoding, workingBuffer, newLine, cancellationToken, metrics)); } }
public MetricHealthCheck(string name, IMetricsHost host, Func <IMetricsHost, TMetric> builderFunc, Func <TMetric, TValue> valueFunc, Func <TValue, bool> checkFunc, HealthStatus onCheckFailure = HealthStatus.Unhealthy) { _name = name; _host = host; _builderFunc = builderFunc; _valueFunc = valueFunc; _checkFunc = checkFunc; _onCheckFailure = onCheckFailure; }
public static long ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, string separator, NewValueAsSpan onNewValue, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { unsafe { NewLine onNewLine = (lineNumber, partial, start, length, e, m) => { LineValuesReader.ReadValues(lineNumber, start, length, e, separator, onNewValue, m); }; return(ReadLines(stream, encoding, workingBuffer, onNewLine, metrics, cancellationToken)); } }
public IEnumerable <T> Read(Stream stream, IComparer <T> sort, ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var list = new List <T>(); LineReader.ReadLines(stream, Encoding.UTF8, (lineNumber, line) => { var t = _deserialize(line); list.Add(t); }, logger, metrics, cancellationToken); list.Sort(sort); return(list); }
public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, NewLineAsString onNewLine, ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { unsafe { void NewLine(long n, bool p, byte *s, int l, Encoding e) { onNewLine?.Invoke(n, e.GetString(s, l)); } return(ReadOrCountLines(stream, encoding, buffer, NewLine, logger, metrics, cancellationToken)); } }
public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, string separator, NewValue onNewValue, ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { unsafe { void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding e) { LineValuesReader.ReadValues(lineNumber, start, length, e, separator, onNewValue); } return(ReadLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken)); } }
public static void ReadValues(long lineNumber, ReadOnlySpan <byte> line, Encoding encoding, byte[] separator, NewValueAsSpan newValue, IMetricsHost metrics = null) { var position = 0; while (true) { var next = line.IndexOf(separator); if (next == -1) { newValue?.Invoke(lineNumber, position, line, encoding, metrics); break; } newValue?.Invoke(lineNumber, position, line.Slice(0, next), encoding, metrics); line = line.Slice(next + separator.Length); position += next + separator.Length; } }
public SegmentStats Segment(string label, IEnumerable <T> stream, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var histogram = metrics?.Histogram(typeof(FileMemoryProvider <T>), "line_length", SampleType.Uniform); var count = 0L; var segments = 0; var sw = new StreamWriter(CreateSegment(label, segments)); try { foreach (var item in stream) { var line = _serialize(item); histogram?.Update(line.Length); sw.WriteLine(line); count++; if (maxWorkingMemoryBytes == 0 || sw.BaseStream.Length < maxWorkingMemoryBytes) { continue; } sw.Flush(); sw.Close(); segments++; sw = new StreamWriter(CreateSegment(label, segments)); } } finally { sw.Flush(); sw.Close(); } return(new SegmentStats { RecordCount = count, RecordLength = (int)(histogram?.Mean ?? 0) + 1, SegmentCount = segments }); }
public SegmentStats Segment(string label, IEnumerable <T> stream, int maxWorkingMemoryBytes = 0, ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var count = 0L; var segments = 0; var sw = new StreamWriter(CreateSegment(label, segments)); try { foreach (var item in stream) { var line = _serialize(item); BuiltInMetrics.LineLength <T>(metrics, line.Length); sw.WriteLine(line); count++; if (maxWorkingMemoryBytes == 0 || sw.BaseStream.Length < maxWorkingMemoryBytes) { continue; } sw.Flush(); sw.Close(); segments++; sw = new StreamWriter(CreateSegment(label, segments)); } } finally { sw.Flush(); sw.Close(); } return(new SegmentStats { RecordCount = count, RecordLength = (int)BuiltInMetrics.GetMeanLineLength <T>(metrics) + 1, SegmentCount = segments }); }
public static unsafe void ReadValues(long lineNumber, byte *start, int length, Encoding encoding, byte[] separator, NewValue newValue, IMetricsHost metrics = null) { var position = 0; while (true) { var line = new ReadOnlySpan <byte>(start, length); var next = line.IndexOf(separator); if (next == -1) { newValue?.Invoke(lineNumber, position, start, length, encoding, metrics); break; } newValue?.Invoke(lineNumber, position, start, next, encoding, metrics); var consumed = next + separator.Length; start += consumed; position += consumed; } }
public static IEnumerable <LineConstructor> ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, byte[] separator, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>()); void ReadLines(Encoding e) { try { unsafe { LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, start, length, x, m) => { // TODO convert to row buffer/allocator var buffer = new byte[length]; new ReadOnlySpan <byte>(start, length).CopyTo(buffer); var ctor = new LineConstructor { lineNumber = lineNumber, length = length, start = buffer }; queue.Add(ctor, cancellationToken); }, metrics, cancellationToken); } } catch (Exception ex) { throw ex; } finally { queue.CompleteAdding(); } } Task.Run(() => ReadLines(encoding), cancellationToken); return(queue.GetConsumingEnumerable()); }
public MetricsHost(IMetricsHost host) { _host = host; }
// Derived from MimeKit's MimeParser private static long ReadOrCountLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLine onNewLine, CancellationToken cancellationToken, IMetricsHost metrics) { var count = 0L; var offset = stream.CanSeek ? stream.Position : 0L; var from = Constants.ReadAheadSize; var to = Constants.ReadAheadSize; var endOfStream = false; var preamble = encoding.GetPreambleBuffer(); unsafe { fixed(byte *buffer = workingBuffer) { if (stream.CanSeek && stream.Position != offset) { stream.Seek(offset, SeekOrigin.Begin); } if (!ReadPreamble(stream, preamble, buffer, workingBuffer, ref from, ref to, ref endOfStream, cancellationToken)) { throw new FormatException(ErrorStrings.UnexpectedEndOfStream); } do { if (ReadAhead(stream, workingBuffer, Constants.ReadAheadSize, 2, ref from, ref to, ref endOfStream, cancellationToken) <= 0) { break; } var position = buffer + from; var end = buffer + to; var startIndex = from; *end = (byte)'\n'; while (position < end) { var alignment = (startIndex + 3) & ~3; var aligned = buffer + alignment; var start = position; var c = *aligned; *aligned = Constants.LineFeed; while (*position != Constants.LineFeed) { position++; } *aligned = c; if (position == aligned && c != Constants.LineFeed) { var dword = (uint *)position; uint mask; do { mask = *dword++ ^ 0x0A0A0A0A; mask = (mask - 0x01010101) & ~mask & 0x80808080; } while (mask == 0); position = (byte *)(dword - 1); while (*position != Constants.LineFeed) { position++; } } var length = (int)(position - start); if (position < end) { length++; position++; count++; BytesPerSecond(metrics, length); onNewLine?.Invoke(count, false, start, length, encoding, metrics); } else if (count == 0 && position == end) { BytesPerSecond(metrics, length); onNewLine?.Invoke(count, false, start, length, encoding, metrics); return(1); } else { // line spans across the read-ahead buffer BytesPerSecond(metrics, length); onNewLine?.Invoke(count, true, start, length, encoding, metrics); } startIndex += length; } from = startIndex; } while (true); } } return(count); }
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] workingBuffer, byte[] separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>()); void ReadLines(Encoding e) { var pendingLength = 0; byte[] buffer = null; // TODO convert to allocator try { unsafe { LineReader.ReadLines(stream, e, workingBuffer, (lineNumber, partial, start, length, x, m) => { if (buffer == null) { buffer = new byte[Math.Max(length, Constants.ReadAheadSize * 2)]; } var target = new Span <byte>(buffer, pendingLength, length); var segment = new ReadOnlySpan <byte>(start, length); segment.CopyTo(target); if (partial) { pendingLength = length; } else { var ctor = new LineConstructor { lineNumber = lineNumber, length = length + pendingLength, buffer = buffer }; if (maxWorkingMemoryBytes > 0) { var usedBytes = queue.Count * (buffer.Length + sizeof(long) + sizeof(int)); while (usedBytes > maxWorkingMemoryBytes) { Task.Delay(10, cancellationToken).Wait(cancellationToken); } } queue.Add(ctor, cancellationToken); pendingLength = 0; buffer = null; } }, metrics, cancellationToken); } } finally { queue.CompleteAdding(); } } Task.Run(() => ReadLines(encoding), cancellationToken); return(queue.GetConsumingEnumerable()); }
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { return(StreamLines(stream, encoding, Constants.Buffer, separator, maxWorkingMemoryBytes, metrics, cancellationToken)); }
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] workingBuffer, string separator, int maxWorkingMemoryBytes = 0, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { return(StreamLines(stream, encoding, workingBuffer, encoding.GetSeparatorBuffer(separator ?? Environment.NewLine), maxWorkingMemoryBytes, metrics, cancellationToken)); }
public static long ReadLines(Stream stream, Encoding encoding, string separator, NewValueAsSpan onNewValue, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { return(ReadLines(stream, encoding, Constants.Buffer, separator, onNewValue, metrics, cancellationToken)); }
public static long ReadLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLine onNewLine, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { return(ReadOrCountLines(stream, encoding, workingBuffer, onNewLine, cancellationToken, metrics)); }
public static long CountLines(Stream stream, Encoding encoding, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { return(CountLines(stream, encoding, Constants.Buffer, metrics, cancellationToken)); }
private static void BytesPerSecond(IMetricsHost metrics, int length) { metrics?.Meter(typeof(LineReader), "bytes_read_per_second", "bytes", TimeUnit.Seconds).Mark(length); }
public static long CountLines(Stream stream, Encoding encoding, byte[] workingBuffer, ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default) { return(ReadOrCountLines(stream, encoding, workingBuffer, null, logger, metrics, cancellationToken)); }