public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor parseCursor = null) { var reader = new StreamReader(streamFactory()); var s = reader.ReadToEnd(); var j = s.StartsWith("[") ? (JToken)JArray.Parse(s) : JObject.Parse(s); var result = new List <IDictionary <string, string> >(); if (j.Type == JTokenType.Array) { foreach (var child in j.Children()) { if (child.Type == JTokenType.Object) { result.Add(ToDic(child, source)); } } } else { result.Add(ToDic(j, source)); } return(result); }
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null) { /* * * "count": 4, * "total": 126, * "minimum": 0, * "maximum": 63, * "average": 31.5, * "resourceId": "/SUBSCRIPTIONS/9614FC94-9519-46FA-B7EC-DD1B0411DB13/RESOURCEGROUPS/WHASHA/PROVIDERS/MICROSOFT.CACHE/REDIS/FILLAPDWHASHAPRODUCTSEYHOOACHE", * "time": "2018-01-18T12:55:00.0000000Z", * "metricName": "connectedclients", * "timeGrain": "PT1M" */ var ms = new MemoryStream(); var body = streamFactory(); body.CopyTo(ms); ms.Position = 0; var text = new StreamReader(ms).ReadToEnd(); var col = JsonConvert.DeserializeObject <InsightMetricCollection>(text); foreach (var r in col.Records) { var subscriptionGuidFirstPart = r.ResourceId.Split('/')[2].Split('-')[0]; var pk = string.Format($"{subscriptionGuidFirstPart}_{string.Join("_", r.ResourceId.Split('/').Reverse().Take(3))}_{r.MetricName}"); var rk = r.Time.ToString("yyyyMMddHHmmss"); yield return(new Dictionary <string, string>() { { "@timestamp", r.Time.ToString() }, { "PartitionKey", pk }, { "RowKey", rk }, { "metricName", r.MetricName }, { "resourceId", r.ResourceId }, { "timeGrain", r.TimeGrain }, { "count", r.Count.ToString(CultureInfo.InvariantCulture) }, { "total", r.Total.ToString(CultureInfo.InvariantCulture) }, { "minimum", r.Minimum.ToString(CultureInfo.InvariantCulture) }, { "maximum", r.Maximum.ToString(CultureInfo.InvariantCulture) }, { "average", r.Average.ToString(CultureInfo.InvariantCulture) } }); } }
private IEnumerable <IDictionary <string, string> > ParseInternal(StreamReader reader, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor) { if (cursor.EndPosition == 0) { cursor.EndPosition = long.MaxValue; } var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray(); var partitionKey = string.Join("_", idSegments.Take(idSegments.Length - 1)); var rowKeyPrefix = Path.GetFileNameWithoutExtension(idSegments.Last()); var fields = cursor.StartReadPosition > 0 ? ReadFirstHeaders(reader) : null; reader.DiscardBufferedData(); reader.BaseStream.Position = cursor.StartReadPosition; var headersHaveChanged = false; string line; var offset = cursor.StartReadPosition; var lineCount = 0; //In case starting offset was set to a line break var lineBreakIndex = Environment.NewLine.IndexOf((char)reader.Peek()); if (lineBreakIndex >= 0) { reader.ReadLine(); offset += Environment.NewLine.Length - lineBreakIndex; } while (offset < cursor.EndPosition && (line = reader.ReadLine()) != null) { lineCount++; offset += line.Length + Environment.NewLine.Length; if (line.StartsWith("#Fields: ")) { fields = BuildFields(line); continue; } if (line.StartsWith("#") || fields == null)//make sure that fields are set before parsing them { continue; } //skip until start offset in case of re-read if (offset <= cursor.StartParsePosition) { continue; } var entries = GetEntries(line); if (fields.Length + 2 != entries.Length)//first 2 entries - date and time are collapsed into @timestamp { if (cursor.StartReadPosition == 0) { throw new InvalidOperationException("fields column mismatch"); } //e.g. in case startReadPosition was pointing to the middle of the line //have to skip to next line if (lineCount == 1) { continue; } headersHaveChanged = true; break; } var doc = ParseEntity(fields, entries, source.TypeName, partitionKey, $"{rowKeyPrefix}_{offset}"); if (doc != null) { yield return(doc); } cursor.StartReadPosition = offset; } if (!headersHaveChanged) { yield break; } //headers have changed since the beginning of the file - have to read whole file cursor.StartReadPosition = 0; foreach (var doc in ParseInternal(reader, id, source, cursor)) { yield return(doc); } }
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null) { cursor = cursor ?? new ParseCursor(0); StorageException lastException = null; var resumeAttemptCount = 0; while (resumeAttemptCount < 3) { using (var stream = streamFactory()) using (var reader = new StreamReader(stream, Encoding.ASCII, false, 1024 * 1024 * 6)) using (var enumerator = ParseInternal(reader, id, source, cursor).GetEnumerator()) { var wasInterrupted = false; bool hasMoved; do { if (enumerator.Current != null) { yield return(enumerator.Current); } try { hasMoved = enumerator.MoveNext(); } catch (StorageException ex) { lastException = ex; wasInterrupted = true; resumeAttemptCount++; break; } } while (hasMoved); if (!wasInterrupted) { cursor.EndPosition = stream.Length; yield break; } } } }
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null) { cursor = cursor ?? new ParseCursor(0); var body = streamFactory(); if (body.Position != 0) { body.Position = 0; } if (cursor.EndPosition == 0) { cursor.EndPosition = long.MaxValue; } string line; var lineNumber = 0; var fileName = GetFileName(id); var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray(); var partionKey = string.Join("_", idSegments.Take(idSegments.Length - 1)); var fileDate = GetFileDate(fileName); var logLineParser = new SitecoreLogLineParser(); SitecoreLogEntry currentEntry = null; var reader = new StreamReader(body); while (body.Position < cursor.EndPosition && (line = reader.ReadLine()) != null) { lineNumber++; if (body.Position < cursor.StartReadPosition) { continue; } if (string.IsNullOrWhiteSpace(line)) { continue; } //remove connection string passwords in exceptions etc (thanks sitecore) line = ReplaceSecureInformation(line, "password="******"**PASSWORD**REDACTED**"); line = ReplaceSecureInformation(line, "user id=", "**USER**REDACTED**"); var logItem = logLineParser.ParseLine(line, fileDate); if (logItem != null) { var logText = logItem.Message; //filter out rubbish logs. eg blank INFO during sitecore startup if (string.IsNullOrWhiteSpace(logText) || logText.StartsWith("*****")) { continue; } logItem.LogLineNumber = lineNumber; } if (logItem == null && currentEntry != null) { // Existing multiline message currentEntry.AppendMessageText(line); } else if (logItem != null && currentEntry == null) { // new entry found currentEntry = logItem; } else if (currentEntry != null) { //new entry found, current one is completed. yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName)); currentEntry = logItem; } } if (currentEntry != null) { yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName)); } }