public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory,
Uri id,
DiagnosticsSourceSummary source,
ParseCursor parseCursor = null)
{
var reader = new StreamReader(streamFactory());
var s = reader.ReadToEnd();
var j = s.StartsWith("[") ? (JToken)JArray.Parse(s) : JObject.Parse(s);
var result = new List <IDictionary <string, string> >();
if (j.Type == JTokenType.Array)
{
foreach (var child in j.Children())
{
if (child.Type == JTokenType.Object)
{
result.Add(ToDic(child, source));
}
}
}
else
{
result.Add(ToDic(j, source));
}
return(result);
}
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory,
Uri id,
DiagnosticsSourceSummary source,
ParseCursor cursor = null)
{
/*
*
* "count": 4,
* "total": 126,
* "minimum": 0,
* "maximum": 63,
* "average": 31.5,
* "resourceId": "/SUBSCRIPTIONS/9614FC94-9519-46FA-B7EC-DD1B0411DB13/RESOURCEGROUPS/WHASHA/PROVIDERS/MICROSOFT.CACHE/REDIS/FILLAPDWHASHAPRODUCTSEYHOOACHE",
* "time": "2018-01-18T12:55:00.0000000Z",
* "metricName": "connectedclients",
* "timeGrain": "PT1M"
*/
var ms = new MemoryStream();
var body = streamFactory();
body.CopyTo(ms);
ms.Position = 0;
var text = new StreamReader(ms).ReadToEnd();
var col = JsonConvert.DeserializeObject <InsightMetricCollection>(text);
foreach (var r in col.Records)
{
var subscriptionGuidFirstPart = r.ResourceId.Split('/')[2].Split('-')[0];
var pk = string.Format($"{subscriptionGuidFirstPart}_{string.Join("_", r.ResourceId.Split('/').Reverse().Take(3))}_{r.MetricName}");
var rk = r.Time.ToString("yyyyMMddHHmmss");
yield return(new Dictionary <string, string>()
{
{ "@timestamp", r.Time.ToString() },
{ "PartitionKey", pk },
{ "RowKey", rk },
{ "metricName", r.MetricName },
{ "resourceId", r.ResourceId },
{ "timeGrain", r.TimeGrain },
{ "count", r.Count.ToString(CultureInfo.InvariantCulture) },
{ "total", r.Total.ToString(CultureInfo.InvariantCulture) },
{ "minimum", r.Minimum.ToString(CultureInfo.InvariantCulture) },
{ "maximum", r.Maximum.ToString(CultureInfo.InvariantCulture) },
{ "average", r.Average.ToString(CultureInfo.InvariantCulture) }
});
}
}
private IEnumerable <IDictionary <string, string> > ParseInternal(StreamReader reader, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor)
{
if (cursor.EndPosition == 0)
{
cursor.EndPosition = long.MaxValue;
}
var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray();
var partitionKey = string.Join("_", idSegments.Take(idSegments.Length - 1));
var rowKeyPrefix = Path.GetFileNameWithoutExtension(idSegments.Last());
var fields = cursor.StartReadPosition > 0 ? ReadFirstHeaders(reader) : null;
reader.DiscardBufferedData();
reader.BaseStream.Position = cursor.StartReadPosition;
var headersHaveChanged = false;
string line;
var offset = cursor.StartReadPosition;
var lineCount = 0;
//In case starting offset was set to a line break
var lineBreakIndex = Environment.NewLine.IndexOf((char)reader.Peek());
if (lineBreakIndex >= 0)
{
reader.ReadLine();
offset += Environment.NewLine.Length - lineBreakIndex;
}
while (offset < cursor.EndPosition && (line = reader.ReadLine()) != null)
{
lineCount++;
offset += line.Length + Environment.NewLine.Length;
if (line.StartsWith("#Fields: "))
{
fields = BuildFields(line);
continue;
}
if (line.StartsWith("#") || fields == null)//make sure that fields are set before parsing them
{
continue;
}
//skip until start offset in case of re-read
if (offset <= cursor.StartParsePosition)
{
continue;
}
var entries = GetEntries(line);
if (fields.Length + 2 != entries.Length)//first 2 entries - date and time are collapsed into @timestamp
{
if (cursor.StartReadPosition == 0)
{
throw new InvalidOperationException("fields column mismatch");
}
//e.g. in case startReadPosition was pointing to the middle of the line
//have to skip to next line
if (lineCount == 1)
{
continue;
}
headersHaveChanged = true;
break;
}
var doc = ParseEntity(fields, entries, source.TypeName, partitionKey, $"{rowKeyPrefix}_{offset}");
if (doc != null)
{
yield return(doc);
}
cursor.StartReadPosition = offset;
}
if (!headersHaveChanged)
{
yield break;
}
//headers have changed since the beginning of the file - have to read whole file
cursor.StartReadPosition = 0;
foreach (var doc in ParseInternal(reader, id, source, cursor))
{
yield return(doc);
}
}
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null)
{
cursor = cursor ?? new ParseCursor(0);
StorageException lastException = null;
var resumeAttemptCount = 0;
while (resumeAttemptCount < 3)
{
using (var stream = streamFactory())
using (var reader = new StreamReader(stream, Encoding.ASCII, false, 1024 * 1024 * 6))
using (var enumerator = ParseInternal(reader, id, source, cursor).GetEnumerator())
{
var wasInterrupted = false;
bool hasMoved;
do
{
if (enumerator.Current != null)
{
yield return(enumerator.Current);
}
try
{
hasMoved = enumerator.MoveNext();
}
catch (StorageException ex)
{
lastException = ex;
wasInterrupted = true;
resumeAttemptCount++;
break;
}
} while (hasMoved);
if (!wasInterrupted)
{
cursor.EndPosition = stream.Length;
yield break;
}
}
}
}
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null)
{
cursor = cursor ?? new ParseCursor(0);
var body = streamFactory();
if (body.Position != 0)
{
body.Position = 0;
}
if (cursor.EndPosition == 0)
{
cursor.EndPosition = long.MaxValue;
}
string line;
var lineNumber = 0;
var fileName = GetFileName(id);
var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray();
var partionKey = string.Join("_", idSegments.Take(idSegments.Length - 1));
var fileDate = GetFileDate(fileName);
var logLineParser = new SitecoreLogLineParser();
SitecoreLogEntry currentEntry = null;
var reader = new StreamReader(body);
while (body.Position < cursor.EndPosition && (line = reader.ReadLine()) != null)
{
lineNumber++;
if (body.Position < cursor.StartReadPosition)
{
continue;
}
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
//remove connection string passwords in exceptions etc (thanks sitecore)
line = ReplaceSecureInformation(line, "password="******"**PASSWORD**REDACTED**");
line = ReplaceSecureInformation(line, "user id=", "**USER**REDACTED**");
var logItem = logLineParser.ParseLine(line, fileDate);
if (logItem != null)
{
var logText = logItem.Message;
//filter out rubbish logs. eg blank INFO during sitecore startup
if (string.IsNullOrWhiteSpace(logText) || logText.StartsWith("*****"))
{
continue;
}
logItem.LogLineNumber = lineNumber;
}
if (logItem == null && currentEntry != null)
{
// Existing multiline message
currentEntry.AppendMessageText(line);
}
else if (logItem != null && currentEntry == null)
{
// new entry found
currentEntry = logItem;
}
else if (currentEntry != null)
{
//new entry found, current one is completed.
yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName));
currentEntry = logItem;
}
}
if (currentEntry != null)
{
yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName));
}
}
