public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory,
Uri id,
DiagnosticsSourceSummary source,
ParseCursor parseCursor = null)
{
var reader = new StreamReader(streamFactory());
var s = reader.ReadToEnd();
var j = s.StartsWith("[") ? (JToken)JArray.Parse(s) : JObject.Parse(s);
var result = new List <IDictionary <string, string> >();
if (j.Type == JTokenType.Array)
{
foreach (var child in j.Children())
{
if (child.Type == JTokenType.Object)
{
result.Add(ToDic(child, source));
}
}
}
else
{
result.Add(ToDic(j, source));
}
return(result);
}
public EventProcessor(NestBatchPusher elasticsearchBatchPusher,
IParser parser,
DiagnosticsSourceSummary source)
{
this._elasticsearchBatchPusher = elasticsearchBatchPusher;
this._parser = parser;
this._source = source;
}
public void IfTimestampFieldSpecifiedOverridesTimestamp()
{
var ahora = DateTimeOffset.Now;
var jest = new DynamicTableEntity("ali", "ostad", "eTag", new Dictionary <string, EntityProperty>
{
{ "whah??", EntityProperty.GeneratePropertyForDateTimeOffset(ahora) }
});
jest.Timestamp = ahora.Subtract(TimeSpan.FromDays(42));
var source = new DiagnosticsSourceSummary();
source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] = "whah??";
Assert.Equal(ahora, jest.GetTimestamp(source));
}
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory,
Uri id,
DiagnosticsSourceSummary source,
ParseCursor cursor = null)
{
/*
*
* "count": 4,
* "total": 126,
* "minimum": 0,
* "maximum": 63,
* "average": 31.5,
* "resourceId": "/SUBSCRIPTIONS/9614FC94-9519-46FA-B7EC-DD1B0411DB13/RESOURCEGROUPS/WHASHA/PROVIDERS/MICROSOFT.CACHE/REDIS/FILLAPDWHASHAPRODUCTSEYHOOACHE",
* "time": "2018-01-18T12:55:00.0000000Z",
* "metricName": "connectedclients",
* "timeGrain": "PT1M"
*/
var ms = new MemoryStream();
var body = streamFactory();
body.CopyTo(ms);
ms.Position = 0;
var text = new StreamReader(ms).ReadToEnd();
var col = JsonConvert.DeserializeObject <InsightMetricCollection>(text);
foreach (var r in col.Records)
{
var subscriptionGuidFirstPart = r.ResourceId.Split('/')[2].Split('-')[0];
var pk = string.Format($"{subscriptionGuidFirstPart}_{string.Join("_", r.ResourceId.Split('/').Reverse().Take(3))}_{r.MetricName}");
var rk = r.Time.ToString("yyyyMMddHHmmss");
yield return(new Dictionary <string, string>()
{
{ "@timestamp", r.Time.ToString() },
{ "PartitionKey", pk },
{ "RowKey", rk },
{ "metricName", r.MetricName },
{ "resourceId", r.ResourceId },
{ "timeGrain", r.TimeGrain },
{ "count", r.Count.ToString(CultureInfo.InvariantCulture) },
{ "total", r.Total.ToString(CultureInfo.InvariantCulture) },
{ "minimum", r.Minimum.ToString(CultureInfo.InvariantCulture) },
{ "maximum", r.Maximum.ToString(CultureInfo.InvariantCulture) },
{ "average", r.Average.ToString(CultureInfo.InvariantCulture) }
});
}
}
public static DateTimeOffset GetTimestamp(this DynamicTableEntity entity, DiagnosticsSourceSummary source)
{
if (source.DynamicProperties.ContainsKey(ConveyorBeltConstants.TimestampFieldName) &&
source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] != null &&
source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] is string &&
entity.Properties.ContainsKey((string)source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName]))
{
var fieldName = (string)source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName];
if (entity.Properties[fieldName].PropertyType == EdmType.DateTime)
{
return(entity.Properties[fieldName].DateTimeOffsetValue.Value);
}
}
return(entity.Timestamp);
}
private DiagnosticsSourceSummary GetASummary()
{
var summary = new DiagnosticsSourceSummary()
{
ConnectionString = "cn",
IndexName = "in",
PartitionKey = "pk",
RowKey = "rk"
};
summary.DynamicProperties["dpi"] = (Int64)2;
summary.DynamicProperties["dps"] = "man";
summary.DynamicProperties["dpd"] = DateTime.UtcNow;
summary.DynamicProperties["dpb"] = true;
return(summary);
}
/// <summary>
/// Custom Source Properties:
/// 1- Parser
/// 2- EventHubName
/// 3- StorageConnectionString
/// </summary>
/// <param name="pusher"></param>
/// <param name="source"></param>
public EventHubConsumer(NestBatchPusher pusher, DiagnosticsSourceSummary source)
{
this._pusher = pusher;
this.Source = source;
_parser = FactoryHelper.Create <IParser>(source.DynamicProperties["Parser"].ToString());
_eventProcessorHost = new EventProcessorHost(
"ConveyorBelt",
source.DynamicProperties["EventHubName"].ToString(),
EventHubConsumerGroup.DefaultGroupName,
source.ConnectionString,
source.DynamicProperties["StorageConnectionString"].ToString());
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { TheTrace.TraceError(e.Exception.ToString()); };
_eventProcessorHost.RegisterEventProcessorFactoryAsync(this, options).Wait();
}
public DiagnosticsSourceSummary ToSummary()
{
var dss = new DiagnosticsSourceSummary()
{
ConnectionString = ConnectionString,
IndexName = IndexName,
PartitionKey = PartitionKey,
RowKey = RowKey,
TypeName = ToTypeKey(),
DynamicProperties = new Dictionary <string, object>()
};
foreach (var kv in _entity.Properties)
{
dss.DynamicProperties.Add(kv.Key, kv.Value.PropertyAsObject);
}
return(dss);
}
public DiagnosticsSourceSummary ToSummary()
{
var dss = new DiagnosticsSourceSummary()
{
ConnectionString = ConnectionString,
AccountSasKey = AccountSasKey,
AccountName = AccountName,
IndexName = IndexName,
PartitionKey = PartitionKey,
RowKey = RowKey,
TypeName = ToTypeKey(),
LastTimeOffset = FileOffset.Parse(LastOffsetPoint)?.TimeOffset,
DynamicProperties = new Dictionary <string, object>()
};
foreach (var kv in _entity.Properties)
{
dss.DynamicProperties.Add(kv.Key, kv.Value.PropertyAsObject);
}
return(dss);
}
public void TriesMultipleTimesAndSucceed()
{
var client = new Mock <IHttpClient>(MockBehavior.Loose);
var pusher = new ElasticsearchBatchPusher(client.Object, new AzureConfigurationValueProvider(), "http://google.com");
var summary = new DiagnosticsSourceSummary()
{
ConnectionString = String.Empty,
PartitionKey = "pk",
RowKey = "rk"
};
pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait();
pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait();
pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait();
pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait();
client.Setup(x => x.PostAsync(It.IsAny <string>(), It.IsAny <HttpContent>())).ReturnsAsync(
new HttpResponseMessage(HttpStatusCode.Accepted)
{
Content = new StringContent(File.ReadAllText(@"data\es_response_allsuxes.json"))
});
pusher.FlushAsync().Wait();
}
public void ConvertTableEntityToDictionary()
{
var ahora = DateTimeOffset.FromFileTime(129000000000000000).UtcDateTime;
var entity = new DynamicTableEntity("ali", "ostad", "eTag", new Dictionary <string, EntityProperty> {
{ "whah??", EntityProperty.GeneratePropertyForDateTimeOffset(ahora) },
{ "inty", EntityProperty.GeneratePropertyForInt(123) },
{ "doubly", EntityProperty.GeneratePropertyForDouble(123.23) },
{ "booly", EntityProperty.GeneratePropertyForBool(false) },
{ "stringy", EntityProperty.GeneratePropertyForString("magical unicorns") },
{ "ignored1", EntityProperty.GeneratePropertyForString(",") },
{ "ignored2", EntityProperty.GeneratePropertyForString("") }
})
{
Timestamp = ahora.Subtract(TimeSpan.FromDays(42))
};
var source = new DiagnosticsSourceSummary {
TypeName = "typename"
};
source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] = "whah??";
var dict = entity.ToDictionary(source);
Assert.Equal("typename", dict["cb_type"]);
Assert.Equal("ali", dict["PartitionKey"]);
Assert.Equal("ostad", dict["RowKey"]);
Assert.Equal(ahora.ToString("s"), dict["whah??"]);
Assert.Equal("123", dict["inty"]);
Assert.Equal("123.23", dict["doubly"]);
Assert.Equal("false", dict["booly"]);
Assert.Equal("magical unicorns", dict["stringy"]);
Assert.Equal(ahora.ToString("s"), dict["@timestamp"]);
Assert.False(dict.ContainsKey("ignored1"));
Assert.False(dict.ContainsKey("ignored2"));
Assert.Equal(9, dict.Count);
}
public static IDictionary <string, string> ToDictionary(this DynamicTableEntity entity, DiagnosticsSourceSummary source)
{
const string PartitionKey = "PartitionKey";
const string RowKey = "RowKey";
const string CbType = "cb_type";
const string Timestamp = "@timestamp";
var result = new Dictionary <string, string> {
{ PartitionKey, entity.PartitionKey },
{ RowKey, entity.RowKey },
{ CbType, source.TypeName },
{ Timestamp, entity.GetTimestamp(source).ToString("s") }
};
foreach (var property in entity.Properties)
{
string val;
switch (property.Value.PropertyType)
{
case EdmType.DateTime:
val = property.Value.DateTimeOffsetValue?.ToString("s");
break;
case EdmType.Boolean:
val = property.Value.BooleanValue?.ToString().ToLower();
break;
default:
val = property.Value.PropertyAsObject.ToString();
break;
}
if (string.IsNullOrWhiteSpace(val) || val == ",")
{
continue;
}
result.Add(property.Key, val);
}
return(result);
}
private static IDictionary <string, string> ToDic(JToken singleJ, DiagnosticsSourceSummary source)
{
var dic = new Dictionary <string, string>();
string goodDateTime = null;
string okDateTime = null;
string anyDateTime = null;
foreach (var child in singleJ.Children())
{
if (child.Type == JTokenType.Property)
{
var prop = (JProperty)child;
var value = ((JValue)prop.Value).Value;
if (value != null)
{
string dateValue = null;
if (value.GetType() == typeof(DateTime))
{
dic.Add(prop.Name, ((DateTime)value).ToString("O"));
dateValue = dic[prop.Name];
}
else if (value.GetType() == typeof(DateTimeOffset))
{
dic.Add(prop.Name, ((DateTimeOffset)value).ToString("O"));
dateValue = dic[prop.Name];
}
else
{
dic.Add(prop.Name, value.ToString());
}
if (dateValue != null)
{
if (prop.Name.Equals("Timestamp", StringComparison.CurrentCultureIgnoreCase) || prop.Name.Equals("EventDate", StringComparison.CurrentCultureIgnoreCase))
{
goodDateTime = goodDateTime ?? dateValue;
}
if (prop.Name.ToLower().Contains("date") || prop.Name.ToLower().Contains("time"))
{
okDateTime = okDateTime ?? dateValue;
}
anyDateTime = anyDateTime ?? dateValue;
}
}
}
}
var datetimeValue = goodDateTime ?? okDateTime ?? anyDateTime ?? DateTimeOffset.UtcNow.ToString("O");
dic.Add("@timestamp", datetimeValue);
if (!dic.ContainsKey("PartitionKey") || !dic.ContainsKey("RowKey"))
{
dic["PartitionKey"] = "nopart_";
dic["RowKey"] = Guid.NewGuid().ToString("N");
}
dic["cb_type"] = source.TypeName;
return(dic);
}
private static IDictionary <string, string> MapLogItemToTableEntity(SitecoreLogEntry logItem, DiagnosticsSourceSummary source, string partitionKey, string fileName)
{
return(new Dictionary <string, string> {
{ SitecoreLogFields.Timestamp, logItem.LogDateTime.ToString("s") },
{ SitecoreLogFields.PartitionKey, partitionKey },
{ SitecoreLogFields.RowKey, $"{fileName}_{logItem.LogLineNumber}" },
{ SitecoreLogFields.CbType, source.TypeName },
{ SitecoreLogFields.Level, logItem.Level },
{ SitecoreLogFields.ProcessId, logItem.EventSource },
{ SitecoreLogFields.Message, logItem.Message }
});
}
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null)
{
cursor = cursor ?? new ParseCursor(0);
var body = streamFactory();
if (body.Position != 0)
{
body.Position = 0;
}
if (cursor.EndPosition == 0)
{
cursor.EndPosition = long.MaxValue;
}
string line;
var lineNumber = 0;
var fileName = GetFileName(id);
var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray();
var partionKey = string.Join("_", idSegments.Take(idSegments.Length - 1));
var fileDate = GetFileDate(fileName);
var logLineParser = new SitecoreLogLineParser();
SitecoreLogEntry currentEntry = null;
var reader = new StreamReader(body);
while (body.Position < cursor.EndPosition && (line = reader.ReadLine()) != null)
{
lineNumber++;
if (body.Position < cursor.StartReadPosition)
{
continue;
}
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
//remove connection string passwords in exceptions etc (thanks sitecore)
line = ReplaceSecureInformation(line, "password="******"**PASSWORD**REDACTED**");
line = ReplaceSecureInformation(line, "user id=", "**USER**REDACTED**");
var logItem = logLineParser.ParseLine(line, fileDate);
if (logItem != null)
{
var logText = logItem.Message;
//filter out rubbish logs. eg blank INFO during sitecore startup
if (string.IsNullOrWhiteSpace(logText) || logText.StartsWith("*****"))
{
continue;
}
logItem.LogLineNumber = lineNumber;
}
if (logItem == null && currentEntry != null)
{
// Existing multiline message
currentEntry.AppendMessageText(line);
}
else if (logItem != null && currentEntry == null)
{
// new entry found
currentEntry = logItem;
}
else if (currentEntry != null)
{
//new entry found, current one is completed.
yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName));
currentEntry = logItem;
}
}
if (currentEntry != null)
{
yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName));
}
}
private IEnumerable <IDictionary <string, string> > ParseInternal(StreamReader reader, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor)
{
if (cursor.EndPosition == 0)
{
cursor.EndPosition = long.MaxValue;
}
var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray();
var partitionKey = string.Join("_", idSegments.Take(idSegments.Length - 1));
var rowKeyPrefix = Path.GetFileNameWithoutExtension(idSegments.Last());
var fields = cursor.StartReadPosition > 0 ? ReadFirstHeaders(reader) : null;
reader.DiscardBufferedData();
reader.BaseStream.Position = cursor.StartReadPosition;
var headersHaveChanged = false;
string line;
var offset = cursor.StartReadPosition;
var lineCount = 0;
//In case starting offset was set to a line break
var lineBreakIndex = Environment.NewLine.IndexOf((char)reader.Peek());
if (lineBreakIndex >= 0)
{
reader.ReadLine();
offset += Environment.NewLine.Length - lineBreakIndex;
}
while (offset < cursor.EndPosition && (line = reader.ReadLine()) != null)
{
lineCount++;
offset += line.Length + Environment.NewLine.Length;
if (line.StartsWith("#Fields: "))
{
fields = BuildFields(line);
continue;
}
if (line.StartsWith("#") || fields == null)//make sure that fields are set before parsing them
{
continue;
}
//skip until start offset in case of re-read
if (offset <= cursor.StartParsePosition)
{
continue;
}
var entries = GetEntries(line);
if (fields.Length + 2 != entries.Length)//first 2 entries - date and time are collapsed into @timestamp
{
if (cursor.StartReadPosition == 0)
{
throw new InvalidOperationException("fields column mismatch");
}
//e.g. in case startReadPosition was pointing to the middle of the line
//have to skip to next line
if (lineCount == 1)
{
continue;
}
headersHaveChanged = true;
break;
}
var doc = ParseEntity(fields, entries, source.TypeName, partitionKey, $"{rowKeyPrefix}_{offset}");
if (doc != null)
{
yield return(doc);
}
cursor.StartReadPosition = offset;
}
if (!headersHaveChanged)
{
yield break;
}
//headers have changed since the beginning of the file - have to read whole file
cursor.StartReadPosition = 0;
foreach (var doc in ParseInternal(reader, id, source, cursor))
{
yield return(doc);
}
}
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null)
{
cursor = cursor ?? new ParseCursor(0);
StorageException lastException = null;
var resumeAttemptCount = 0;
while (resumeAttemptCount < 3)
{
using (var stream = streamFactory())
using (var reader = new StreamReader(stream, Encoding.ASCII, false, 1024 * 1024 * 6))
using (var enumerator = ParseInternal(reader, id, source, cursor).GetEnumerator())
{
var wasInterrupted = false;
bool hasMoved;
do
{
if (enumerator.Current != null)
{
yield return(enumerator.Current);
}
try
{
hasMoved = enumerator.MoveNext();
}
catch (StorageException ex)
{
lastException = ex;
wasInterrupted = true;
resumeAttemptCount++;
break;
}
} while (hasMoved);
if (!wasInterrupted)
{
cursor.EndPosition = stream.Length;
yield break;
}
}
}
}
private DiagnosticsSourceSummary GetASummary()
{
var summary = new DiagnosticsSourceSummary()
{
ConnectionString = "cn",
IndexName = "in",
PartitionKey = "pk",
RowKey = "rk"
};
summary.DynamicProperties["dpi"] = (Int64) 2;
summary.DynamicProperties["dps"] = "man";
summary.DynamicProperties["dpd"] = DateTime.UtcNow;
summary.DynamicProperties["dpb"] = true;
return summary;
}
