public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor parseCursor = null) { var reader = new StreamReader(streamFactory()); var s = reader.ReadToEnd(); var j = s.StartsWith("[") ? (JToken)JArray.Parse(s) : JObject.Parse(s); var result = new List <IDictionary <string, string> >(); if (j.Type == JTokenType.Array) { foreach (var child in j.Children()) { if (child.Type == JTokenType.Object) { result.Add(ToDic(child, source)); } } } else { result.Add(ToDic(j, source)); } return(result); }
public EventProcessor(NestBatchPusher elasticsearchBatchPusher, IParser parser, DiagnosticsSourceSummary source) { this._elasticsearchBatchPusher = elasticsearchBatchPusher; this._parser = parser; this._source = source; }
public void IfTimestampFieldSpecifiedOverridesTimestamp() { var ahora = DateTimeOffset.Now; var jest = new DynamicTableEntity("ali", "ostad", "eTag", new Dictionary <string, EntityProperty> { { "whah??", EntityProperty.GeneratePropertyForDateTimeOffset(ahora) } }); jest.Timestamp = ahora.Subtract(TimeSpan.FromDays(42)); var source = new DiagnosticsSourceSummary(); source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] = "whah??"; Assert.Equal(ahora, jest.GetTimestamp(source)); }
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null) { /* * * "count": 4, * "total": 126, * "minimum": 0, * "maximum": 63, * "average": 31.5, * "resourceId": "/SUBSCRIPTIONS/9614FC94-9519-46FA-B7EC-DD1B0411DB13/RESOURCEGROUPS/WHASHA/PROVIDERS/MICROSOFT.CACHE/REDIS/FILLAPDWHASHAPRODUCTSEYHOOACHE", * "time": "2018-01-18T12:55:00.0000000Z", * "metricName": "connectedclients", * "timeGrain": "PT1M" */ var ms = new MemoryStream(); var body = streamFactory(); body.CopyTo(ms); ms.Position = 0; var text = new StreamReader(ms).ReadToEnd(); var col = JsonConvert.DeserializeObject <InsightMetricCollection>(text); foreach (var r in col.Records) { var subscriptionGuidFirstPart = r.ResourceId.Split('/')[2].Split('-')[0]; var pk = string.Format($"{subscriptionGuidFirstPart}_{string.Join("_", r.ResourceId.Split('/').Reverse().Take(3))}_{r.MetricName}"); var rk = r.Time.ToString("yyyyMMddHHmmss"); yield return(new Dictionary <string, string>() { { "@timestamp", r.Time.ToString() }, { "PartitionKey", pk }, { "RowKey", rk }, { "metricName", r.MetricName }, { "resourceId", r.ResourceId }, { "timeGrain", r.TimeGrain }, { "count", r.Count.ToString(CultureInfo.InvariantCulture) }, { "total", r.Total.ToString(CultureInfo.InvariantCulture) }, { "minimum", r.Minimum.ToString(CultureInfo.InvariantCulture) }, { "maximum", r.Maximum.ToString(CultureInfo.InvariantCulture) }, { "average", r.Average.ToString(CultureInfo.InvariantCulture) } }); } }
public static DateTimeOffset GetTimestamp(this DynamicTableEntity entity, DiagnosticsSourceSummary source) { if (source.DynamicProperties.ContainsKey(ConveyorBeltConstants.TimestampFieldName) && source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] != null && source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] is string && entity.Properties.ContainsKey((string)source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName])) { var fieldName = (string)source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName]; if (entity.Properties[fieldName].PropertyType == EdmType.DateTime) { return(entity.Properties[fieldName].DateTimeOffsetValue.Value); } } return(entity.Timestamp); }
private DiagnosticsSourceSummary GetASummary() { var summary = new DiagnosticsSourceSummary() { ConnectionString = "cn", IndexName = "in", PartitionKey = "pk", RowKey = "rk" }; summary.DynamicProperties["dpi"] = (Int64)2; summary.DynamicProperties["dps"] = "man"; summary.DynamicProperties["dpd"] = DateTime.UtcNow; summary.DynamicProperties["dpb"] = true; return(summary); }
/// <summary> /// Custom Source Properties: /// 1- Parser /// 2- EventHubName /// 3- StorageConnectionString /// </summary> /// <param name="pusher"></param> /// <param name="source"></param> public EventHubConsumer(NestBatchPusher pusher, DiagnosticsSourceSummary source) { this._pusher = pusher; this.Source = source; _parser = FactoryHelper.Create <IParser>(source.DynamicProperties["Parser"].ToString()); _eventProcessorHost = new EventProcessorHost( "ConveyorBelt", source.DynamicProperties["EventHubName"].ToString(), EventHubConsumerGroup.DefaultGroupName, source.ConnectionString, source.DynamicProperties["StorageConnectionString"].ToString()); var options = new EventProcessorOptions(); options.ExceptionReceived += (sender, e) => { TheTrace.TraceError(e.Exception.ToString()); }; _eventProcessorHost.RegisterEventProcessorFactoryAsync(this, options).Wait(); }
public DiagnosticsSourceSummary ToSummary() { var dss = new DiagnosticsSourceSummary() { ConnectionString = ConnectionString, IndexName = IndexName, PartitionKey = PartitionKey, RowKey = RowKey, TypeName = ToTypeKey(), DynamicProperties = new Dictionary <string, object>() }; foreach (var kv in _entity.Properties) { dss.DynamicProperties.Add(kv.Key, kv.Value.PropertyAsObject); } return(dss); }
public DiagnosticsSourceSummary ToSummary() { var dss = new DiagnosticsSourceSummary() { ConnectionString = ConnectionString, AccountSasKey = AccountSasKey, AccountName = AccountName, IndexName = IndexName, PartitionKey = PartitionKey, RowKey = RowKey, TypeName = ToTypeKey(), LastTimeOffset = FileOffset.Parse(LastOffsetPoint)?.TimeOffset, DynamicProperties = new Dictionary <string, object>() }; foreach (var kv in _entity.Properties) { dss.DynamicProperties.Add(kv.Key, kv.Value.PropertyAsObject); } return(dss); }
public void TriesMultipleTimesAndSucceed() { var client = new Mock <IHttpClient>(MockBehavior.Loose); var pusher = new ElasticsearchBatchPusher(client.Object, new AzureConfigurationValueProvider(), "http://google.com"); var summary = new DiagnosticsSourceSummary() { ConnectionString = String.Empty, PartitionKey = "pk", RowKey = "rk" }; pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait(); pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait(); pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait(); pusher.PushAsync(new DynamicTableEntity("pk", "rk"), summary).Wait(); client.Setup(x => x.PostAsync(It.IsAny <string>(), It.IsAny <HttpContent>())).ReturnsAsync( new HttpResponseMessage(HttpStatusCode.Accepted) { Content = new StringContent(File.ReadAllText(@"data\es_response_allsuxes.json")) }); pusher.FlushAsync().Wait(); }
public void ConvertTableEntityToDictionary() { var ahora = DateTimeOffset.FromFileTime(129000000000000000).UtcDateTime; var entity = new DynamicTableEntity("ali", "ostad", "eTag", new Dictionary <string, EntityProperty> { { "whah??", EntityProperty.GeneratePropertyForDateTimeOffset(ahora) }, { "inty", EntityProperty.GeneratePropertyForInt(123) }, { "doubly", EntityProperty.GeneratePropertyForDouble(123.23) }, { "booly", EntityProperty.GeneratePropertyForBool(false) }, { "stringy", EntityProperty.GeneratePropertyForString("magical unicorns") }, { "ignored1", EntityProperty.GeneratePropertyForString(",") }, { "ignored2", EntityProperty.GeneratePropertyForString("") } }) { Timestamp = ahora.Subtract(TimeSpan.FromDays(42)) }; var source = new DiagnosticsSourceSummary { TypeName = "typename" }; source.DynamicProperties[ConveyorBeltConstants.TimestampFieldName] = "whah??"; var dict = entity.ToDictionary(source); Assert.Equal("typename", dict["cb_type"]); Assert.Equal("ali", dict["PartitionKey"]); Assert.Equal("ostad", dict["RowKey"]); Assert.Equal(ahora.ToString("s"), dict["whah??"]); Assert.Equal("123", dict["inty"]); Assert.Equal("123.23", dict["doubly"]); Assert.Equal("false", dict["booly"]); Assert.Equal("magical unicorns", dict["stringy"]); Assert.Equal(ahora.ToString("s"), dict["@timestamp"]); Assert.False(dict.ContainsKey("ignored1")); Assert.False(dict.ContainsKey("ignored2")); Assert.Equal(9, dict.Count); }
public static IDictionary <string, string> ToDictionary(this DynamicTableEntity entity, DiagnosticsSourceSummary source) { const string PartitionKey = "PartitionKey"; const string RowKey = "RowKey"; const string CbType = "cb_type"; const string Timestamp = "@timestamp"; var result = new Dictionary <string, string> { { PartitionKey, entity.PartitionKey }, { RowKey, entity.RowKey }, { CbType, source.TypeName }, { Timestamp, entity.GetTimestamp(source).ToString("s") } }; foreach (var property in entity.Properties) { string val; switch (property.Value.PropertyType) { case EdmType.DateTime: val = property.Value.DateTimeOffsetValue?.ToString("s"); break; case EdmType.Boolean: val = property.Value.BooleanValue?.ToString().ToLower(); break; default: val = property.Value.PropertyAsObject.ToString(); break; } if (string.IsNullOrWhiteSpace(val) || val == ",") { continue; } result.Add(property.Key, val); } return(result); }
private static IDictionary <string, string> ToDic(JToken singleJ, DiagnosticsSourceSummary source) { var dic = new Dictionary <string, string>(); string goodDateTime = null; string okDateTime = null; string anyDateTime = null; foreach (var child in singleJ.Children()) { if (child.Type == JTokenType.Property) { var prop = (JProperty)child; var value = ((JValue)prop.Value).Value; if (value != null) { string dateValue = null; if (value.GetType() == typeof(DateTime)) { dic.Add(prop.Name, ((DateTime)value).ToString("O")); dateValue = dic[prop.Name]; } else if (value.GetType() == typeof(DateTimeOffset)) { dic.Add(prop.Name, ((DateTimeOffset)value).ToString("O")); dateValue = dic[prop.Name]; } else { dic.Add(prop.Name, value.ToString()); } if (dateValue != null) { if (prop.Name.Equals("Timestamp", StringComparison.CurrentCultureIgnoreCase) || prop.Name.Equals("EventDate", StringComparison.CurrentCultureIgnoreCase)) { goodDateTime = goodDateTime ?? dateValue; } if (prop.Name.ToLower().Contains("date") || prop.Name.ToLower().Contains("time")) { okDateTime = okDateTime ?? dateValue; } anyDateTime = anyDateTime ?? dateValue; } } } } var datetimeValue = goodDateTime ?? okDateTime ?? anyDateTime ?? DateTimeOffset.UtcNow.ToString("O"); dic.Add("@timestamp", datetimeValue); if (!dic.ContainsKey("PartitionKey") || !dic.ContainsKey("RowKey")) { dic["PartitionKey"] = "nopart_"; dic["RowKey"] = Guid.NewGuid().ToString("N"); } dic["cb_type"] = source.TypeName; return(dic); }
private static IDictionary <string, string> MapLogItemToTableEntity(SitecoreLogEntry logItem, DiagnosticsSourceSummary source, string partitionKey, string fileName) { return(new Dictionary <string, string> { { SitecoreLogFields.Timestamp, logItem.LogDateTime.ToString("s") }, { SitecoreLogFields.PartitionKey, partitionKey }, { SitecoreLogFields.RowKey, $"{fileName}_{logItem.LogLineNumber}" }, { SitecoreLogFields.CbType, source.TypeName }, { SitecoreLogFields.Level, logItem.Level }, { SitecoreLogFields.ProcessId, logItem.EventSource }, { SitecoreLogFields.Message, logItem.Message } }); }
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null) { cursor = cursor ?? new ParseCursor(0); var body = streamFactory(); if (body.Position != 0) { body.Position = 0; } if (cursor.EndPosition == 0) { cursor.EndPosition = long.MaxValue; } string line; var lineNumber = 0; var fileName = GetFileName(id); var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray(); var partionKey = string.Join("_", idSegments.Take(idSegments.Length - 1)); var fileDate = GetFileDate(fileName); var logLineParser = new SitecoreLogLineParser(); SitecoreLogEntry currentEntry = null; var reader = new StreamReader(body); while (body.Position < cursor.EndPosition && (line = reader.ReadLine()) != null) { lineNumber++; if (body.Position < cursor.StartReadPosition) { continue; } if (string.IsNullOrWhiteSpace(line)) { continue; } //remove connection string passwords in exceptions etc (thanks sitecore) line = ReplaceSecureInformation(line, "password="******"**PASSWORD**REDACTED**"); line = ReplaceSecureInformation(line, "user id=", "**USER**REDACTED**"); var logItem = logLineParser.ParseLine(line, fileDate); if (logItem != null) { var logText = logItem.Message; //filter out rubbish logs. eg blank INFO during sitecore startup if (string.IsNullOrWhiteSpace(logText) || logText.StartsWith("*****")) { continue; } logItem.LogLineNumber = lineNumber; } if (logItem == null && currentEntry != null) { // Existing multiline message currentEntry.AppendMessageText(line); } else if (logItem != null && currentEntry == null) { // new entry found currentEntry = logItem; } else if (currentEntry != null) { //new entry found, current one is completed. yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName)); currentEntry = logItem; } } if (currentEntry != null) { yield return(MapLogItemToTableEntity(currentEntry, source, partionKey, fileName)); } }
private IEnumerable <IDictionary <string, string> > ParseInternal(StreamReader reader, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor) { if (cursor.EndPosition == 0) { cursor.EndPosition = long.MaxValue; } var idSegments = id.Segments.Skip(2).Select(x => x.Replace("/", "")).ToArray(); var partitionKey = string.Join("_", idSegments.Take(idSegments.Length - 1)); var rowKeyPrefix = Path.GetFileNameWithoutExtension(idSegments.Last()); var fields = cursor.StartReadPosition > 0 ? ReadFirstHeaders(reader) : null; reader.DiscardBufferedData(); reader.BaseStream.Position = cursor.StartReadPosition; var headersHaveChanged = false; string line; var offset = cursor.StartReadPosition; var lineCount = 0; //In case starting offset was set to a line break var lineBreakIndex = Environment.NewLine.IndexOf((char)reader.Peek()); if (lineBreakIndex >= 0) { reader.ReadLine(); offset += Environment.NewLine.Length - lineBreakIndex; } while (offset < cursor.EndPosition && (line = reader.ReadLine()) != null) { lineCount++; offset += line.Length + Environment.NewLine.Length; if (line.StartsWith("#Fields: ")) { fields = BuildFields(line); continue; } if (line.StartsWith("#") || fields == null)//make sure that fields are set before parsing them { continue; } //skip until start offset in case of re-read if (offset <= cursor.StartParsePosition) { continue; } var entries = GetEntries(line); if (fields.Length + 2 != entries.Length)//first 2 entries - date and time are collapsed into @timestamp { if (cursor.StartReadPosition == 0) { throw new InvalidOperationException("fields column mismatch"); } //e.g. in case startReadPosition was pointing to the middle of the line //have to skip to next line if (lineCount == 1) { continue; } headersHaveChanged = true; break; } var doc = ParseEntity(fields, entries, source.TypeName, partitionKey, $"{rowKeyPrefix}_{offset}"); if (doc != null) { yield return(doc); } cursor.StartReadPosition = offset; } if (!headersHaveChanged) { yield break; } //headers have changed since the beginning of the file - have to read whole file cursor.StartReadPosition = 0; foreach (var doc in ParseInternal(reader, id, source, cursor)) { yield return(doc); } }
public IEnumerable <IDictionary <string, string> > Parse(Func <Stream> streamFactory, Uri id, DiagnosticsSourceSummary source, ParseCursor cursor = null) { cursor = cursor ?? new ParseCursor(0); StorageException lastException = null; var resumeAttemptCount = 0; while (resumeAttemptCount < 3) { using (var stream = streamFactory()) using (var reader = new StreamReader(stream, Encoding.ASCII, false, 1024 * 1024 * 6)) using (var enumerator = ParseInternal(reader, id, source, cursor).GetEnumerator()) { var wasInterrupted = false; bool hasMoved; do { if (enumerator.Current != null) { yield return(enumerator.Current); } try { hasMoved = enumerator.MoveNext(); } catch (StorageException ex) { lastException = ex; wasInterrupted = true; resumeAttemptCount++; break; } } while (hasMoved); if (!wasInterrupted) { cursor.EndPosition = stream.Length; yield break; } } } }
private DiagnosticsSourceSummary GetASummary() { var summary = new DiagnosticsSourceSummary() { ConnectionString = "cn", IndexName = "in", PartitionKey = "pk", RowKey = "rk" }; summary.DynamicProperties["dpi"] = (Int64) 2; summary.DynamicProperties["dps"] = "man"; summary.DynamicProperties["dpd"] = DateTime.UtcNow; summary.DynamicProperties["dpb"] = true; return summary; }