public async Task PushAsync(DynamicTableEntity entity, DiagnosticsSourceSummary source) { if (source.Filter == null) { source.Filter = string.Empty; } if (!_filters.ContainsKey(source.Filter)) { _filters.Add(source.Filter, new SimpleFilter(source.Filter)); } if (!_filters[source.Filter].Satisfies(entity)) { return; } var op = new { index = new { _index = source.IndexName ?? _indexNamer.BuildName(entity.Timestamp, source.DynamicProperties["MappingName"].ToString().ToLowerInvariant()), _type = source.DynamicProperties["MappingName"].ToString(), _id = entity.PartitionKey + entity.RowKey } }; var doc = new JObject(); doc.Add("@timestamp", entity.Timestamp); doc.Add("PartitionKey", entity.PartitionKey); doc.Add("RowKey", entity.RowKey); doc.Add("cb_type", source.TypeName); foreach (var property in entity.Properties) { if (property.Key != DiagnosticsSource.CustomAttributesFieldName) { doc[property.Key] = JToken.FromObject(property.Value.PropertyAsObject); } } if (entity.Properties.ContainsKey(DiagnosticsSource.CustomAttributesFieldName)) { foreach (var keyValue in GetNameValues(entity.Properties[DiagnosticsSource.CustomAttributesFieldName].StringValue)) { doc[keyValue.Key] = keyValue.Value; } } _batch.AddDoc(JsonConvert.SerializeObject(op).Replace("\r\n", " "), doc.ToString().Replace("\r\n", " ")); if (_batch.Count >= _batchSize) { await PushbatchAsync(); TheTrace.TraceInformation("ConveyorBelt_Pusher: Pushed records to ElasticSearch for {0}-{1}", source.PartitionKey, source.RowKey); } }
public async Task PushAsync(DynamicTableEntity entity, DiagnosticsSourceSummary source) { var op = new { index = new { _index = source.IndexName ?? entity.Timestamp.ToString("yyyyMMdd"), _type = source.TypeName, _id = entity.PartitionKey + entity.RowKey } }; var doc = new JObject(); doc.Add("@timestamp", entity.Timestamp); doc.Add("PartitionKey", entity.PartitionKey); doc.Add("RowKey", entity.RowKey); foreach (var property in entity.Properties) { if (property.Key != DiagnosticsSource.CustomAttributesFieldName) { doc[property.Key] = JToken.FromObject(property.Value.PropertyAsObject); } } if (entity.Properties.ContainsKey(DiagnosticsSource.CustomAttributesFieldName)) { foreach (var keyValue in GetNameValues(entity.Properties[DiagnosticsSource.CustomAttributesFieldName].StringValue)) { doc[keyValue.Key] = keyValue.Value; } } _stringBuilder.Append(JsonConvert.SerializeObject(op).Replace("\r\n", " ")); _stringBuilder.Append('\n'); _stringBuilder.Append(doc.ToString().Replace("\r\n", " ")); _stringBuilder.Append('\n'); if ((_numberOfRecords++) >= _batchSize) { var nrec = _numberOfRecords; await PushbatchAsync(); TheTrace.TraceInformation("ConveyorBelt_Pusher: Pushed {0} records to ElasticSearch for {1}-{2}", nrec, source.PartitionKey, source.RowKey); } }
public async Task PushAsync(DynamicTableEntity entity, DiagnosticsSourceSummary source) { var op = new { index = new { _index = source.IndexName ?? entity.Timestamp.ToString("yyyyMMdd"), _type = source.TypeName, _id = entity.PartitionKey + entity.RowKey } }; var doc = new JObject(); doc.Add("@timestamp", entity.Timestamp); doc.Add("PartitionKey", entity.PartitionKey); doc.Add("RowKey", entity.RowKey); foreach (var property in entity.Properties) { if (property.Key != DiagnosticsSource.CustomAttributesFieldName) doc[property.Key] = JToken.FromObject(property.Value.PropertyAsObject); } if (entity.Properties.ContainsKey(DiagnosticsSource.CustomAttributesFieldName)) { foreach (var keyValue in GetNameValues(entity.Properties[DiagnosticsSource.CustomAttributesFieldName].StringValue)) { doc[keyValue.Key] = keyValue.Value; } } _stringBuilder.Append(JsonConvert.SerializeObject(op).Replace("\r\n", " ")); _stringBuilder.Append('\n'); _stringBuilder.Append(doc.ToString().Replace("\r\n", " ")); _stringBuilder.Append('\n'); if ((_numberOfRecords++) >= _batchSize) { var nrec = _numberOfRecords; await PushbatchAsync(); TheTrace.TraceInformation("ConveyorBelt_Pusher: Pushed {0} records to ElasticSearch for {1}-{2}", nrec, source.PartitionKey, source.RowKey); } }
public async Task <int> PushAll(IEnumerable <DynamicTableEntity> lazyEnumerable, DiagnosticsSourceSummary source) { return(await PushAllImpl( lazyEnumerable.Select(entity => entity.ToDictionary(source)), source.DynamicProperties["MappingName"].ToString() ).ConfigureAwait(false)); }
public async Task <int> PushAll(IEnumerable <IDictionary <string, string> > lazyEnumerable, DiagnosticsSourceSummary source) { return(await PushAllImpl( lazyEnumerable, source.DynamicProperties["MappingName"].ToString() ).ConfigureAwait(false)); }