private async Task SetupMappingsAsync(DiagnosticsSource source) { foreach (var indexName in source.GetIndexNames()) { var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl); await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName); if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey())) { var jsonPath = string.Format("{0}{1}.json", _configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath), source.GetMappingName()); var response = await _nonAuthenticatingClient.GetAsync(jsonPath); if (response.Content == null) { throw new ApplicationException(response.ToString()); } var content = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { throw new InvalidOperationException(content); } var mapping = content.Replace("___type_name___", source.ToTypeKey()); await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping); } } }
public async Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source) { // if Stop offset has been reached if (!string.IsNullOrEmpty(source.StopOffsetPoint) && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0) { return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false)); } var lockToken = new LockToken(source.ToTypeKey()); int seconds = Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds)); if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000))) { TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey()); return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false)); } try { var events = await DoSchedule(source); return(new Tuple <IEnumerable <Event>, bool>(events, true)); } finally { Task.Run(() => _lockStore.ReleaseLockAsync(lockToken)).Wait(); } }
public Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source) { var key = source.ToTypeKey(); if (source.IsActive.GetValueOrDefault(true)) { var consume = EventHubConsumer.Consumers.AddOrUpdate(key, new Lazy <EventHubConsumer>(() => { TheTrace.TraceInformation("Just added this eventHub consumer {0}", key); return(new EventHubConsumer(_pusher, source.ToSummary())); }), (kk, vv) => vv); // to make sure it gets accessed and created if new otherwise system is 'lazy' TheTrace.TraceInformation("This is the EventHub thing I was talking about: {0}", consume.Value.Source.TypeName); } else { Lazy <EventHubConsumer> consumer = null; if (EventHubConsumer.Consumers.TryRemove(key, out consumer)) { consumer.Value.Dispose(); TheTrace.TraceInformation("Just removed this eventHub consumer {0}", key); } } return(Task.FromResult(new Tuple <IEnumerable <Event>, bool>(new Event[0], false))); }
private async Task SetupMappingsAsync(DiagnosticsSource source) { foreach (var indexName in source.GetIndexNames()) { var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl); await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName); if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey())) { var jsonPath = string.Format("{0}{1}.json", _configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath), source.GetMappingName()); var client = new WebClient(); var mapping = client.DownloadString(jsonPath).Replace("___type_name___", source.ToTypeKey()); await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping); } } }
public void CanConvertFromSourceToSummary_WithAlternateTypeName() { var entity = new DynamicTableEntity("pk", "rk"); entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2); entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi"); var source = new DiagnosticsSource(entity); var summary = source.ToSummary(); Assert.Equal(summary.TypeName, source.ToTypeKey()); }
public void CanConvertFromSourceToSummary_WithAlternateTypeName() { var entity = new DynamicTableEntity("pk", "rk"); entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2); entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi"); var source = new DiagnosticsSource(entity); var summary = source.ToSummary(); Assert.Equal(summary.TypeName, source.ToTypeKey()); }
public async Task<Tuple<IEnumerable<Event>, bool>> TryScheduleAsync(DiagnosticsSource source) { // if Stop offset has been reached if (source.StopOffsetPoint != null && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0) return new Tuple<IEnumerable<Event>, bool>(Enumerable.Empty<Event>(), false); var lockToken = new LockToken(source.ToTypeKey()); int seconds = Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds)); if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000))) { TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey()); return new Tuple<IEnumerable<Event>, bool>(Enumerable.Empty<Event>(), false); } try { var events = await DoSchedule(source); return new Tuple<IEnumerable<Event>, bool>(events, true); } finally { Task.Run( () => _lockStore.ReleaseLockAsync(lockToken)).Wait(); } }
public void CanConvertFromSourceToSummary() { var entity = new DynamicTableEntity("pk", "rk"); entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2); entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man"); entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow); entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true); var source = new DiagnosticsSource(entity); var summary = source.ToSummary(); Assert.Equal(summary.ConnectionString, source.ConnectionString); Assert.Equal(summary.PartitionKey, source.PartitionKey); Assert.Equal(summary.RowKey, source.RowKey); Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty<DateTime>("dpd")); Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty<int>("dpi")); Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty<string>("dps")); Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty<bool>("dpb")); Assert.Equal(summary.TypeName, source.ToTypeKey()); }
public void CanConvertFromSourceToSummary() { var entity = new DynamicTableEntity("pk", "rk"); entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2); entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man"); entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow); entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true); var source = new DiagnosticsSource(entity); var summary = source.ToSummary(); Assert.Equal(summary.ConnectionString, source.ConnectionString); Assert.Equal(summary.PartitionKey, source.PartitionKey); Assert.Equal(summary.RowKey, source.RowKey); Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty <DateTime>("dpd")); Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty <int>("dpi")); Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty <string>("dps")); Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty <bool>("dpb")); Assert.Equal(summary.TypeName, source.ToTypeKey()); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { if (source.LastOffsetPoint == null) { source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-1).DropSecondAndMilliseconds().ToString("O"); } var lastOffset = DateTimeOffset.Parse(source.LastOffsetPoint); var events = new List <Event>(); var graceMinutes = source.GracePeriodMinutes ?? 3; var now = DateTimeOffset.UtcNow.DropSecondAndMilliseconds(); var newLastOffset = lastOffset; int n = 1; // start from a minute after while (now >= lastOffset.Add(TimeSpan.FromMinutes(graceMinutes + n))) { newLastOffset = lastOffset.Add(TimeSpan.FromMinutes(n)) .DropSecondAndMilliseconds(); // just to be sure var shardKeys = GetShardKeys(newLastOffset); events.AddRange(shardKeys.Select(shardKey => new Event(new ShardKeyArrived { Source = source.ToSummary(), ShardKey = shardKey }))); if (source.MaxItemsInAScheduleRun.HasValue && n >= source.MaxItemsInAScheduleRun) { break; } n++; TheTrace.TraceInformation("Scheduling {0} for minute {1} and shardkey {2} => {0}_{1} AND {0}_{2}", source.ToTypeKey(), newLastOffset.ToString("yyyyMMddHHmm"), shardKeys.First()); } source.LastOffsetPoint = newLastOffset.ToString("O"); return(Task.FromResult((IEnumerable <Event>)events)); }
private async Task SetupMappingsAsync(DiagnosticsSource source) { foreach (var indexName in source.GetIndexNames()) { var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl); await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName); if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey())) { var jsonPath = string.Format("{0}{1}.json", _configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath), source.GetMappingName()); var client = new WebClient(); var mapping = client.DownloadString(jsonPath).Replace("___type_name___", source.ToTypeKey()); await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping); } } }
private async Task <DiagnosticsSource> TryScheduleSourceAsync(DiagnosticsSource source) { try { source = _sourceConfiguration.RefreshSource(source); TheTrace.TraceInformation("MasterScheduler - Scheduling {0}", source.ToTypeKey()); if (!source.IsActive.HasValue || !source.IsActive.Value) { TheTrace.TraceInformation("MasterScheduler - NOT active: {0}", source.ToTypeKey()); return(null); } var createMappings = _configurationValueProvider.GetValue(ConfigurationKeys.EsCreateMappings); if (Convert.ToBoolean(createMappings)) { await SetupMappingsAsync(source); } if (!source.LastScheduled.HasValue) { source.LastScheduled = DateTimeOffset.UtcNow.AddDays(-1); } // if has been recently scheduled if (source.LastScheduled.Value.AddMinutes(source.SchedulingFrequencyMinutes.Value) > DateTimeOffset.UtcNow) { TheTrace.TraceInformation("MasterScheduler - Nothing to do with {0}. LastScheduled in Future {1}", source.ToTypeKey(), source.LastScheduled.Value); return(null); } _telemetryProvider.WriteTelemetry( "MasterScheduler duration since last scheduled", (long)(DateTime.UtcNow - source.LastScheduled).Value.TotalMilliseconds, source.ToTypeKey()); var schedulerType = Assembly.GetExecutingAssembly().GetType(source.SchedulerType) ?? Type.GetType(source.SchedulerType); if (schedulerType == null) { source.ErrorMessage = "Could not find SchedulerType: " + source.SchedulerType; } else { await _scheduleDurationInstrumentor.InstrumentAsync(async() => { var scheduler = (ISourceScheduler)_locator.GetService(schedulerType); var result = await scheduler.TryScheduleAsync(source); source.LastScheduled = DateTimeOffset.UtcNow; TheTrace.TraceInformation( "MasterScheduler - Got result for TryScheduleAsync in {0}. Success => {1}", source.ToTypeKey(), result.Item1); if (result.Item2) { await _eventQueueOperator.PushBatchAsync(result.Item1); } source.ErrorMessage = string.Empty; TheTrace.TraceInformation("MasterScheduler - Finished Scheduling {0}", source.ToTypeKey()); }, source.ToTypeKey()); } return(source); } catch (Exception e) { TheTrace.TraceError(e.ToString()); source.ErrorMessage = e.ToString(); return(source); } }