private async Task SetupMappingsAsync(DiagnosticsSource source)
{
foreach (var indexName in source.GetIndexNames())
{
var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl);
await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName);
if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey()))
{
var jsonPath = string.Format("{0}{1}.json",
_configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath),
source.GetMappingName());
var response = await _nonAuthenticatingClient.GetAsync(jsonPath);
if (response.Content == null)
{
throw new ApplicationException(response.ToString());
}
var content = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new InvalidOperationException(content);
}
var mapping = content.Replace("___type_name___", source.ToTypeKey());
await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping);
}
}
}
public async Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
// if Stop offset has been reached
if (!string.IsNullOrEmpty(source.StopOffsetPoint) && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0)
{
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
var lockToken = new LockToken(source.ToTypeKey());
int seconds =
Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds));
if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000)))
{
TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey());
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
try
{
var events = await DoSchedule(source);
return(new Tuple <IEnumerable <Event>, bool>(events, true));
}
finally
{
Task.Run(() => _lockStore.ReleaseLockAsync(lockToken)).Wait();
}
}
public Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
var key = source.ToTypeKey();
if (source.IsActive.GetValueOrDefault(true))
{
var consume = EventHubConsumer.Consumers.AddOrUpdate(key,
new Lazy <EventHubConsumer>(() =>
{
TheTrace.TraceInformation("Just added this eventHub consumer {0}", key);
return(new EventHubConsumer(_pusher, source.ToSummary()));
}),
(kk, vv) => vv);
// to make sure it gets accessed and created if new otherwise system is 'lazy'
TheTrace.TraceInformation("This is the EventHub thing I was talking about: {0}", consume.Value.Source.TypeName);
}
else
{
Lazy <EventHubConsumer> consumer = null;
if (EventHubConsumer.Consumers.TryRemove(key, out consumer))
{
consumer.Value.Dispose();
TheTrace.TraceInformation("Just removed this eventHub consumer {0}", key);
}
}
return(Task.FromResult(new Tuple <IEnumerable <Event>, bool>(new Event[0], false)));
}
private async Task SetupMappingsAsync(DiagnosticsSource source)
{
foreach (var indexName in source.GetIndexNames())
{
var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl);
await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName);
if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey()))
{
var jsonPath = string.Format("{0}{1}.json",
_configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath),
source.GetMappingName());
var client = new WebClient();
var mapping = client.DownloadString(jsonPath).Replace("___type_name___", source.ToTypeKey());
await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping);
}
}
}
public void CanConvertFromSourceToSummary_WithAlternateTypeName()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi");
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
public void CanConvertFromSourceToSummary_WithAlternateTypeName()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi");
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
public async Task<Tuple<IEnumerable<Event>, bool>> TryScheduleAsync(DiagnosticsSource source)
{
// if Stop offset has been reached
if (source.StopOffsetPoint != null && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0)
return new Tuple<IEnumerable<Event>, bool>(Enumerable.Empty<Event>(), false);
var lockToken = new LockToken(source.ToTypeKey());
int seconds =
Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds));
if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000)))
{
TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey());
return new Tuple<IEnumerable<Event>, bool>(Enumerable.Empty<Event>(), false);
}
try
{
var events = await DoSchedule(source);
return new Tuple<IEnumerable<Event>, bool>(events, true);
}
finally
{
Task.Run( () => _lockStore.ReleaseLockAsync(lockToken)).Wait();
}
}
public void CanConvertFromSourceToSummary()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man");
entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow);
entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true);
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.ConnectionString, source.ConnectionString);
Assert.Equal(summary.PartitionKey, source.PartitionKey);
Assert.Equal(summary.RowKey, source.RowKey);
Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty<DateTime>("dpd"));
Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty<int>("dpi"));
Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty<string>("dps"));
Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty<bool>("dpb"));
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
public void CanConvertFromSourceToSummary()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man");
entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow);
entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true);
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.ConnectionString, source.ConnectionString);
Assert.Equal(summary.PartitionKey, source.PartitionKey);
Assert.Equal(summary.RowKey, source.RowKey);
Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty <DateTime>("dpd"));
Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty <int>("dpi"));
Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty <string>("dps"));
Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty <bool>("dpb"));
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
if (source.LastOffsetPoint == null)
{
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-1).DropSecondAndMilliseconds().ToString("O");
}
var lastOffset = DateTimeOffset.Parse(source.LastOffsetPoint);
var events = new List <Event>();
var graceMinutes = source.GracePeriodMinutes ?? 3;
var now = DateTimeOffset.UtcNow.DropSecondAndMilliseconds();
var newLastOffset = lastOffset;
int n = 1; // start from a minute after
while (now >= lastOffset.Add(TimeSpan.FromMinutes(graceMinutes + n)))
{
newLastOffset = lastOffset.Add(TimeSpan.FromMinutes(n))
.DropSecondAndMilliseconds(); // just to be sure
var shardKeys = GetShardKeys(newLastOffset);
events.AddRange(shardKeys.Select(shardKey => new Event(new ShardKeyArrived {
Source = source.ToSummary(), ShardKey = shardKey
})));
if (source.MaxItemsInAScheduleRun.HasValue && n >= source.MaxItemsInAScheduleRun)
{
break;
}
n++;
TheTrace.TraceInformation("Scheduling {0} for minute {1} and shardkey {2} => {0}_{1} AND {0}_{2}", source.ToTypeKey(), newLastOffset.ToString("yyyyMMddHHmm"), shardKeys.First());
}
source.LastOffsetPoint = newLastOffset.ToString("O");
return(Task.FromResult((IEnumerable <Event>)events));
}
private async Task SetupMappingsAsync(DiagnosticsSource source)
{
foreach (var indexName in source.GetIndexNames())
{
var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl);
await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName);
if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey()))
{
var jsonPath = string.Format("{0}{1}.json",
_configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath),
source.GetMappingName());
var client = new WebClient();
var mapping = client.DownloadString(jsonPath).Replace("___type_name___", source.ToTypeKey());
await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping);
}
}
}
private async Task <DiagnosticsSource> TryScheduleSourceAsync(DiagnosticsSource source)
{
try
{
source = _sourceConfiguration.RefreshSource(source);
TheTrace.TraceInformation("MasterScheduler - Scheduling {0}", source.ToTypeKey());
if (!source.IsActive.HasValue || !source.IsActive.Value)
{
TheTrace.TraceInformation("MasterScheduler - NOT active: {0}", source.ToTypeKey());
return(null);
}
var createMappings = _configurationValueProvider.GetValue(ConfigurationKeys.EsCreateMappings);
if (Convert.ToBoolean(createMappings))
{
await SetupMappingsAsync(source);
}
if (!source.LastScheduled.HasValue)
{
source.LastScheduled = DateTimeOffset.UtcNow.AddDays(-1);
}
// if has been recently scheduled
if (source.LastScheduled.Value.AddMinutes(source.SchedulingFrequencyMinutes.Value) >
DateTimeOffset.UtcNow)
{
TheTrace.TraceInformation("MasterScheduler - Nothing to do with {0}. LastScheduled in Future {1}",
source.ToTypeKey(), source.LastScheduled.Value);
return(null);
}
_telemetryProvider.WriteTelemetry(
"MasterScheduler duration since last scheduled",
(long)(DateTime.UtcNow - source.LastScheduled).Value.TotalMilliseconds,
source.ToTypeKey());
var schedulerType = Assembly.GetExecutingAssembly().GetType(source.SchedulerType) ??
Type.GetType(source.SchedulerType);
if (schedulerType == null)
{
source.ErrorMessage = "Could not find SchedulerType: " + source.SchedulerType;
}
else
{
await _scheduleDurationInstrumentor.InstrumentAsync(async() =>
{
var scheduler = (ISourceScheduler)_locator.GetService(schedulerType);
var result = await scheduler.TryScheduleAsync(source);
source.LastScheduled = DateTimeOffset.UtcNow;
TheTrace.TraceInformation(
"MasterScheduler - Got result for TryScheduleAsync in {0}. Success => {1}",
source.ToTypeKey(), result.Item1);
if (result.Item2)
{
await _eventQueueOperator.PushBatchAsync(result.Item1);
}
source.ErrorMessage = string.Empty;
TheTrace.TraceInformation("MasterScheduler - Finished Scheduling {0}", source.ToTypeKey());
}, source.ToTypeKey());
}
return(source);
}
catch (Exception e)
{
TheTrace.TraceError(e.ToString());
source.ErrorMessage = e.ToString();
return(source);
}
}