public IEnumerable <string> GetIndexNames(DiagnosticsSource source, int daysToGoBack = 7)
{
// static index name
if (!string.IsNullOrEmpty(source.IndexName))
{
return new[] { source.IndexName }
}
;
if (String.IsNullOrEmpty(source.LastOffsetPoint))
{
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-daysToGoBack).ToString("O");
}
var dateTimeOffset = FileOffset.Parse(source.LastOffsetPoint);
var days = (int)(DateTimeOffset.UtcNow.AddDays(1) - dateTimeOffset.TimeOffset).TotalDays + 1; // to cover today as well - Aboo was here
if (days <= 0)
{
return(Enumerable.Empty <string>());
}
return(Enumerable.Range(0, days).Select(x => DateTimeOffset.UtcNow.AddDays(1).AddDays(-x))
.Select(z => _indexNamer.BuildName(dateTimeOffset.TimeOffset, source.GetMappingName().ToLowerInvariant())));
}
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
//var account = CloudStorageAccount.Parse(source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(source.GetProperty <string>("TableName"));
var entities = table.ExecuteQuery(new TableQuery().Where(
TableQuery.GenerateFilterCondition("PartitionKey", "gt", source.LastOffsetPoint)));
foreach (var entity in entities)
{
}
throw new NotImplementedException();
}
public async Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
// if Stop offset has been reached
if (!string.IsNullOrEmpty(source.StopOffsetPoint) && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0)
{
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
var lockToken = new LockToken(source.ToTypeKey());
int seconds =
Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds));
if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000)))
{
TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey());
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
try
{
var events = await DoSchedule(source);
return(new Tuple <IEnumerable <Event>, bool>(events, true));
}
finally
{
Task.Run(() => _lockStore.ReleaseLockAsync(lockToken)).Wait();
}
}
private async Task SetupMappingsAsync(DiagnosticsSource source)
{
foreach (var indexName in source.GetIndexNames())
{
var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl);
await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName);
if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey()))
{
var jsonPath = string.Format("{0}{1}.json",
_configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath),
source.GetMappingName());
var response = await _nonAuthenticatingClient.GetAsync(jsonPath);
if (response.Content == null)
{
throw new ApplicationException(response.ToString());
}
var content = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new InvalidOperationException(content);
}
var mapping = content.Replace("___type_name___", source.ToTypeKey());
await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping);
}
}
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
if (source.LastOffsetPoint == null)
{
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-1).DropSecondAndMilliseconds().ToString("O");
}
var lastOffset = DateTimeOffset.Parse(source.LastOffsetPoint);
var events = new List <Event>();
var graceMinutes = source.GracePeriodMinutes ?? 3;
var now = DateTimeOffset.UtcNow.DropSecondAndMilliseconds();
var newLastOffset = lastOffset;
int n = 1; // start from a minute after
while (now >= lastOffset.Add(TimeSpan.FromMinutes(graceMinutes + n)))
{
newLastOffset = lastOffset.Add(TimeSpan.FromMinutes(n))
.DropSecondAndMilliseconds(); // just to be sure
var shardKeys = GetShardKeys(newLastOffset);
events.AddRange(shardKeys.Select(shardKey => new Event(new ShardKeyArrived {
Source = source.ToSummary(), ShardKey = shardKey
})));
if (source.MaxItemsInAScheduleRun.HasValue && n >= source.MaxItemsInAScheduleRun)
{
break;
}
n++;
TheTrace.TraceInformation("Scheduling {0} for minute {1} and shardkey {2} => {0}_{1} AND {0}_{2}", source.ToTypeKey(), newLastOffset.ToString("yyyyMMddHHmm"), shardKeys.First());
}
source.LastOffsetPoint = newLastOffset.ToString("O");
return(Task.FromResult((IEnumerable <Event>)events));
}
public Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
var key = source.ToTypeKey();
if (source.IsActive.GetValueOrDefault(true))
{
var consume = EventHubConsumer.Consumers.AddOrUpdate(key,
new Lazy <EventHubConsumer>(() =>
{
TheTrace.TraceInformation("Just added this eventHub consumer {0}", key);
return(new EventHubConsumer(_pusher, source.ToSummary()));
}),
(kk, vv) => vv);
// to make sure it gets accessed and created if new otherwise system is 'lazy'
TheTrace.TraceInformation("This is the EventHub thing I was talking about: {0}", consume.Value.Source.TypeName);
}
else
{
Lazy <EventHubConsumer> consumer = null;
if (EventHubConsumer.Consumers.TryRemove(key, out consumer))
{
consumer.Value.Dispose();
TheTrace.TraceInformation("Just removed this eventHub consumer {0}", key);
}
}
return(Task.FromResult(new Tuple <IEnumerable <Event>, bool>(new Event[0], false)));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
if (source.LastOffsetPoint == null)
{
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-7).DropSecondAndMilliseconds().ToString("O");
}
var offset = DateTimeOffset.Parse(source.LastOffsetPoint);
var events = new List <Event>();
var totalMinutes = DateTimeOffset.UtcNow.Subtract(offset.AddMinutes(source.GracePeriodMinutes.Value)).TotalMinutes;
var ofsted = DateTimeOffset.UtcNow;
for (int i = 0; i < totalMinutes; i++)
{
ofsted = offset.AddMinutes(i + 1);
var shardKey = GetShardKey(ofsted);
events.Add(new Event(new ShardKeyArrived()
{
Source = source.ToSummary(), ShardKey = shardKey
}));
if (source.MaxItemsInAScheduleRun.HasValue && i >= source.MaxItemsInAScheduleRun)
{
break;
}
}
source.LastOffsetPoint = ofsted.DropSecondAndMilliseconds().ToString("O");
return(Task.FromResult((IEnumerable <Event>)events));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
//var account = CloudStorageAccount.Parse(source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var blobPath = source.GetProperty <string>("BlobPath");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", blobPath);
blobPath = blobPath.TrimEnd('/') + "/"; // ensure path ends with /
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
DateTimeOffset maxOffset = offset.TimeOffset;
FileOffset newOffset = null;
var events = new List <Event>();
foreach (var blob in client.ListBlobs(blobPath).Where(itm => itm is CloudBlockBlob)
.Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
{
if (blob.Properties.LastModified > offset.TimeOffset)
{
var filename = blob.Uri.ToString();
newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, 0);
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
events.Add(new Event(new BlobFileArrived()
{
Source = source.ToSummary(),
BlobId = filename,
Position = 0,
EndPosition = blob.Properties.Length
}));
TheTrace.TraceInformation("Created BlobFileArrived for file: {0}", filename);
}
}
source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
public void CanConvertFromSourceToSummary_WithAlternateTypeName()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi");
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
public void CanConvertFromSourceToSummary_WithAlternateTypeName()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi");
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
public async Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
{
// if Stop offset has been reached
if (!string.IsNullOrEmpty(source.StopOffsetPoint) && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0)
{
return(new Tuple <IEnumerable <Event>, bool>(Enumerable.Empty <Event>(), false));
}
var events = await DoSchedule(source);
return(new Tuple <IEnumerable <Event>, bool>(events, true));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
var account = CloudStorageAccount.Parse(source.ConnectionString);
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(source.GetProperty <string>("TableName"));
var entities = table.ExecuteQuery(new TableQuery().Where(
TableQuery.GenerateFilterCondition("PartitionKey", "gt", source.LastOffsetPoint)));
foreach (var entity in entities)
{
}
throw new NotImplementedException();
}
protected async override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
var account = CloudStorageAccount.Parse(source.ConnectionString);
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty <string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var offset = DateTimeOffset.Parse(source.LastOffsetPoint);
int instanceIndex = 0;
DateTimeOffset maxOffset = offset;
var events = new List <Event>();
while (true)
{
bool found = false;
var path = string.Format(pathFormat, instanceIndex);
TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path);
foreach (var itm in client.ListBlobs(path))
{
var blob = itm as CloudBlockBlob;
if (blob != null && blob.Properties.LastModified > offset)
{
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
found = true;
maxOffset = offset > blob.Properties.LastModified.Value
? offset
: blob.Properties.LastModified.Value;
events.Add(new Event(new BlobFileArrived()
{
Source = source.ToSummary(), BlobId = blob.Uri.ToString()
}));
}
}
if (!found)
{
TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex);
break;
}
instanceIndex++;
}
source.LastOffsetPoint = maxOffset.ToString("O");
return(events);
}
protected override Task<IEnumerable<Event>> DoSchedule(DiagnosticsSource source)
{
var account = CloudStorageAccount.Parse(source.ConnectionString);
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(source.GetProperty<string>("TableName"));
var entities = table.ExecuteQuery(new TableQuery().Where(
TableQuery.GenerateFilterCondition("PartitionKey", "gt", source.LastOffsetPoint)));
foreach (var entity in entities)
{
}
throw new NotImplementedException();
}
private async Task SetupMappingsAsync(DiagnosticsSource source)
{
foreach (var indexName in source.GetIndexNames())
{
var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl);
await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName);
if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey()))
{
var jsonPath = string.Format("{0}{1}.json",
_configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath),
source.GetMappingName());
var client = new WebClient();
var mapping = client.DownloadString(jsonPath).Replace("___type_name___", source.ToTypeKey());
await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping);
}
}
}
public void CanConvertFromSourceToSummary()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man");
entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow);
entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true);
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.ConnectionString, source.ConnectionString);
Assert.Equal(summary.PartitionKey, source.PartitionKey);
Assert.Equal(summary.RowKey, source.RowKey);
Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty<DateTime>("dpd"));
Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty<int>("dpi"));
Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty<string>("dps"));
Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty<bool>("dpb"));
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
protected async override Task<IEnumerable<Event>> DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
var account = CloudStorageAccount.Parse(source.ConnectionString);
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty<string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var offset = DateTimeOffset.Parse(source.LastOffsetPoint);
int instanceIndex = 0;
DateTimeOffset maxOffset = offset;
var events = new List<Event>();
while (true)
{
bool found = false;
var path = string.Format(pathFormat, instanceIndex);
TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path);
foreach (var itm in client.ListBlobs(path))
{
var blob = itm as CloudBlockBlob;
if (blob != null && blob.Properties.LastModified > offset)
{
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
found = true;
maxOffset = offset > blob.Properties.LastModified.Value
? offset
: blob.Properties.LastModified.Value;
events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = blob.Uri.ToString() }));
}
}
if (!found)
{
TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex);
break;
}
instanceIndex++;
}
source.LastOffsetPoint = maxOffset.ToString("O");
return events;
}
public void BetweenAnHourWithGraceOf3ThereIs57()
{
var lockStore = new Mock <ILockStore>();
lockStore.Setup(
x => x.TryLockAsync(It.IsAny <LockToken>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <int>())).Returns(Task.FromResult(true));
var config = new Mock <IConfigurationValueProvider>();
var scheduler = new MinuteTableShardScheduler(config.Object);
var entity = new DynamicTableEntity("dd", "fff");
entity.Properties.Add("GracePeriodMinutes", EntityProperty.GeneratePropertyForInt(3));
entity.Properties.Add("IsActive", EntityProperty.GeneratePropertyForBool(true));
var source = new DiagnosticsSource(entity);
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddHours(-1).DropSecondAndMilliseconds().ToString("O");
source.LastScheduled = DateTimeOffset.UtcNow.AddDays(-1);
var result = scheduler.TryScheduleAsync(source).Result;
Assert.Equal(57, result.Item1.Count());
}
public void CanConvertFromSourceToSummary()
{
var entity = new DynamicTableEntity("pk", "rk");
entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man");
entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow);
entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true);
var source = new DiagnosticsSource(entity);
var summary = source.ToSummary();
Assert.Equal(summary.ConnectionString, source.ConnectionString);
Assert.Equal(summary.PartitionKey, source.PartitionKey);
Assert.Equal(summary.RowKey, source.RowKey);
Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty <DateTime>("dpd"));
Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty <int>("dpi"));
Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty <string>("dps"));
Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty <bool>("dpb"));
Assert.Equal(summary.TypeName, source.ToTypeKey());
}
public async Task<Tuple<IEnumerable<Event>, bool>> TryScheduleAsync(DiagnosticsSource source)
{
// if Stop offset has been reached
if (source.StopOffsetPoint != null && source.LastOffsetPoint != null && source.LastOffsetPoint.CompareTo(source.StopOffsetPoint) >= 0)
return new Tuple<IEnumerable<Event>, bool>(Enumerable.Empty<Event>(), false);
var lockToken = new LockToken(source.ToTypeKey());
int seconds =
Convert.ToInt32(_configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockDurationSeconds));
if (!(await _lockStore.TryLockAsync(lockToken, 2, 1000, seconds * 1000)))
{
TheTrace.TraceInformation("I could NOT be master for {0}", source.ToTypeKey());
return new Tuple<IEnumerable<Event>, bool>(Enumerable.Empty<Event>(), false);
}
try
{
var events = await DoSchedule(source);
return new Tuple<IEnumerable<Event>, bool>(events, true);
}
finally
{
Task.Run( () => _lockStore.ReleaseLockAsync(lockToken)).Wait();
}
}
protected abstract Task<IEnumerable<Event>> DoSchedule(DiagnosticsSource source);
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty <string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
int instanceIndex = 0;
DateTimeOffset maxOffset = offset.TimeOffset;
FileOffset newOffset = null;
var events = new List <Event>();
while (true)
{
bool found = false;
var path = string.Format(pathFormat, instanceIndex);
var isSingleInstance = path == pathFormat;
TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path);
foreach (var blob in client.ListBlobs(path).Where(itm => itm is CloudBlockBlob)
.Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
{
if (blob.Properties.LastModified > offset.TimeOffset)
{
var filename = blob.Uri.ToString();
if (!found) // first time running
{
newOffset = new FileOffset(filename,
blob.Properties.LastModified ?? DateTimeOffset.UtcNow, blob.Properties.Length);
}
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
found = true;
events.Add(new Event(new BlobFileArrived()
{
Source = source.ToSummary(),
BlobId = filename,
Position = (filename == offset.FileName) ? offset.Position : 0, // if same file then pass offset
EndPosition = blob.Properties.Length
}));
}
}
if (!found || isSingleInstance)
{
TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex);
break;
}
instanceIndex++;
}
source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH";
TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling");
var account = CloudStorageAccount.Parse(source.ConnectionString);
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty <string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ?? DefaultIisLogFileFormatConvention;
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
int instanceIndex = 0;
var nextOffset = DateTimeOffset.UtcNow;
var events = new List <Event>();
var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset);
if (fullNumberOfHoursInBetween == 0)
{
return(Task.FromResult((IEnumerable <Event>)events));
}
while (true)
{
var path = string.Format(pathFormat, instanceIndex);
instanceIndex++;
TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path);
var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob);
if (!any)
{
break;
}
for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++)
{
var fileOffset = offset.TimeOffset.AddHours(i);
var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
events.Add(new Event(new BlobFileScheduled()
{
FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume,
PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile,
NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile,
Source = source.ToSummary(),
StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80))
}));
TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume);
}
}
source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
public void UpdateSource(DiagnosticsSource source)
{
_table.Execute(TableOperation.InsertOrReplace(source.ToEntity()));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH";
TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling");
//var account = CloudStorageAccount.Parse(source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty <string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ??
DefaultIisLogFileFormatConvention;
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
int instanceIndex = 0;
var nextOffset = DateTimeOffset.UtcNow;
var events = new List <Event>();
var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset);
if (fullNumberOfHoursInBetween == 0)
{
return(Task.FromResult((IEnumerable <Event>)events));
}
while (true)
{
var path = string.Format(pathFormat, instanceIndex);
var isSingleInstance = path == pathFormat;
instanceIndex++;
TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path);
var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob);
if (!any)
{
break;
}
for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++)
{
var fileOffset = offset.TimeOffset.AddHours(i);
var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
events.Add(new Event(new BlobFileScheduled()
{
FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume,
PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile,
NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile,
Source = source.ToSummary(),
StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80)),
IsRepeat = true
}));
TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume);
}
if (isSingleInstance) // this is for when you want to consume IIS logs from only a single VM and not used {0} in blbb format
{
break;
}
}
source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
protected abstract Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source);
private async Task <DiagnosticsSource> TryScheduleSourceAsync(DiagnosticsSource source)
{
try
{
source = _sourceConfiguration.RefreshSource(source);
TheTrace.TraceInformation("MasterScheduler - Scheduling {0}", source.ToTypeKey());
if (!source.IsActive.HasValue || !source.IsActive.Value)
{
TheTrace.TraceInformation("MasterScheduler - NOT active: {0}", source.ToTypeKey());
return(null);
}
var createMappings = _configurationValueProvider.GetValue(ConfigurationKeys.EsCreateMappings);
if (Convert.ToBoolean(createMappings))
{
await SetupMappingsAsync(source);
}
if (!source.LastScheduled.HasValue)
{
source.LastScheduled = DateTimeOffset.UtcNow.AddDays(-1);
}
// if has been recently scheduled
if (source.LastScheduled.Value.AddMinutes(source.SchedulingFrequencyMinutes.Value) >
DateTimeOffset.UtcNow)
{
TheTrace.TraceInformation("MasterScheduler - Nothing to do with {0}. LastScheduled in Future {1}",
source.ToTypeKey(), source.LastScheduled.Value);
return(null);
}
_telemetryProvider.WriteTelemetry(
"MasterScheduler duration since last scheduled",
(long)(DateTime.UtcNow - source.LastScheduled).Value.TotalMilliseconds,
source.ToTypeKey());
var schedulerType = Assembly.GetExecutingAssembly().GetType(source.SchedulerType) ??
Type.GetType(source.SchedulerType);
if (schedulerType == null)
{
source.ErrorMessage = "Could not find SchedulerType: " + source.SchedulerType;
}
else
{
await _scheduleDurationInstrumentor.InstrumentAsync(async() =>
{
var scheduler = (ISourceScheduler)_locator.GetService(schedulerType);
var result = await scheduler.TryScheduleAsync(source);
source.LastScheduled = DateTimeOffset.UtcNow;
TheTrace.TraceInformation(
"MasterScheduler - Got result for TryScheduleAsync in {0}. Success => {1}",
source.ToTypeKey(), result.Item1);
if (result.Item2)
{
await _eventQueueOperator.PushBatchAsync(result.Item1);
}
source.ErrorMessage = string.Empty;
TheTrace.TraceInformation("MasterScheduler - Finished Scheduling {0}", source.ToTypeKey());
}, source.ToTypeKey());
}
return(source);
}
catch (Exception e)
{
TheTrace.TraceError(e.ToString());
source.ErrorMessage = e.ToString();
return(source);
}
}
private async Task SetupMappingsAsync(DiagnosticsSource source)
{
foreach (var indexName in source.GetIndexNames())
{
var esUrl = _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl);
await _elasticsearchClient.CreateIndexIfNotExistsAsync(esUrl, indexName);
if (!await _elasticsearchClient.MappingExistsAsync(esUrl, indexName, source.ToTypeKey()))
{
var jsonPath = string.Format("{0}{1}.json",
_configurationValueProvider.GetValue(ConfigurationKeys.MappingsPath),
source.GetMappingName());
var client = new WebClient();
var mapping = client.DownloadString(jsonPath).Replace("___type_name___", source.ToTypeKey());
await _elasticsearchClient.UpdateMappingAsync(esUrl, indexName, source.ToTypeKey(), mapping);
}
}
}
