public Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source) { var key = source.ToTypeKey(); if (source.IsActive.GetValueOrDefault(true)) { var consume = EventHubConsumer.Consumers.AddOrUpdate(key, new Lazy <EventHubConsumer>(() => { TheTrace.TraceInformation("Just added this eventHub consumer {0}", key); return(new EventHubConsumer(_pusher, source.ToSummary())); }), (kk, vv) => vv); // to make sure it gets accessed and created if new otherwise system is 'lazy' TheTrace.TraceInformation("This is the EventHub thing I was talking about: {0}", consume.Value.Source.TypeName); } else { Lazy <EventHubConsumer> consumer = null; if (EventHubConsumer.Consumers.TryRemove(key, out consumer)) { consumer.Value.Dispose(); TheTrace.TraceInformation("Just removed this eventHub consumer {0}", key); } } return(Task.FromResult(new Tuple <IEnumerable <Event>, bool>(new Event[0], false))); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { if (source.LastOffsetPoint == null) { source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-7).DropSecondAndMilliseconds().ToString("O"); } var offset = DateTimeOffset.Parse(source.LastOffsetPoint); var events = new List <Event>(); var totalMinutes = DateTimeOffset.UtcNow.Subtract(offset.AddMinutes(source.GracePeriodMinutes.Value)).TotalMinutes; var ofsted = DateTimeOffset.UtcNow; for (int i = 0; i < totalMinutes; i++) { ofsted = offset.AddMinutes(i + 1); var shardKey = GetShardKey(ofsted); events.Add(new Event(new ShardKeyArrived() { Source = source.ToSummary(), ShardKey = shardKey })); if (source.MaxItemsInAScheduleRun.HasValue && i >= source.MaxItemsInAScheduleRun) { break; } } source.LastOffsetPoint = ofsted.DropSecondAndMilliseconds().ToString("O"); return(Task.FromResult((IEnumerable <Event>)events)); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { if (source.LastOffsetPoint == null) { source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-1).DropSecondAndMilliseconds().ToString("O"); } var lastOffset = DateTimeOffset.Parse(source.LastOffsetPoint); var events = new List <Event>(); var graceMinutes = source.GracePeriodMinutes ?? 3; var now = DateTimeOffset.UtcNow.DropSecondAndMilliseconds(); var newLastOffset = lastOffset; int n = 1; // start from a minute after while (now >= lastOffset.Add(TimeSpan.FromMinutes(graceMinutes + n))) { newLastOffset = lastOffset.Add(TimeSpan.FromMinutes(n)) .DropSecondAndMilliseconds(); // just to be sure var shardKeys = GetShardKeys(newLastOffset); events.AddRange(shardKeys.Select(shardKey => new Event(new ShardKeyArrived { Source = source.ToSummary(), ShardKey = shardKey }))); if (source.MaxItemsInAScheduleRun.HasValue && n >= source.MaxItemsInAScheduleRun) { break; } n++; TheTrace.TraceInformation("Scheduling {0} for minute {1} and shardkey {2} => {0}_{1} AND {0}_{2}", source.ToTypeKey(), newLastOffset.ToString("yyyyMMddHHmm"), shardKeys.First()); } source.LastOffsetPoint = newLastOffset.ToString("O"); return(Task.FromResult((IEnumerable <Event>)events)); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling"); //var account = CloudStorageAccount.Parse(source.ConnectionString); CloudStorageAccount account; if (!String.IsNullOrWhiteSpace(source.AccountSasKey)) { // Create new storage credentials using the SAS token. var accountSas = new StorageCredentials(source.AccountSasKey); // Use these credentials and the account name to create a Blob service client. account = new CloudStorageAccount(accountSas, source.AccountName, "", useHttps: true); } else { account = CloudStorageAccount.Parse(source.ConnectionString); } var client = account.CreateCloudBlobClient(); var blobPath = source.GetProperty <string>("BlobPath"); TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", blobPath); blobPath = blobPath.TrimEnd('/') + "/"; // ensure path ends with / var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } DateTimeOffset maxOffset = offset.TimeOffset; FileOffset newOffset = null; var events = new List <Event>(); foreach (var blob in client.ListBlobs(blobPath).Where(itm => itm is CloudBlockBlob) .Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified)) { if (blob.Properties.LastModified > offset.TimeOffset) { var filename = blob.Uri.ToString(); newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, 0); TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri); events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = filename, Position = 0, EndPosition = blob.Properties.Length })); TheTrace.TraceInformation("Created BlobFileArrived for file: {0}", filename); } } source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }
public void CanConvertFromSourceToSummary_WithAlternateTypeName() { var entity = new DynamicTableEntity("pk", "rk"); entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2); entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi"); var source = new DiagnosticsSource(entity); var summary = source.ToSummary(); Assert.Equal(summary.TypeName, source.ToTypeKey()); }
protected async override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling"); var account = CloudStorageAccount.Parse(source.ConnectionString); var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty <string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var offset = DateTimeOffset.Parse(source.LastOffsetPoint); int instanceIndex = 0; DateTimeOffset maxOffset = offset; var events = new List <Event>(); while (true) { bool found = false; var path = string.Format(pathFormat, instanceIndex); TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path); foreach (var itm in client.ListBlobs(path)) { var blob = itm as CloudBlockBlob; if (blob != null && blob.Properties.LastModified > offset) { TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri); found = true; maxOffset = offset > blob.Properties.LastModified.Value ? offset : blob.Properties.LastModified.Value; events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = blob.Uri.ToString() })); } } if (!found) { TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex); break; } instanceIndex++; } source.LastOffsetPoint = maxOffset.ToString("O"); return(events); }
protected async override Task<IEnumerable<Event>> DoSchedule(DiagnosticsSource source) { TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling"); var account = CloudStorageAccount.Parse(source.ConnectionString); var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty<string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var offset = DateTimeOffset.Parse(source.LastOffsetPoint); int instanceIndex = 0; DateTimeOffset maxOffset = offset; var events = new List<Event>(); while (true) { bool found = false; var path = string.Format(pathFormat, instanceIndex); TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path); foreach (var itm in client.ListBlobs(path)) { var blob = itm as CloudBlockBlob; if (blob != null && blob.Properties.LastModified > offset) { TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri); found = true; maxOffset = offset > blob.Properties.LastModified.Value ? offset : blob.Properties.LastModified.Value; events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = blob.Uri.ToString() })); } } if (!found) { TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex); break; } instanceIndex++; } source.LastOffsetPoint = maxOffset.ToString("O"); return events; }
public void CanConvertFromSourceToSummary() { var entity = new DynamicTableEntity("pk", "rk"); entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2); entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man"); entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow); entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true); var source = new DiagnosticsSource(entity); var summary = source.ToSummary(); Assert.Equal(summary.ConnectionString, source.ConnectionString); Assert.Equal(summary.PartitionKey, source.PartitionKey); Assert.Equal(summary.RowKey, source.RowKey); Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty<DateTime>("dpd")); Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty<int>("dpi")); Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty<string>("dps")); Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty<bool>("dpb")); Assert.Equal(summary.TypeName, source.ToTypeKey()); }
public void CanConvertFromSourceToSummary() { var entity = new DynamicTableEntity("pk", "rk"); entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2); entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man"); entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow); entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true); var source = new DiagnosticsSource(entity); var summary = source.ToSummary(); Assert.Equal(summary.ConnectionString, source.ConnectionString); Assert.Equal(summary.PartitionKey, source.PartitionKey); Assert.Equal(summary.RowKey, source.RowKey); Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty <DateTime>("dpd")); Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty <int>("dpi")); Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty <string>("dps")); Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty <bool>("dpb")); Assert.Equal(summary.TypeName, source.ToTypeKey()); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling"); CloudStorageAccount account; if (!String.IsNullOrWhiteSpace(source.AccountSasKey)) { // Create new storage credentials using the SAS token. var accountSas = new StorageCredentials(source.AccountSasKey); // Use these credentials and the account name to create a Blob service client. account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true); } else { account = CloudStorageAccount.Parse(source.ConnectionString); } var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty <string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } int instanceIndex = 0; DateTimeOffset maxOffset = offset.TimeOffset; FileOffset newOffset = null; var events = new List <Event>(); while (true) { bool found = false; var path = string.Format(pathFormat, instanceIndex); var isSingleInstance = path == pathFormat; TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path); foreach (var blob in client.ListBlobs(path).Where(itm => itm is CloudBlockBlob) .Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified)) { if (blob.Properties.LastModified > offset.TimeOffset) { var filename = blob.Uri.ToString(); if (!found) // first time running { newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, blob.Properties.Length); } TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri); found = true; events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = filename, Position = (filename == offset.FileName) ? offset.Position : 0, // if same file then pass offset EndPosition = blob.Properties.Length })); } } if (!found || isSingleInstance) { TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex); break; } instanceIndex++; } source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH"; TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling"); var account = CloudStorageAccount.Parse(source.ConnectionString); var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty <string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ?? DefaultIisLogFileFormatConvention; var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } int instanceIndex = 0; var nextOffset = DateTimeOffset.UtcNow; var events = new List <Event>(); var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset); if (fullNumberOfHoursInBetween == 0) { return(Task.FromResult((IEnumerable <Event>)events)); } while (true) { var path = string.Format(pathFormat, instanceIndex); instanceIndex++; TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path); var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob); if (!any) { break; } for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++) { var fileOffset = offset.TimeOffset.AddHours(i); var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; events.Add(new Event(new BlobFileScheduled() { FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume, PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile, NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile, Source = source.ToSummary(), StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80)) })); TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume); } } source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH"; TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling"); //var account = CloudStorageAccount.Parse(source.ConnectionString); CloudStorageAccount account; if (!String.IsNullOrWhiteSpace(source.AccountSasKey)) { // Create new storage credentials using the SAS token. var accountSas = new StorageCredentials(source.AccountSasKey); // Use these credentials and the account name to create a Blob service client. account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true); } else { account = CloudStorageAccount.Parse(source.ConnectionString); } var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty <string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ?? DefaultIisLogFileFormatConvention; var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } int instanceIndex = 0; var nextOffset = DateTimeOffset.UtcNow; var events = new List <Event>(); var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset); if (fullNumberOfHoursInBetween == 0) { return(Task.FromResult((IEnumerable <Event>)events)); } while (true) { var path = string.Format(pathFormat, instanceIndex); var isSingleInstance = path == pathFormat; instanceIndex++; TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path); var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob); if (!any) { break; } for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++) { var fileOffset = offset.TimeOffset.AddHours(i); var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; events.Add(new Event(new BlobFileScheduled() { FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume, PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile, NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile, Source = source.ToSummary(), StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80)), IsRepeat = true })); TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume); } if (isSingleInstance) // this is for when you want to consume IIS logs from only a single VM and not used {0} in blbb format { break; } } source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }