public IEnumerable <string> GetIndexNames(DiagnosticsSource source, int daysToGoBack = 7) { // static index name if (!string.IsNullOrEmpty(source.IndexName)) { return new[] { source.IndexName } } ; if (String.IsNullOrEmpty(source.LastOffsetPoint)) { source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-daysToGoBack).ToString("O"); } var dateTimeOffset = FileOffset.Parse(source.LastOffsetPoint); var days = (int)(DateTimeOffset.UtcNow.AddDays(1) - dateTimeOffset.TimeOffset).TotalDays + 1; // to cover today as well - Aboo was here if (days <= 0) { return(Enumerable.Empty <string>()); } return(Enumerable.Range(0, days).Select(x => DateTimeOffset.UtcNow.AddDays(1).AddDays(-x)) .Select(z => _indexNamer.BuildName(dateTimeOffset.TimeOffset, source.GetMappingName().ToLowerInvariant()))); } }
public IEnumerable <string> GetIndexNames(int daysToGoBack = 7) { if (!string.IsNullOrEmpty(IndexName)) { return new [] { IndexName } } ; if (String.IsNullOrEmpty(LastOffsetPoint)) { LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-daysToGoBack).ToString("O"); } var dateTimeOffset = FileOffset.Parse(LastOffsetPoint); var days = (int)(DateTimeOffset.UtcNow.AddDays(1) - dateTimeOffset.TimeOffset).TotalDays + 1; // to cover today as well - Aboo was here if (days <= 0) { return(Enumerable.Empty <string>()); } return(Enumerable.Range(0, days).Select(x => DateTimeOffset.UtcNow.AddDays(1).AddDays(-x)) .Select(y => y.ToString("yyyyMMdd"))); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling"); //var account = CloudStorageAccount.Parse(source.ConnectionString); CloudStorageAccount account; if (!String.IsNullOrWhiteSpace(source.AccountSasKey)) { // Create new storage credentials using the SAS token. var accountSas = new StorageCredentials(source.AccountSasKey); // Use these credentials and the account name to create a Blob service client. account = new CloudStorageAccount(accountSas, source.AccountName, "", useHttps: true); } else { account = CloudStorageAccount.Parse(source.ConnectionString); } var client = account.CreateCloudBlobClient(); var blobPath = source.GetProperty <string>("BlobPath"); TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", blobPath); blobPath = blobPath.TrimEnd('/') + "/"; // ensure path ends with / var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } DateTimeOffset maxOffset = offset.TimeOffset; FileOffset newOffset = null; var events = new List <Event>(); foreach (var blob in client.ListBlobs(blobPath).Where(itm => itm is CloudBlockBlob) .Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified)) { if (blob.Properties.LastModified > offset.TimeOffset) { var filename = blob.Uri.ToString(); newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, 0); TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri); events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = filename, Position = 0, EndPosition = blob.Properties.Length })); TheTrace.TraceInformation("Created BlobFileArrived for file: {0}", filename); } } source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }
public DiagnosticsSourceSummary ToSummary() { var dss = new DiagnosticsSourceSummary() { ConnectionString = ConnectionString, AccountSasKey = AccountSasKey, AccountName = AccountName, IndexName = IndexName, PartitionKey = PartitionKey, RowKey = RowKey, TypeName = ToTypeKey(), LastTimeOffset = FileOffset.Parse(LastOffsetPoint)?.TimeOffset, DynamicProperties = new Dictionary <string, object>() }; foreach (var kv in _entity.Properties) { dss.DynamicProperties.Add(kv.Key, kv.Value.PropertyAsObject); } return(dss); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling"); CloudStorageAccount account; if (!String.IsNullOrWhiteSpace(source.AccountSasKey)) { // Create new storage credentials using the SAS token. var accountSas = new StorageCredentials(source.AccountSasKey); // Use these credentials and the account name to create a Blob service client. account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true); } else { account = CloudStorageAccount.Parse(source.ConnectionString); } var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty <string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } int instanceIndex = 0; DateTimeOffset maxOffset = offset.TimeOffset; FileOffset newOffset = null; var events = new List <Event>(); while (true) { bool found = false; var path = string.Format(pathFormat, instanceIndex); var isSingleInstance = path == pathFormat; TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path); foreach (var blob in client.ListBlobs(path).Where(itm => itm is CloudBlockBlob) .Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified)) { if (blob.Properties.LastModified > offset.TimeOffset) { var filename = blob.Uri.ToString(); if (!found) // first time running { newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, blob.Properties.Length); } TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri); found = true; events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = filename, Position = (filename == offset.FileName) ? offset.Position : 0, // if same file then pass offset EndPosition = blob.Properties.Length })); } } if (!found || isSingleInstance) { TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex); break; } instanceIndex++; } source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH"; TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling"); var account = CloudStorageAccount.Parse(source.ConnectionString); var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty <string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ?? DefaultIisLogFileFormatConvention; var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } int instanceIndex = 0; var nextOffset = DateTimeOffset.UtcNow; var events = new List <Event>(); var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset); if (fullNumberOfHoursInBetween == 0) { return(Task.FromResult((IEnumerable <Event>)events)); } while (true) { var path = string.Format(pathFormat, instanceIndex); instanceIndex++; TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path); var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob); if (!any) { break; } for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++) { var fileOffset = offset.TimeOffset.AddHours(i); var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; events.Add(new Event(new BlobFileScheduled() { FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume, PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile, NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile, Source = source.ToSummary(), StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80)) })); TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume); } } source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source) { const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH"; TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling"); //var account = CloudStorageAccount.Parse(source.ConnectionString); CloudStorageAccount account; if (!String.IsNullOrWhiteSpace(source.AccountSasKey)) { // Create new storage credentials using the SAS token. var accountSas = new StorageCredentials(source.AccountSasKey); // Use these credentials and the account name to create a Blob service client. account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true); } else { account = CloudStorageAccount.Parse(source.ConnectionString); } var client = account.CreateCloudBlobClient(); var pathFormat = source.GetProperty <string>("BlobPathFormat"); TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat); pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with / var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ?? DefaultIisLogFileFormatConvention; var offset = FileOffset.Parse(source.LastOffsetPoint); if (offset == null) { throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint); } int instanceIndex = 0; var nextOffset = DateTimeOffset.UtcNow; var events = new List <Event>(); var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset); if (fullNumberOfHoursInBetween == 0) { return(Task.FromResult((IEnumerable <Event>)events)); } while (true) { var path = string.Format(pathFormat, instanceIndex); var isSingleInstance = path == pathFormat; instanceIndex++; TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path); var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob); if (!any) { break; } for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++) { var fileOffset = offset.TimeOffset.AddHours(i); var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log"; events.Add(new Event(new BlobFileScheduled() { FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume, PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile, NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile, Source = source.ToSummary(), StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80)), IsRepeat = true })); TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume); } if (isSingleInstance) // this is for when you want to consume IIS logs from only a single VM and not used {0} in blbb format { break; } } source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString(); return(Task.FromResult((IEnumerable <Event>)events)); }