public IEnumerable <string> GetIndexNames(DiagnosticsSource source, int daysToGoBack = 7)
{
// static index name
if (!string.IsNullOrEmpty(source.IndexName))
{
return new[] { source.IndexName }
}
;
if (String.IsNullOrEmpty(source.LastOffsetPoint))
{
source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-daysToGoBack).ToString("O");
}
var dateTimeOffset = FileOffset.Parse(source.LastOffsetPoint);
var days = (int)(DateTimeOffset.UtcNow.AddDays(1) - dateTimeOffset.TimeOffset).TotalDays + 1; // to cover today as well - Aboo was here
if (days <= 0)
{
return(Enumerable.Empty <string>());
}
return(Enumerable.Range(0, days).Select(x => DateTimeOffset.UtcNow.AddDays(1).AddDays(-x))
.Select(z => _indexNamer.BuildName(dateTimeOffset.TimeOffset, source.GetMappingName().ToLowerInvariant())));
}
}
public IEnumerable <string> GetIndexNames(int daysToGoBack = 7)
{
if (!string.IsNullOrEmpty(IndexName))
{
return new [] { IndexName }
}
;
if (String.IsNullOrEmpty(LastOffsetPoint))
{
LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-daysToGoBack).ToString("O");
}
var dateTimeOffset = FileOffset.Parse(LastOffsetPoint);
var days = (int)(DateTimeOffset.UtcNow.AddDays(1) - dateTimeOffset.TimeOffset).TotalDays + 1; // to cover today as well - Aboo was here
if (days <= 0)
{
return(Enumerable.Empty <string>());
}
return(Enumerable.Range(0, days).Select(x => DateTimeOffset.UtcNow.AddDays(1).AddDays(-x))
.Select(y => y.ToString("yyyyMMdd")));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
//var account = CloudStorageAccount.Parse(source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var blobPath = source.GetProperty <string>("BlobPath");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", blobPath);
blobPath = blobPath.TrimEnd('/') + "/"; // ensure path ends with /
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
DateTimeOffset maxOffset = offset.TimeOffset;
FileOffset newOffset = null;
var events = new List <Event>();
foreach (var blob in client.ListBlobs(blobPath).Where(itm => itm is CloudBlockBlob)
.Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
{
if (blob.Properties.LastModified > offset.TimeOffset)
{
var filename = blob.Uri.ToString();
newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, 0);
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
events.Add(new Event(new BlobFileArrived()
{
Source = source.ToSummary(),
BlobId = filename,
Position = 0,
EndPosition = blob.Properties.Length
}));
TheTrace.TraceInformation("Created BlobFileArrived for file: {0}", filename);
}
}
source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
public DiagnosticsSourceSummary ToSummary()
{
var dss = new DiagnosticsSourceSummary()
{
ConnectionString = ConnectionString,
AccountSasKey = AccountSasKey,
AccountName = AccountName,
IndexName = IndexName,
PartitionKey = PartitionKey,
RowKey = RowKey,
TypeName = ToTypeKey(),
LastTimeOffset = FileOffset.Parse(LastOffsetPoint)?.TimeOffset,
DynamicProperties = new Dictionary <string, object>()
};
foreach (var kv in _entity.Properties)
{
dss.DynamicProperties.Add(kv.Key, kv.Value.PropertyAsObject);
}
return(dss);
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty <string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
int instanceIndex = 0;
DateTimeOffset maxOffset = offset.TimeOffset;
FileOffset newOffset = null;
var events = new List <Event>();
while (true)
{
bool found = false;
var path = string.Format(pathFormat, instanceIndex);
var isSingleInstance = path == pathFormat;
TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path);
foreach (var blob in client.ListBlobs(path).Where(itm => itm is CloudBlockBlob)
.Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
{
if (blob.Properties.LastModified > offset.TimeOffset)
{
var filename = blob.Uri.ToString();
if (!found) // first time running
{
newOffset = new FileOffset(filename,
blob.Properties.LastModified ?? DateTimeOffset.UtcNow, blob.Properties.Length);
}
TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
found = true;
events.Add(new Event(new BlobFileArrived()
{
Source = source.ToSummary(),
BlobId = filename,
Position = (filename == offset.FileName) ? offset.Position : 0, // if same file then pass offset
EndPosition = blob.Properties.Length
}));
}
}
if (!found || isSingleInstance)
{
TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex);
break;
}
instanceIndex++;
}
source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH";
TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling");
var account = CloudStorageAccount.Parse(source.ConnectionString);
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty <string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ?? DefaultIisLogFileFormatConvention;
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
int instanceIndex = 0;
var nextOffset = DateTimeOffset.UtcNow;
var events = new List <Event>();
var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset);
if (fullNumberOfHoursInBetween == 0)
{
return(Task.FromResult((IEnumerable <Event>)events));
}
while (true)
{
var path = string.Format(pathFormat, instanceIndex);
instanceIndex++;
TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path);
var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob);
if (!any)
{
break;
}
for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++)
{
var fileOffset = offset.TimeOffset.AddHours(i);
var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
events.Add(new Event(new BlobFileScheduled()
{
FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume,
PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile,
NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile,
Source = source.ToSummary(),
StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80))
}));
TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume);
}
}
source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}
protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
{
const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH";
TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling");
//var account = CloudStorageAccount.Parse(source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var pathFormat = source.GetProperty <string>("BlobPathFormat");
TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat);
pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ??
DefaultIisLogFileFormatConvention;
var offset = FileOffset.Parse(source.LastOffsetPoint);
if (offset == null)
{
throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
}
int instanceIndex = 0;
var nextOffset = DateTimeOffset.UtcNow;
var events = new List <Event>();
var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset);
if (fullNumberOfHoursInBetween == 0)
{
return(Task.FromResult((IEnumerable <Event>)events));
}
while (true)
{
var path = string.Format(pathFormat, instanceIndex);
var isSingleInstance = path == pathFormat;
instanceIndex++;
TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path);
var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob);
if (!any)
{
break;
}
for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++)
{
var fileOffset = offset.TimeOffset.AddHours(i);
var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var previousFile = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
var nextFile = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
events.Add(new Event(new BlobFileScheduled()
{
FileToConsume = path.Replace("wad-iis-logfiles/", "") + fileToConsume,
PreviousFile = path.Replace("wad-iis-logfiles/", "") + previousFile,
NextFile = path.Replace("wad-iis-logfiles/", "") + nextFile,
Source = source.ToSummary(),
StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80)),
IsRepeat = true
}));
TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume);
}
if (isSingleInstance) // this is for when you want to consume IIS logs from only a single VM and not used {0} in blbb format
{
break;
}
}
source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString();
return(Task.FromResult((IEnumerable <Event>)events));
}