Exemplo n.º 1
0
        public Task <Tuple <IEnumerable <Event>, bool> > TryScheduleAsync(DiagnosticsSource source)
        {
            var key = source.ToTypeKey();

            if (source.IsActive.GetValueOrDefault(true))
            {
                var consume = EventHubConsumer.Consumers.AddOrUpdate(key,
                                                                     new Lazy <EventHubConsumer>(() =>
                {
                    TheTrace.TraceInformation("Just added this eventHub consumer {0}", key);
                    return(new EventHubConsumer(_pusher, source.ToSummary()));
                }),
                                                                     (kk, vv) => vv);

                // to make sure it gets accessed and created if new otherwise system is 'lazy'
                TheTrace.TraceInformation("This is the EventHub thing I was talking about: {0}", consume.Value.Source.TypeName);
            }
            else
            {
                Lazy <EventHubConsumer> consumer = null;

                if (EventHubConsumer.Consumers.TryRemove(key, out consumer))
                {
                    consumer.Value.Dispose();
                    TheTrace.TraceInformation("Just removed this eventHub consumer {0}", key);
                }
            }

            return(Task.FromResult(new Tuple <IEnumerable <Event>, bool>(new Event[0], false)));
        }
Exemplo n.º 2
0
        protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
        {
            if (source.LastOffsetPoint == null)
            {
                source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-7).DropSecondAndMilliseconds().ToString("O");
            }

            var offset       = DateTimeOffset.Parse(source.LastOffsetPoint);
            var events       = new List <Event>();
            var totalMinutes = DateTimeOffset.UtcNow.Subtract(offset.AddMinutes(source.GracePeriodMinutes.Value)).TotalMinutes;

            var ofsted = DateTimeOffset.UtcNow;

            for (int i = 0; i < totalMinutes; i++)
            {
                ofsted = offset.AddMinutes(i + 1);
                var shardKey = GetShardKey(ofsted);
                events.Add(new Event(new ShardKeyArrived()
                {
                    Source = source.ToSummary(), ShardKey = shardKey
                }));
                if (source.MaxItemsInAScheduleRun.HasValue && i >= source.MaxItemsInAScheduleRun)
                {
                    break;
                }
            }

            source.LastOffsetPoint = ofsted.DropSecondAndMilliseconds().ToString("O");
            return(Task.FromResult((IEnumerable <Event>)events));
        }
        protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
        {
            if (source.LastOffsetPoint == null)
            {
                source.LastOffsetPoint = DateTimeOffset.UtcNow.AddDays(-1).DropSecondAndMilliseconds().ToString("O");
            }

            var lastOffset   = DateTimeOffset.Parse(source.LastOffsetPoint);
            var events       = new List <Event>();
            var graceMinutes = source.GracePeriodMinutes ?? 3;

            var now           = DateTimeOffset.UtcNow.DropSecondAndMilliseconds();
            var newLastOffset = lastOffset;
            int n             = 1; // start from a minute after

            while (now >= lastOffset.Add(TimeSpan.FromMinutes(graceMinutes + n)))
            {
                newLastOffset = lastOffset.Add(TimeSpan.FromMinutes(n))
                                .DropSecondAndMilliseconds(); // just to be sure
                var shardKeys = GetShardKeys(newLastOffset);
                events.AddRange(shardKeys.Select(shardKey => new Event(new ShardKeyArrived {
                    Source = source.ToSummary(), ShardKey = shardKey
                })));
                if (source.MaxItemsInAScheduleRun.HasValue && n >= source.MaxItemsInAScheduleRun)
                {
                    break;
                }
                n++;
                TheTrace.TraceInformation("Scheduling {0} for minute {1} and shardkey {2} => {0}_{1} AND {0}_{2}", source.ToTypeKey(), newLastOffset.ToString("yyyyMMddHHmm"), shardKeys.First());
            }

            source.LastOffsetPoint = newLastOffset.ToString("O");
            return(Task.FromResult((IEnumerable <Event>)events));
        }
Exemplo n.º 4
0
        protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
        {
            TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
            //var account = CloudStorageAccount.Parse(source.ConnectionString);
            CloudStorageAccount account;

            if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
            {
                // Create new storage credentials using the SAS token.
                var accountSas = new StorageCredentials(source.AccountSasKey);
                // Use these credentials and the account name to create a Blob service client.
                account = new CloudStorageAccount(accountSas, source.AccountName, "", useHttps: true);
            }
            else
            {
                account = CloudStorageAccount.Parse(source.ConnectionString);
            }
            var client   = account.CreateCloudBlobClient();
            var blobPath = source.GetProperty <string>("BlobPath");

            TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", blobPath);
            blobPath = blobPath.TrimEnd('/') + "/"; // ensure path ends with /
            var offset = FileOffset.Parse(source.LastOffsetPoint);

            if (offset == null)
            {
                throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
            }

            DateTimeOffset maxOffset = offset.TimeOffset;
            FileOffset     newOffset = null;
            var            events    = new List <Event>();

            foreach (var blob in client.ListBlobs(blobPath).Where(itm => itm is CloudBlockBlob)
                     .Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
            {
                if (blob.Properties.LastModified > offset.TimeOffset)
                {
                    var filename = blob.Uri.ToString();
                    newOffset = new FileOffset(filename, blob.Properties.LastModified ?? DateTimeOffset.UtcNow, 0);
                    TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);

                    events.Add(new Event(new BlobFileArrived()
                    {
                        Source      = source.ToSummary(),
                        BlobId      = filename,
                        Position    = 0,
                        EndPosition = blob.Properties.Length
                    }));

                    TheTrace.TraceInformation("Created BlobFileArrived for file: {0}", filename);
                }
            }

            source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();

            return(Task.FromResult((IEnumerable <Event>)events));
        }
        public void CanConvertFromSourceToSummary_WithAlternateTypeName()
        {
            var entity = new DynamicTableEntity("pk", "rk");
            entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
            entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi");
            var source = new DiagnosticsSource(entity);

            var summary = source.ToSummary();

            Assert.Equal(summary.TypeName, source.ToTypeKey());
        }
        public void CanConvertFromSourceToSummary_WithAlternateTypeName()
        {
            var entity = new DynamicTableEntity("pk", "rk");

            entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
            entity.Properties["AlternateTypeName"] = EntityProperty.GeneratePropertyForString("vahshi");
            var source = new DiagnosticsSource(entity);

            var summary = source.ToSummary();

            Assert.Equal(summary.TypeName, source.ToTypeKey());
        }
Exemplo n.º 7
0
        protected async override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
        {
            TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
            var account    = CloudStorageAccount.Parse(source.ConnectionString);
            var client     = account.CreateCloudBlobClient();
            var pathFormat = source.GetProperty <string>("BlobPathFormat");

            TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat);
            pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /

            var            offset        = DateTimeOffset.Parse(source.LastOffsetPoint);
            int            instanceIndex = 0;
            DateTimeOffset maxOffset     = offset;
            var            events        = new List <Event>();

            while (true)
            {
                bool found = false;
                var  path  = string.Format(pathFormat, instanceIndex);
                TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path);
                foreach (var itm in client.ListBlobs(path))
                {
                    var blob = itm as CloudBlockBlob;
                    if (blob != null && blob.Properties.LastModified > offset)
                    {
                        TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
                        found     = true;
                        maxOffset = offset > blob.Properties.LastModified.Value
                            ? offset
                            : blob.Properties.LastModified.Value;
                        events.Add(new Event(new BlobFileArrived()
                        {
                            Source = source.ToSummary(), BlobId = blob.Uri.ToString()
                        }));
                    }
                }

                if (!found)
                {
                    TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex);
                    break;
                }
                instanceIndex++;
            }

            source.LastOffsetPoint = maxOffset.ToString("O");
            return(events);
        }
        protected async override Task<IEnumerable<Event>> DoSchedule(DiagnosticsSource source)
        {
            TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");
            var account = CloudStorageAccount.Parse(source.ConnectionString);
            var client = account.CreateCloudBlobClient();
            var pathFormat = source.GetProperty<string>("BlobPathFormat");
            TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat);
            pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /

            var offset = DateTimeOffset.Parse(source.LastOffsetPoint);
            int instanceIndex = 0;
            DateTimeOffset maxOffset = offset;
            var events = new List<Event>();
            while (true)
            {
                bool found = false;
                var path = string.Format(pathFormat, instanceIndex);
                TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path);
                foreach (var itm in client.ListBlobs(path))
                {
                    var blob = itm as CloudBlockBlob;
                    if (blob != null && blob.Properties.LastModified > offset)
                    {
                        TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
                        found = true;
                        maxOffset = offset > blob.Properties.LastModified.Value
                            ? offset
                            : blob.Properties.LastModified.Value;
                        events.Add(new Event(new BlobFileArrived() { Source = source.ToSummary(), BlobId = blob.Uri.ToString() }));
                    }
                }

                if (!found)
                {
                    TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex);
                    break;
                }
                instanceIndex++;
            }

            source.LastOffsetPoint = maxOffset.ToString("O");
            return events;
        }
        public void CanConvertFromSourceToSummary()
        {
            var entity = new DynamicTableEntity("pk", "rk");
            entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
            entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man");
            entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow);
            entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true);
            var source = new DiagnosticsSource(entity);

            var summary = source.ToSummary();
            Assert.Equal(summary.ConnectionString, source.ConnectionString);
            Assert.Equal(summary.PartitionKey, source.PartitionKey);
            Assert.Equal(summary.RowKey, source.RowKey);
            Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty<DateTime>("dpd"));
            Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty<int>("dpi"));
            Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty<string>("dps"));
            Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty<bool>("dpb"));
            Assert.Equal(summary.TypeName, source.ToTypeKey());
        }
        public void CanConvertFromSourceToSummary()
        {
            var entity = new DynamicTableEntity("pk", "rk");

            entity.Properties["dpi"] = EntityProperty.GeneratePropertyForInt(2);
            entity.Properties["dps"] = EntityProperty.GeneratePropertyForString("man");
            entity.Properties["dpd"] = EntityProperty.GeneratePropertyForDateTimeOffset(DateTime.UtcNow);
            entity.Properties["dpb"] = EntityProperty.GeneratePropertyForBool(true);
            var source = new DiagnosticsSource(entity);

            var summary = source.ToSummary();

            Assert.Equal(summary.ConnectionString, source.ConnectionString);
            Assert.Equal(summary.PartitionKey, source.PartitionKey);
            Assert.Equal(summary.RowKey, source.RowKey);
            Assert.Equal(summary.DynamicProperties["dpd"], source.GetProperty <DateTime>("dpd"));
            Assert.Equal(summary.DynamicProperties["dpi"], source.GetProperty <int>("dpi"));
            Assert.Equal(summary.DynamicProperties["dps"], source.GetProperty <string>("dps"));
            Assert.Equal(summary.DynamicProperties["dpb"], source.GetProperty <bool>("dpb"));
            Assert.Equal(summary.TypeName, source.ToTypeKey());
        }
Exemplo n.º 11
0
        protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
        {
            TheTrace.TraceInformation("IisBlobScheduler - Starting scheduling");

            CloudStorageAccount account;

            if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
            {
                // Create new storage credentials using the SAS token.
                var accountSas = new StorageCredentials(source.AccountSasKey);
                // Use these credentials and the account name to create a Blob service client.
                account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true);
            }
            else
            {
                account = CloudStorageAccount.Parse(source.ConnectionString);
            }

            var client     = account.CreateCloudBlobClient();
            var pathFormat = source.GetProperty <string>("BlobPathFormat");

            TheTrace.TraceInformation("IisBlobScheduler - pathformat: {0}", pathFormat);
            pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /

            var offset = FileOffset.Parse(source.LastOffsetPoint);

            if (offset == null)
            {
                throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
            }
            int            instanceIndex = 0;
            DateTimeOffset maxOffset     = offset.TimeOffset;
            FileOffset     newOffset     = null;
            var            events        = new List <Event>();

            while (true)
            {
                bool found = false;

                var path             = string.Format(pathFormat, instanceIndex);
                var isSingleInstance = path == pathFormat;


                TheTrace.TraceInformation("IisBlobScheduler - Looking into {0}", path);
                foreach (var blob in client.ListBlobs(path).Where(itm => itm is CloudBlockBlob)
                         .Cast <CloudBlockBlob>().OrderBy(x => x.Properties.LastModified))
                {
                    if (blob.Properties.LastModified > offset.TimeOffset)
                    {
                        var filename = blob.Uri.ToString();
                        if (!found) // first time running
                        {
                            newOffset = new FileOffset(filename,
                                                       blob.Properties.LastModified ?? DateTimeOffset.UtcNow, blob.Properties.Length);
                        }

                        TheTrace.TraceInformation("IisBlobScheduler - found {0}", blob.Uri);
                        found = true;
                        events.Add(new Event(new BlobFileArrived()
                        {
                            Source      = source.ToSummary(),
                            BlobId      = filename,
                            Position    = (filename == offset.FileName) ? offset.Position : 0, // if same file then pass offset
                            EndPosition = blob.Properties.Length
                        }));
                    }
                }

                if (!found || isSingleInstance)
                {
                    TheTrace.TraceInformation("IisBlobScheduler - Breaking out with index of {0}", instanceIndex);
                    break;
                }

                instanceIndex++;
            }

            source.LastOffsetPoint = newOffset == null?offset.ToString() : newOffset.ToString();

            return(Task.FromResult((IEnumerable <Event>)events));
        }
Exemplo n.º 12
0
        protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
        {
            const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH";

            TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling");
            var account    = CloudStorageAccount.Parse(source.ConnectionString);
            var client     = account.CreateCloudBlobClient();
            var pathFormat = source.GetProperty <string>("BlobPathFormat");

            TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat);
            pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
            var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ?? DefaultIisLogFileFormatConvention;

            var offset = FileOffset.Parse(source.LastOffsetPoint);

            if (offset == null)
            {
                throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
            }

            int instanceIndex = 0;

            var nextOffset = DateTimeOffset.UtcNow;
            var events     = new List <Event>();
            var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset);

            if (fullNumberOfHoursInBetween == 0)
            {
                return(Task.FromResult((IEnumerable <Event>)events));
            }

            while (true)
            {
                var path = string.Format(pathFormat, instanceIndex);
                instanceIndex++;
                TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path);
                var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob);
                if (!any)
                {
                    break;
                }

                for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++)
                {
                    var fileOffset    = offset.TimeOffset.AddHours(i);
                    var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
                    var previousFile  = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
                    var nextFile      = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
                    events.Add(new Event(new BlobFileScheduled()
                    {
                        FileToConsume    = path.Replace("wad-iis-logfiles/", "") + fileToConsume,
                        PreviousFile     = path.Replace("wad-iis-logfiles/", "") + previousFile,
                        NextFile         = path.Replace("wad-iis-logfiles/", "") + nextFile,
                        Source           = source.ToSummary(),
                        StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80))
                    }));

                    TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume);
                }
            }

            source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString();
            return(Task.FromResult((IEnumerable <Event>)events));
        }
        protected override Task <IEnumerable <Event> > DoSchedule(DiagnosticsSource source)
        {
            const string DefaultIisLogFileFormatConvention = "u_exyyMMddHH";

            TheTrace.TraceInformation("IisBlobConventionScheduler - Starting scheduling");
            //var account = CloudStorageAccount.Parse(source.ConnectionString);
            CloudStorageAccount account;

            if (!String.IsNullOrWhiteSpace(source.AccountSasKey))
            {
                // Create new storage credentials using the SAS token.
                var accountSas = new StorageCredentials(source.AccountSasKey);
                // Use these credentials and the account name to create a Blob service client.
                account = new CloudStorageAccount(accountSas, source.AccountName, endpointSuffix: "", useHttps: true);
            }
            else
            {
                account = CloudStorageAccount.Parse(source.ConnectionString);
            }
            var client     = account.CreateCloudBlobClient();
            var pathFormat = source.GetProperty <string>("BlobPathFormat");

            TheTrace.TraceInformation("IisBlobConventionScheduler - pathformat: {0}", pathFormat);
            pathFormat = pathFormat.TrimEnd('/') + "/"; // ensure path ends with /
            var iisLogFileFormatConvention = source.GetProperty <string>("IisLogFileFormatConvention") ??
                                             DefaultIisLogFileFormatConvention;

            var offset = FileOffset.Parse(source.LastOffsetPoint);

            if (offset == null)
            {
                throw new InvalidOperationException("FileOffset failed parsing: => " + source.LastOffsetPoint);
            }

            int instanceIndex = 0;

            var nextOffset = DateTimeOffset.UtcNow;
            var events     = new List <Event>();
            var fullNumberOfHoursInBetween = offset.TimeOffset.GetFullNumberOfHoursInBetween(nextOffset);

            if (fullNumberOfHoursInBetween == 0)
            {
                return(Task.FromResult((IEnumerable <Event>)events));
            }

            while (true)
            {
                var path             = string.Format(pathFormat, instanceIndex);
                var isSingleInstance = path == pathFormat;

                instanceIndex++;
                TheTrace.TraceInformation("IisBlobConventionScheduler - Looking into {0}", path);
                var any = client.ListBlobs(path).Any(itm => itm is CloudBlockBlob);
                if (!any)
                {
                    break;
                }

                for (int i = 1; i < fullNumberOfHoursInBetween + 1; i++)
                {
                    var fileOffset    = offset.TimeOffset.AddHours(i);
                    var fileToConsume = fileOffset.UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
                    var previousFile  = offset.TimeOffset.AddHours(i - 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
                    var nextFile      = offset.TimeOffset.AddHours(i + 1).UtcDateTime.ToString(iisLogFileFormatConvention) + ".log";
                    events.Add(new Event(new BlobFileScheduled()
                    {
                        FileToConsume    = path.Replace("wad-iis-logfiles/", "") + fileToConsume,
                        PreviousFile     = path.Replace("wad-iis-logfiles/", "") + previousFile,
                        NextFile         = path.Replace("wad-iis-logfiles/", "") + nextFile,
                        Source           = source.ToSummary(),
                        StopChasingAfter = fileOffset.Add(TimeSpan.FromMinutes(80)),
                        IsRepeat         = true
                    }));

                    TheTrace.TraceInformation("IisBlobConventionScheduler - Scheduled Event: {0}", fileToConsume);
                }

                if (isSingleInstance) // this is for when you want to consume IIS logs from only a single VM and not used {0} in blbb format
                {
                    break;
                }
            }

            source.LastOffsetPoint = new FileOffset(string.Empty, nextOffset).ToString();
            return(Task.FromResult((IEnumerable <Event>)events));
        }