コード例 #1
0
        public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
        {
            var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();

            _telemetryProvider.WriteTelemetry(
                "ShardKey receive message delay duration",
                (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
                shardKeyArrived.Source.TypeName);

            await _durationInstrumentor.InstrumentAsync(async() =>
            {
                TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
                                          shardKeyArrived.Source.TypeName);

                var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
                var query           = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
                var entities        = await query.QueryAsync(shardKeyArrived);

                var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");

                await _pusher.PushAll(PreprocessEntities(entities, shardKeyArrived, shardKeyTime), shardKeyArrived.Source).ConfigureAwait(false);
            }).ConfigureAwait(false);

            return(Enumerable.Empty <Event>());
        }
コード例 #2
0
        public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
        {
            var blobFileArrived = evnt.GetBody <BlobFileArrived>();

            _telemetryProvider.WriteTelemetry(
                "BlobFileActor receive message delay duration",
                (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
                blobFileArrived.Source.TypeName);

            await _durationInstrumentor.InstrumentAsync(async() =>
            {
                TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
                                          blobFileArrived.Source.TypeName);
                //var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
                CloudStorageAccount account;
                if (!String.IsNullOrWhiteSpace(blobFileArrived.Source.AccountSasKey))
                {
                    // Create new storage credentials using the SAS token.
                    var accountSas = new StorageCredentials(blobFileArrived.Source.AccountSasKey);
                    // Use these credentials and the account name to create a Blob service client.
                    account = new CloudStorageAccount(accountSas, blobFileArrived.Source.AccountName, "", useHttps: true);
                }
                else
                {
                    account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
                }
                var client    = account.CreateCloudBlobClient();
                var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
                var uri       = new Uri(blobFileArrived.BlobId);
                var id        = string.Join("", uri.Segments.Skip(2));
                var blob      = container.GetBlockBlobReference(id);
                if (!blob.Exists())
                {
                    throw new InvalidOperationException("Blob does not exist: " + id);
                }
                var stream = new MemoryStream();
                await blob.DownloadToStreamAsync(stream);
                var parser      = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
                var hasAnything = false;
                var minDateTime = DateTimeOffset.UtcNow;

                var records = parser.Parse(() => stream, blob.Uri, blobFileArrived.Source, new ParseCursor(blobFileArrived.Position ?? 0)
                {
                    EndPosition = blobFileArrived.EndPosition ?? 0
                });
                var seenPages = await _pusher.PushAll(records, blobFileArrived.Source).ConfigureAwait(false);
                hasAnything   = seenPages > 0;

                if (hasAnything)
                {
                    _telemetryProvider.WriteTelemetry(
                        "BlobFileActor message processing duration",
                        (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
                        blobFileArrived.Source.TypeName);
                }
            }, blobFileArrived.Source.TypeName);

            return(Enumerable.Empty <Event>());
        }
コード例 #3
0
        public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
        {
            var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();

            _telemetryProvider.WriteTelemetry(
                "ShardKey receive message delay duration",
                (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
                shardKeyArrived.Source.TypeName);

            await _durationInstrumentor.InstrumentAsync(async() =>
            {
                TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
                                          shardKeyArrived.Source.TypeName);

                var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
                var query           = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
                var entities        = await query.QueryAsync(shardKeyArrived);

                var minDateTime  = DateTimeOffset.MaxValue;
                var hasAnything  = false;
                int n            = 0;
                var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");

                foreach (var entity in entities)
                {
                    var eventDateTimeOffset = entity.GetEventDateTimeOffset();
                    var delayInSeconds      = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
                    if (delayInSeconds >= _shardKeyDelayWarning)
                    {
                        TheTrace.TraceWarning("SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
                                              delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
                    }

                    entity.Timestamp = eventDateTimeOffset;
                    await _pusher.PushAsync(entity, shardKeyArrived.Source);
                    hasAnything = true;
                    minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
                    n++;
                }

                TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
                                          shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);

                if (hasAnything)
                {
                    await _pusher.FlushAsync();

                    _telemetryProvider.WriteTelemetry(
                        "ShardKeyArrivedActor log delay duration",
                        (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
                        shardKeyArrived.Source.TypeName);
                }
            });

            return(Enumerable.Empty <Event>());
        }
コード例 #4
0
        public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
        {
            var blobFileArrived = evnt.GetBody <BlobFileArrived>();

            _telemetryProvider.WriteTelemetry(
                "BlobFileActor receive message delay duration",
                (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
                blobFileArrived.Source.TypeName);

            await _durationInstrumentor.InstrumentAsync(async() =>
            {
                TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
                                          blobFileArrived.Source.TypeName);
                var account   = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
                var client    = account.CreateCloudBlobClient();
                var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
                var uri       = new Uri(blobFileArrived.BlobId);
                var id        = string.Join("", uri.Segments.Skip(2));
                var blob      = container.GetBlockBlobReference(id);
                if (!blob.Exists())
                {
                    throw new InvalidOperationException("Blob does not exist: " + id);
                }
                var stream = new MemoryStream();
                await blob.DownloadToStreamAsync(stream);
                var parser      = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
                var hasAnything = false;
                var minDateTime = DateTimeOffset.MaxValue;
                foreach (var entity in parser.Parse(stream, blob.Uri, blobFileArrived.Position ?? 0, blobFileArrived.EndPosition ?? 0))
                {
                    await _pusher.PushAsync(entity, blobFileArrived.Source);
                    hasAnything = true;
                    minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
                }

                if (hasAnything)
                {
                    await _pusher.FlushAsync();
                    _telemetryProvider.WriteTelemetry(
                        "BlobFileActor log delay duration",
                        (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
                        blobFileArrived.Source.TypeName);
                }
            }, blobFileArrived.Source.TypeName);

            return(Enumerable.Empty <Event>());
        }
コード例 #5
0
        public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
        {
            var events            = new List <Event>();
            var blobFileScheduled = evnt.GetBody <BlobFileScheduled>();

            _telemetryProvider.WriteTelemetry(
                "BlobFileScheduled receive message delay duration",
                (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
                blobFileScheduled.Source.TypeName);

            await _durationInstrumentor.InstrumentAsync(async() =>
            {
                var account   = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString);
                var client    = account.CreateCloudBlobClient();
                var container = client.GetContainerReference("wad-iis-logfiles");

                bool nextBlobExists;
                bool previousBlobExists;
                bool mainBlobExists;
                Func <string, string> alternater = s => s.Replace(".log", "_x.log");
                container.GetBlobReferenceWithAlternateName(blobFileScheduled.NextFile, alternater, out nextBlobExists);
                container.GetBlobReferenceWithAlternateName(blobFileScheduled.PreviousFile, alternater, out previousBlobExists);
                var mainBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.FileToConsume, alternater, out mainBlobExists);

                if (!previousBlobExists && !mainBlobExists)
                {
                    TheTrace.TraceInformation("BlobFileConventionActor - previous blob does not exist. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                    return; // will never be here. Stop chasing it.
                }

                long currentLength = 0;
                if (mainBlobExists)
                {
                    currentLength = mainBlob.Properties.Length;
                    if (currentLength == blobFileScheduled.LastPosition)
                    {
                        if (nextBlobExists)
                        {
                            TheTrace.TraceInformation("BlobFileConventionActor - Next blob exists. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                            return; // done and dusted. Stop chasing it.
                        }

                        if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
                        {
                            TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                            return; // Stop chasing it.
                        }
                    }
                    else
                    {
                        var stream = await DownloadToFileAsync(mainBlob);
                        try
                        {
                            currentLength   = stream.Length;
                            var parser      = FactoryHelper.Create <IParser>(blobFileScheduled.Source.DynamicProperties["Parser"].ToString(), typeof(IisBlobConventionScheduler));
                            var hasAnything = false;
                            var minDateTime = DateTimeOffset.MaxValue;
                            foreach (var entity in parser.Parse(stream, mainBlob.Uri, blobFileScheduled.LastPosition))
                            {
                                await _pusher.PushAsync(entity, blobFileScheduled.Source);
                                hasAnything = true;
                                minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
                            }

                            if (hasAnything)
                            {
                                await _pusher.FlushAsync();
                                TheTrace.TraceInformation("BlobFileConventionActor - pushed records for {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                                _telemetryProvider.WriteTelemetry(
                                    "BlobFileConventionActor log delay duration",
                                    (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
                                    blobFileScheduled.Source.TypeName);
                            }
                        }
                        finally
                        {
                            stream.Close();
                            File.Delete(stream.Name);
                        }
                    }
                }
                else
                {
                    if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
                    {
                        TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                        return; // Stop chasing it.
                    }
                }

                blobFileScheduled.LastPosition = currentLength;

                // let's defer
                events.Add(new Event(blobFileScheduled)
                {
                    EnqueueTime = DateTimeOffset.Now.Add(TimeSpan.FromSeconds(30))
                });

                TheTrace.TraceInformation("BlobFileConventionActor - deferred processing {0}. Length => {1}  at {2}", blobFileScheduled.FileToConsume, currentLength, DateTimeOffset.Now);
            });

            return(events);
        }
コード例 #6
0
        private async Task <DiagnosticsSource> TryScheduleSourceAsync(DiagnosticsSource source)
        {
            try
            {
                source = _sourceConfiguration.RefreshSource(source);

                TheTrace.TraceInformation("MasterScheduler - Scheduling {0}", source.ToTypeKey());

                if (!source.IsActive.HasValue || !source.IsActive.Value)
                {
                    TheTrace.TraceInformation("MasterScheduler - NOT active: {0}", source.ToTypeKey());
                    return(null);
                }

                var createMappings = _configurationValueProvider.GetValue(ConfigurationKeys.EsCreateMappings);
                if (Convert.ToBoolean(createMappings))
                {
                    await SetupMappingsAsync(source);
                }

                if (!source.LastScheduled.HasValue)
                {
                    source.LastScheduled = DateTimeOffset.UtcNow.AddDays(-1);
                }

                // if has been recently scheduled
                if (source.LastScheduled.Value.AddMinutes(source.SchedulingFrequencyMinutes.Value) >
                    DateTimeOffset.UtcNow)
                {
                    TheTrace.TraceInformation("MasterScheduler - Nothing to do with {0}. LastScheduled in Future {1}",
                                              source.ToTypeKey(), source.LastScheduled.Value);
                    return(null);
                }

                _telemetryProvider.WriteTelemetry(
                    "MasterScheduler duration since last scheduled",
                    (long)(DateTime.UtcNow - source.LastScheduled).Value.TotalMilliseconds,
                    source.ToTypeKey());

                var schedulerType = Assembly.GetExecutingAssembly().GetType(source.SchedulerType) ??
                                    Type.GetType(source.SchedulerType);
                if (schedulerType == null)
                {
                    source.ErrorMessage = "Could not find SchedulerType: " + source.SchedulerType;
                }
                else
                {
                    await _scheduleDurationInstrumentor.InstrumentAsync(async() =>
                    {
                        var scheduler        = (ISourceScheduler)_locator.GetService(schedulerType);
                        var result           = await scheduler.TryScheduleAsync(source);
                        source.LastScheduled = DateTimeOffset.UtcNow;
                        TheTrace.TraceInformation(
                            "MasterScheduler - Got result for TryScheduleAsync in {0}. Success => {1}",
                            source.ToTypeKey(), result.Item1);

                        if (result.Item2)
                        {
                            await _eventQueueOperator.PushBatchAsync(result.Item1);
                        }

                        source.ErrorMessage = string.Empty;
                        TheTrace.TraceInformation("MasterScheduler - Finished Scheduling {0}", source.ToTypeKey());
                    }, source.ToTypeKey());
                }

                return(source);
            }
            catch (Exception e)
            {
                TheTrace.TraceError(e.ToString());
                source.ErrorMessage = e.ToString();
                return(source);
            }
        }
コード例 #7
0
        public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
        {
            var events            = new List <Event>();
            var blobFileScheduled = evnt.GetBody <BlobFileScheduled>();

            _telemetryProvider.WriteTelemetry(
                "BlobFileScheduled receive message delay duration",
                (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
                blobFileScheduled.Source.TypeName);

            await _durationInstrumentor.InstrumentAsync(async() =>
            {
                //var account = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString);
                CloudStorageAccount account;
                if (!String.IsNullOrWhiteSpace(blobFileScheduled.Source.AccountSasKey))
                {
                    // Create new storage credentials using the SAS token.
                    var accountSas = new StorageCredentials(blobFileScheduled.Source.AccountSasKey);
                    // Use these credentials and the account name to create a Blob service client.
                    account = new CloudStorageAccount(accountSas, blobFileScheduled.Source.AccountName, "", useHttps: true);
                }
                else
                {
                    account = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString);
                }
                var client    = account.CreateCloudBlobClient();
                var container = client.GetContainerReference("wad-iis-logfiles");

                bool nextBlobExists;
                bool previousBlobExists;
                bool mainBlobExists;
                Func <string, string> alternater = s => s.Replace(".log", "_x.log");
                container.GetBlobReferenceWithAlternateName(blobFileScheduled.NextFile, alternater, out nextBlobExists);
                container.GetBlobReferenceWithAlternateName(blobFileScheduled.PreviousFile, alternater, out previousBlobExists);
                var mainBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.FileToConsume, alternater, out mainBlobExists);

                if (!previousBlobExists && !mainBlobExists)
                {
                    TheTrace.TraceInformation("BlobFileConventionActor - previous blob does not exist. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                    return; // will never be here. Stop chasing it.
                }

                long currentLength = 0;
                long currentOffset = 0;
                if (mainBlobExists)
                {
                    currentLength = mainBlob.Properties.Length;
                    currentOffset = blobFileScheduled.LastPosition;
                    if (currentLength > blobFileScheduled.LastPosition)
                    {
                        var parser      = FactoryHelper.Create <IParser>(blobFileScheduled.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
                        var minDateTime = DateTimeOffset.UtcNow;

                        var cursor        = new ParseCursor(blobFileScheduled.LastPosition);
                        var parsedRecords = parser.Parse(() => StreamFactory(mainBlob), mainBlob.Uri, blobFileScheduled.Source, cursor);
                        var pages         = await _pusher.PushAll(parsedRecords, blobFileScheduled.Source).ConfigureAwait(false);
                        currentOffset     = cursor.EndPosition;

                        if (pages > 0)
                        {
                            TheTrace.TraceInformation("BlobFileConventionActor - pushed records for {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                            _telemetryProvider.WriteTelemetry("BlobFileConventionActor log delay duration", (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds, blobFileScheduled.Source.TypeName);
                        }
                    }

                    currentLength = mainBlob.Properties.Length;
                    if (nextBlobExists)
                    {
                        TheTrace.TraceInformation("BlobFileConventionActor - Next blob exists. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                        return; // done and dusted. Stop chasing it.
                    }
                }

                if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
                {
                    TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
                    return; // Stop chasing it.
                }

                if (currentOffset == blobFileScheduled.LastPosition && currentOffset < blobFileScheduled.LastPosition)
                {
                    throw new Exception("Did nothing but did not reach EOF");
                }

                blobFileScheduled.LastPosition = currentOffset;

                // let's defer
                events.Add(new Event(blobFileScheduled)
                {
                    EnqueueTime = DateTimeOffset.Now.Add(TimeSpan.FromSeconds(30))
                });

                TheTrace.TraceInformation("BlobFileConventionActor - deferred processing {0}. Length => {1}  at {2}", blobFileScheduled.FileToConsume, currentLength, DateTimeOffset.Now);
            });

            return(events);
        }