public async Task <IEnumerable <Event> > ProcessAsync(Event evnt) { var shardKeyArrived = evnt.GetBody <ShardRangeArrived>(); TheTrace.TraceInformation("Got {0}->{1} from {2}", shardKeyArrived.InclusiveStartKey, shardKeyArrived.InclusiveEndKey, shardKeyArrived.Source.TypeName); var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString); var client = account.CreateCloudTableClient(); var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString()); var entities = table.ExecuteQuery(new TableQuery().Where( TableQuery.CombineFilters( TableQuery.GenerateFilterCondition("PartitionKey", "ge", shardKeyArrived.InclusiveStartKey), TableOperators.And, TableQuery.GenerateFilterCondition("PartitionKey", "le", shardKeyArrived.InclusiveEndKey)))); bool hasAnything = false; foreach (var entity in entities) { await _pusher.PushAsync(entity, shardKeyArrived.Source); hasAnything = true; } if (hasAnything) { await _pusher.FlushAsync(); } return(Enumerable.Empty <Event>()); }
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt) { var shardKeyArrived = evnt.GetBody <ShardKeyArrived>(); TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey, shardKeyArrived.Source.TypeName); var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery); var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery)); var entities = await query.QueryAsync(shardKeyArrived); bool hasAnything = false; foreach (var entity in entities) { await _pusher.PushAsync(entity, shardKeyArrived.Source); hasAnything = true; } if (hasAnything) { await _pusher.FlushAsync(); } return(Enumerable.Empty <Event>()); }
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt) { var shardKeyArrived = evnt.GetBody <ShardKeyArrived>(); _telemetryProvider.WriteTelemetry( "ShardKey receive message delay duration", (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds, shardKeyArrived.Source.TypeName); await _durationInstrumentor.InstrumentAsync(async() => { TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey, shardKeyArrived.Source.TypeName); var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery); var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery)); var entities = await query.QueryAsync(shardKeyArrived); var minDateTime = DateTimeOffset.MaxValue; var hasAnything = false; int n = 0; var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm"); foreach (var entity in entities) { var eventDateTimeOffset = entity.GetEventDateTimeOffset(); var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds; if (delayInSeconds >= _shardKeyDelayWarning) { TheTrace.TraceWarning("SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}", delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime); } entity.Timestamp = eventDateTimeOffset; await _pusher.PushAsync(entity, shardKeyArrived.Source); hasAnything = true; minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime; n++; } TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime); if (hasAnything) { await _pusher.FlushAsync(); _telemetryProvider.WriteTelemetry( "ShardKeyArrivedActor log delay duration", (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds, shardKeyArrived.Source.TypeName); } }); return(Enumerable.Empty <Event>()); }
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt) { var blobFileArrived = evnt.GetBody <BlobFileArrived>(); TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId, blobFileArrived.Source.TypeName); var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString); var client = account.CreateCloudBlobClient(); var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString()); var uri = new Uri(blobFileArrived.BlobId); var id = string.Join("", uri.Segments.Skip(2)); var blob = container.GetBlockBlobReference(id); if (!blob.Exists()) { throw new InvalidOperationException("Blob does not exist: " + id); } var stream = new MemoryStream(); await blob.DownloadToStreamAsync(stream); var parserTypeName = blobFileArrived.Source.DynamicProperties["Parser"].ToString(); if (parserTypeName == null) { throw new NullReferenceException("parserTypeName"); } var parserType = Assembly.GetExecutingAssembly().GetType(parserTypeName) ?? Type.GetType(parserTypeName); if (parserType == null) { throw new InvalidOperationException("Parser type was null: " + parserTypeName); } var parser = (IParser)Activator.CreateInstance(parserType); bool hasAnything = false; foreach (var entity in parser.Parse(stream, blob.Uri)) { await _pusher.PushAsync(entity, blobFileArrived.Source); hasAnything = true; } if (hasAnything) { await _pusher.FlushAsync(); } return(Enumerable.Empty <Event>()); }
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt) { var blobFileArrived = evnt.GetBody <BlobFileArrived>(); _telemetryProvider.WriteTelemetry( "BlobFileActor receive message delay duration", (long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds, blobFileArrived.Source.TypeName); await _durationInstrumentor.InstrumentAsync(async() => { TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId, blobFileArrived.Source.TypeName); var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString); var client = account.CreateCloudBlobClient(); var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString()); var uri = new Uri(blobFileArrived.BlobId); var id = string.Join("", uri.Segments.Skip(2)); var blob = container.GetBlockBlobReference(id); if (!blob.Exists()) { throw new InvalidOperationException("Blob does not exist: " + id); } var stream = new MemoryStream(); await blob.DownloadToStreamAsync(stream); var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser)); var hasAnything = false; var minDateTime = DateTimeOffset.MaxValue; foreach (var entity in parser.Parse(stream, blob.Uri, blobFileArrived.Position ?? 0, blobFileArrived.EndPosition ?? 0)) { await _pusher.PushAsync(entity, blobFileArrived.Source); hasAnything = true; minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime; } if (hasAnything) { await _pusher.FlushAsync(); _telemetryProvider.WriteTelemetry( "BlobFileActor log delay duration", (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds, blobFileArrived.Source.TypeName); } }, blobFileArrived.Source.TypeName); return(Enumerable.Empty <Event>()); }
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt) { var blobFileArrived = evnt.GetBody <BlobFileArrived>(); TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId, blobFileArrived.Source.TypeName); var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString); var client = account.CreateCloudBlobClient(); var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString()); var uri = new Uri(blobFileArrived.BlobId); var id = string.Join("", uri.Segments.Skip(2)); var blob = container.GetBlockBlobReference(id); if (!blob.Exists()) { throw new InvalidOperationException("Blob does not exist: " + id); } var stream = new MemoryStream(); await blob.DownloadToStreamAsync(stream); var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser)); bool hasAnything = false; foreach (var entity in parser.Parse(stream, blob.Uri, blobFileArrived.Position ?? 0, blobFileArrived.EndPosition ?? 0)) { await _pusher.PushAsync(entity, blobFileArrived.Source); hasAnything = true; } if (hasAnything) { await _pusher.FlushAsync(); } return(Enumerable.Empty <Event>()); }
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt) { var events = new List <Event>(); var blobFileScheduled = evnt.GetBody <BlobFileScheduled>(); var account = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString); var client = account.CreateCloudBlobClient(); var container = client.GetContainerReference("wad-iis-logfiles"); bool nextBlobExists; bool previousBlobExists; bool mainBlobExists; Func <string, string> alternater = s => s.Replace(".log", "_x.log"); var nextBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.NextFile, alternater, out nextBlobExists); var previousBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.PreviousFile, alternater, out previousBlobExists); var mainBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.FileToConsume, alternater, out mainBlobExists); if (!previousBlobExists && !mainBlobExists) { TheTrace.TraceInformation("BlobFileConventionActor - previous blob does not exist. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now); return(events); // will never be here. Stop chasing it. } long currentLength = 0; if (mainBlobExists) { currentLength = mainBlob.Properties.Length; if (currentLength == blobFileScheduled.LastPosition) { if (nextBlobExists) { TheTrace.TraceInformation("BlobFileConventionActor - Next blob exists. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now); return(events); // done and dusted. Stop chasing it. } if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now) { TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now); return(events); // Stop chasing it. } } else { var stream = await DownloadToFileAsync(mainBlob); try { currentLength = stream.Length; var parser = FactoryHelper.Create <IParser>(blobFileScheduled.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser)); bool hasAnything = false; foreach (var entity in parser.Parse(stream, mainBlob.Uri, blobFileScheduled.LastPosition)) { await _pusher.PushAsync(entity, blobFileScheduled.Source); hasAnything = true; } if (hasAnything) { await _pusher.FlushAsync(); TheTrace.TraceInformation("BlobFileConventionActor - pushed records for {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now); } } finally { stream.Close(); File.Delete(stream.Name); } } } else { if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now) { TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now); return(events); // Stop chasing it. } } blobFileScheduled.LastPosition = currentLength; // let's defer events.Add(new Event(blobFileScheduled) { EnqueueTime = DateTimeOffset.Now.Add(TimeSpan.FromSeconds(30)) }); TheTrace.TraceInformation("BlobFileConventionActor - deferred processing {0}. Length => {1} at {2}", blobFileScheduled.FileToConsume, currentLength, DateTimeOffset.Now); return(events); }